Dep/jemalloc: Update to 5.3.1

dep/PackageList.txt

@@ -18,7 +18,7 @@ G3D (a commercial-grade C++ 3D engine available as Open Source (BSD License)
 
 jemalloc (a general-purpose scalable concurrent malloc-implementation)
   https://github.com/jemalloc/jemalloc
-  Version: 5.3.0
+  Version: 5.3.1
 
 libreadline (command line editing library)
   https://cnswww.cns.cwru.edu/php/chet/readline/rltop.html

dep/jemalloc/CMakeLists.txt

@@ -42,17 +42,18 @@ else()
     set(CPU_SPINWAIT "_mm_pause()")
   else()
     set(CPU_SPINWAIT "__asm__ volatile\(\"pause\"\)")
+    set(JEMALLOC_HAVE_RDTSCP 1)
   endif()
 endif()
 
 # git describe --long --abbrev=40
-set(JEMALLOC_VERSION "5.3.0-0-g54eaed1d8b56b1aa528be3bdd1877e59c56fa90c")
+set(JEMALLOC_VERSION "5.3.1-12-gee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396")
 set(JEMALLOC_VERSION_MAJOR 5)
 set(JEMALLOC_VERSION_MINOR 3)
-set(JEMALLOC_VERSION_BUGFIX 0)
+set(JEMALLOC_VERSION_BUGFIX 1)
-set(JEMALLOC_VERSION_NREV 0)
+set(JEMALLOC_VERSION_NREV 12)
-set(JEMALLOC_VERSION_GID "54eaed1d8b56b1aa528be3bdd1877e59c56fa90c")
+set(JEMALLOC_VERSION_GID "ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396")
-set(JEMALLOC_VERSION_GID_IDENT 54eaed1d8b56b1aa528be3bdd1877e59c56fa90c)
+set(JEMALLOC_VERSION_GID_IDENT ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396)
 
 set(JEMALLOC_MAPS_COALESCE 1)
 set(JEMALLOC_TLS 1)
@@ -67,6 +68,9 @@ if(APPLE)
   set(JEMALLOC_ZONE 1)
   unset(JEMALLOC_TLS)
   check_symbol_exists(mach_absolute_time "mach/mach_time.h" JEMALLOC_HAVE_MACH_ABSOLUTE_TIME)
+  if(LG_SIZEOF_PTR STREQUAL "3")
+    set(JEMALLOC_RETAIN 1)
+  endif()
 elseif(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
   set(JEMALLOC_DSS 1)
   unset(JEMALLOC_TLS)
@@ -114,6 +118,7 @@ if(NOT MSVC)
   set(JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1) # don't bother detecting atomic support, all TC-supported compilers (excluding msvc) have gcc-like atomics
   set(JEMALLOC_HAVE_ATTR 1)
   set(JEMALLOC_TLS_MODEL "__attribute__\(\(tls_model\(\"initial-exec\"\)\)\)")
+  set(JEMALLOC_HAVE_ATTR_DEPRECATED 1)
   set(JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF 1)
   set(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF 1)
   set(JEMALLOC_HAVE_ATTR_FORMAT_ARG 1)
@@ -146,11 +151,15 @@ set(JEMALLOC_CONFIG_MALLOC_CONF "")
 set(JEMALLOC_C11_ATOMICS 1)
 set(JEMALLOC_ENABLE_CXX 1)
 set(JEMALLOC_FILL 1)
+set(JEMALLOC_HAVE_CXX_EXCEPTIONS 1)
 set(JEMALLOC_PRIVATE_NAMESPACE "je_")
 set(JEMALLOC_STATS 1)
+unset(JEMALLOC_CONFIG_ENV)
+unset(JEMALLOC_CONFIG_FILE)
 unset(JEMALLOC_LOG)
 unset(JEMALLOC_OPT_SAFETY_CHECKS)
 unset(JEMALLOC_OPT_SIZE_CHECKS)
+unset(JEMALLOC_PAGEID)
 unset(JEMALLOC_READLINKAT)
 unset(JEMALLOC_UAF_DETECTION)
 unset(JEMALLOC_UTRACE)
@@ -161,6 +170,7 @@ set(CMAKE_REQUIRED_DEFINITIONS ${JEMALLOC_EXTRA_DEFINES})
 
 check_symbol_exists(memalign "malloc.h" JEMALLOC_OVERRIDE_MEMALIGN)
 check_symbol_exists(valloc "stdlib.h" JEMALLOC_OVERRIDE_VALLOC)
+check_symbol_exists(pvalloc "malloc.h" JEMALLOC_OVERRIDE_PVALLOC)
 check_symbol_exists(malloc_size "malloc/malloc.h" JEMALLOC_HAVE_MALLOC_SIZE)
 
 set(wrap_syms "")
@@ -173,6 +183,14 @@ if(NOT JEMALLOC_PREFIX)
   if(JEMALLOC_OVERRIDE___LIBC_FREE)
     list(APPEND wrap_syms "__libc_free")
   endif()
+  check_function_exists(__libc_free_sized JEMALLOC_OVERRIDE___LIBC_FREE_SIZED)
+  if(JEMALLOC_OVERRIDE___LIBC_FREE_SIZED)
+    list(APPEND wrap_syms "__libc_free_sized")
+  endif()
+  check_function_exists(__libc_free_aligned_sized JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED)
+  if(JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED)
+    list(APPEND wrap_syms "__libc_free_aligned_sized")
+  endif()
   check_function_exists(__libc_malloc JEMALLOC_OVERRIDE___LIBC_MALLOC)
   if(JEMALLOC_OVERRIDE___LIBC_MALLOC)
     list(APPEND wrap_syms "__libc_malloc")
@@ -189,6 +207,10 @@ if(NOT JEMALLOC_PREFIX)
   if(JEMALLOC_OVERRIDE___LIBC_VALLOC)
     list(APPEND wrap_syms "__libc_valloc")
   endif()
+  check_function_exists(__libc_pvalloc JEMALLOC_OVERRIDE___LIBC_PVALLOC)
+  if(JEMALLOC_OVERRIDE___LIBC_PVALLOC)
+    list(APPEND wrap_syms "__libc_pvalloc")
+  endif()
   check_function_exists(__posix_memalign JEMALLOC_OVERRIDE___POSIX_MEMALIGN)
   if(JEMALLOC_OVERRIDE___POSIX_MEMALIGN)
     list(APPEND wrap_syms "__posix_memalign")
@@ -203,7 +225,8 @@ if(NOT WIN32)
   check_symbol_exists(pthread_atfork "pthread.h" JEMALLOC_HAVE_PTHREAD_ATFORK)
   check_symbol_exists(pthread_setname_np "pthread.h" JEMALLOC_HAVE_PTHREAD_SETNAME_NP)
   check_symbol_exists(pthread_getname_np "pthread.h" JEMALLOC_HAVE_PTHREAD_GETNAME_NP)
-  check_symbol_exists(pthread_get_name_np "pthread.h" JEMALLOC_HAVE_PTHREAD_GET_NAME_NP)
+  check_symbol_exists(pthread_set_name_np "pthread_np.h" JEMALLOC_HAVE_PTHREAD_SET_NAME_NP)
+  check_symbol_exists(pthread_get_name_np "pthread_np.h" JEMALLOC_HAVE_PTHREAD_GET_NAME_NP)
   check_c_source_compiles("
   #include <pthread.h>
   int main()
@@ -214,9 +237,11 @@ if(NOT WIN32)
     pthread_mutexattr_destroy(&attr);
   }
   " JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
+  check_symbol_exists(gettid "unistd.h" JEMALLOC_HAVE_GETTID)
   check_symbol_exists(CLOCK_MONOTONIC_COARSE "time.h" JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE)
   check_symbol_exists(CLOCK_MONOTONIC "time.h" JEMALLOC_HAVE_CLOCK_MONOTONIC)
   check_symbol_exists(CLOCK_REALTIME "time.h" JEMALLOC_HAVE_CLOCK_REALTIME)
+  check_symbol_exists(clock_gettime_nsec_np "time.h" JEMALLOC_HAVE_CLOCK_GETTIME_NSEC_NP)
 
   set(CMAKE_REQUIRED_FLAGS "-Werror")
   check_c_source_compiles("
@@ -229,6 +254,7 @@ if(NOT WIN32)
   check_symbol_exists(secure_getenv "stdlib.h" JEMALLOC_HAVE_SECURE_GETENV)
   check_symbol_exists(sched_getcpu "sched.h" JEMALLOC_HAVE_SCHED_GETCPU)
   check_symbol_exists(sched_setaffinity "sched.h" JEMALLOC_HAVE_SCHED_SETAFFINITY)
+  check_symbol_exists(pthread_setaffinity_np "pthread.h" JEMALLOC_HAVE_PTHREAD_SETAFFINITY_NP)
   check_symbol_exists(issetugid "unistd.h" JEMALLOC_HAVE_ISSETUGID)
   check_symbol_exists(_malloc_thread_cleanup "pthread.h" JEMALLOC_MALLOC_THREAD_CLEANUP)
   if(JEMALLOC_MALLOC_THREAD_CLEANUP)
@@ -244,6 +270,7 @@ if(NOT WIN32)
   endif()
 
   check_symbol_exists(memcntl "sys/types.h;sys/mman.h" JEMALLOC_HAVE_MEMCNTL)
+  check_symbol_exists(prctl "sys/prctl.h" JEMALLOC_HAVE_PRCTL)
   check_symbol_exists(madvise "sys/mman.h" JEMALLOC_HAVE_MADVISE)
   if(JEMALLOC_HAVE_MADVISE)
     check_symbol_exists(MADV_FREE "sys/mman.h" JEMALLOC_PURGE_MADVISE_FREE)
@@ -254,9 +281,11 @@ if(NOT WIN32)
     check_symbol_exists(MADV_DONTNEED "sys/mman.h" JEMALLOC_PURGE_MADVISE_DONTNEED)
     check_symbol_exists(MADV_DONTDUMP "sys/mman.h" JEMALLOC_MADVISE_DONTDUMP)
     check_symbol_exists(MADV_NOCORE "sys/mman.h" JEMALLOC_MADVISE_NOCORE)
+    check_symbol_exists(MADV_COLLAPSE "sys/mman.h" JEMALLOC_HAVE_MADVISE_COLLAPSE)
     if(TRINITY_SYSTEM_PROCESSOR MATCHES "x86|amd64")
       check_symbol_exists(MADV_HUGEPAGE "sys/mman.h" JEMALLOC_HAVE_MADVISE_HUGE)
     endif()
+    check_symbol_exists(SYS_process_madvise "sys/syscall.h" JEMALLOC_HAVE_PROCESS_MADVISE)
   else()
     check_symbol_exists(posix_madvise "sys/mman.h" JEMALLOC_HAVE_POSIX_MADVISE)
     if(JEMALLOC_HAVE_POSIX_MADVISE)
@@ -283,6 +312,13 @@ if(NOT WIN32)
   }
   " JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE)
   unset(CMAKE_REQUIRED_FLAGS)
+  check_c_source_compiles("
+  int main()
+  {
+    __int128 temp = 0;
+    return temp;
+  }
+  " JEMALLOC_HAVE_INT128)
 endif()
 
 unset(CMAKE_REQUIRED_DEFINITIONS)
@@ -312,6 +348,7 @@ add_library(jemalloc STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/src/buf_writer.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/cache_bin.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/ckh.c 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/conf.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/counter.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/ctl.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/decay.c 
@@ -331,7 +368,9 @@ add_library(jemalloc STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/src/san_bump.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/hook.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa.c 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa_central.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa_hooks.c 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa_utils.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/hpdata.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/inspect.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/large.c 
@@ -341,7 +380,6 @@ add_library(jemalloc STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/src/nstime.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/pa.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/pa_extra.c 
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/pai.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/pac.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/pages.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/peak_event.c 
@@ -349,6 +387,7 @@ add_library(jemalloc STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_data.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_log.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_recent.c 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_stack_range.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_stats.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_sys.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/psset.c 
@@ -361,8 +400,10 @@ add_library(jemalloc STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/src/tcache.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/test_hooks.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/thread_event.c 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/thread_event_registry.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/ticker.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/tsd.c 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/util.c 
   ${CMAKE_CURRENT_SOURCE_DIR}/src/witness.c)
 
 if(APPLE)

dep/jemalloc/include/jemalloc/internal/activity_callback.h

@@ -1,23 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H
-#define JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H
-
-/*
- * The callback to be executed "periodically", in response to some amount of
- * allocator activity.
- *
- * This callback need not be computing any sort of peak (although that's the
- * intended first use case), but we drive it from the peak counter, so it's
- * keeps things tidy to keep it here.
- *
- * The calls to this thunk get driven by the peak_event module.
- */
-#define ACTIVITY_CALLBACK_THUNK_INITIALIZER {NULL, NULL}
-typedef void (*activity_callback_t)(void *uctx, uint64_t allocated,
-    uint64_t deallocated);
-typedef struct activity_callback_thunk_s activity_callback_thunk_t;
-struct activity_callback_thunk_s {
-	activity_callback_t callback;
-	void *uctx;
-};
-
-#endif /* JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H */

dep/jemalloc/include/jemalloc/internal/arena_externs.h

@@ -1,8 +1,11 @@
 #ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H
 #define JEMALLOC_INTERNAL_ARENA_EXTERNS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_stats.h"
 #include "jemalloc/internal/bin.h"
 #include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/extent_dss.h"
 #include "jemalloc/internal/hook.h"
 #include "jemalloc/internal/pages.h"
@@ -18,104 +21,105 @@ extern ssize_t opt_dirty_decay_ms;
 extern ssize_t opt_muzzy_decay_ms;
 
 extern percpu_arena_mode_t opt_percpu_arena;
-extern const char *percpu_arena_mode_names[];
+extern const char *const   percpu_arena_mode_names[];
 
 extern div_info_t arena_binind_div_info[SC_NBINS];
 
-extern malloc_mutex_t arenas_lock;
 extern emap_t arena_emap_global;
 
 extern size_t opt_oversize_threshold;
 extern size_t oversize_threshold;
 
+extern bool      opt_huge_arena_pac_thp;
+extern pac_thp_t huge_arena_pac_thp;
+
 /*
  * arena_bin_offsets[binind] is the offset of the first bin shard for size class
  * binind.
  */
 extern uint32_t arena_bin_offsets[SC_NBINS];
 
-void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
+void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
-    unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms,
+    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
-    ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
+    size_t *nactive, size_t *ndirty, size_t *nmuzzy);
 void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
-    bin_stats_data_t *bstats, arena_stats_large_t *lstats,
+    bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
-    pac_estats_t *estats, hpa_shard_stats_t *hpastats, sec_stats_t *secstats);
+    hpa_shard_stats_t *hpastats);
 void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
-edata_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
+edata_t *arena_extent_alloc_large(
-    size_t usize, size_t alignment, bool zero);
+    tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
-void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
+void arena_extent_dalloc_large_prep(
-    edata_t *edata);
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata);
-void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
+void arena_extent_ralloc_large_shrink(
-    edata_t *edata, size_t oldsize);
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
-void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
+void arena_extent_ralloc_large_expand(
-    edata_t *edata, size_t oldsize);
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
-bool arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, extent_state_t state,
+bool arena_decay_ms_set(
-    ssize_t decay_ms);
+    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
 ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
-void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
+void    arena_decay(
-    bool all);
+       tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
+uint64_t       arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
-void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
+void           arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
-void arena_reset(tsd_t *tsd, arena_t *arena);
+void           arena_reset(tsd_t *tsd, arena_t *arena);
-void arena_destroy(tsd_t *tsd, arena_t *arena);
+void           arena_destroy(tsd_t *tsd, arena_t *arena);
-void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
+cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
-    cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind,
+    szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
-    const unsigned nfill);
+    const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
 
-void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
+void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
-    szind_t ind, bool zero);
+    bool zero, bool slab);
-void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-    size_t alignment, bool zero, tcache_t *tcache);
+    bool zero, bool slab, tcache_t *tcache);
-void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize);
+void  arena_prof_promote(
-void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+     tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
-    bool slow_path);
+void arena_dalloc_promoted(
+    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
 void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
 
-void arena_dalloc_bin_locked_handle_newly_empty(tsdn_t *tsdn, arena_t *arena,
+void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
-    edata_t *slab, bin_t *bin);
+void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
-void arena_dalloc_bin_locked_handle_newly_nonempty(tsdn_t *tsdn, arena_t *arena,
+     cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
-    edata_t *slab, bin_t *bin);
+     arena_t *stats_arena, cache_bin_stats_t merge_stats);
-void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
+bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
+     size_t extra, bool zero, size_t *newsize);
-    size_t extra, bool zero, size_t *newsize);
 void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, tcache_t *tcache,
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
     hook_ralloc_args_t *hook_args);
-dss_prec_t arena_dss_prec_get(arena_t *arena);
+dss_prec_t      arena_dss_prec_get(arena_t *arena);
-ehooks_t *arena_get_ehooks(arena_t *arena);
+ehooks_t       *arena_get_ehooks(arena_t *arena);
-extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
+extent_hooks_t *arena_set_extent_hooks(
-    extent_hooks_t *extent_hooks);
+    tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
-bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
+bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
+void    arena_name_get(arena_t *arena, char *name);
+void    arena_name_set(arena_t *arena, const char *name);
 ssize_t arena_dirty_decay_ms_default_get(void);
-bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
+bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
 ssize_t arena_muzzy_decay_ms_default_get(void);
-bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
+bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
-bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena,
+bool    arena_retain_grow_limit_get_set(
-    size_t *old_limit, size_t *new_limit);
+       tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
 unsigned arena_nthreads_get(arena_t *arena, bool internal);
-void arena_nthreads_inc(arena_t *arena, bool internal);
+void     arena_nthreads_inc(arena_t *arena, bool internal);
-void arena_nthreads_dec(arena_t *arena, bool internal);
+void     arena_nthreads_dec(arena_t *arena, bool internal);
 arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
-bool arena_init_huge(void);
+bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
-bool arena_is_huge(unsigned arena_ind);
 arena_t *arena_choose_huge(tsd_t *tsd);
-bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
-    unsigned *binshard);
 size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     void **ptrs, size_t nfill, bool zero);
-bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
+bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
-void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork3(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork3(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork4(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork4(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork5(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork5(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork6(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork7(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
-void arena_prefork8(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
-void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
-void arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
 
 #endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

dep/jemalloc/include/jemalloc/internal/arena_inlines_a.h

@@ -1,6 +1,9 @@
 #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H
 #define JEMALLOC_INTERNAL_ARENA_INLINES_A_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_structs.h"
+
 static inline unsigned
 arena_ind_get(const arena_t *arena) {
 	return arena->ind;

dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h

@@ -1,20 +1,29 @@
 #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
 #define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena_structs.h"
+#include "jemalloc/internal/bin_inlines.h"
 #include "jemalloc/internal/div.h"
 #include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large_externs.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof_externs.h"
+#include "jemalloc/internal/prof_structs.h"
 #include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache_inlines.h"
 #include "jemalloc/internal/ticker.h"
 
 static inline arena_t *
 arena_get_from_edata(edata_t *edata) {
-	return (arena_t *)atomic_load_p(&arenas[edata_arena_ind_get(edata)],
+	return (arena_t *)atomic_load_p(
-	    ATOMIC_RELAXED);
+	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
 }
 
 JEMALLOC_ALWAYS_INLINE arena_t *
@@ -28,14 +37,48 @@ arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
 	 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
 	 * thread is not assigned to a manual arena.
 	 */
-	if (unlikely(size >= oversize_threshold)) {
+	arena_t *tsd_arena = tsd_arena_get(tsd);
-		arena_t *tsd_arena = tsd_arena_get(tsd);
+	if (tsd_arena == NULL) {
-		if (tsd_arena == NULL || arena_is_auto(tsd_arena)) {
+		tsd_arena = arena_choose(tsd, NULL);
-			return arena_choose_huge(tsd);
-		}
 	}
 
-	return arena_choose(tsd, NULL);
+	size_t threshold = atomic_load_zu(
+	    &tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
+	if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
+		return arena_choose_huge(tsd);
+	}
+
+	return tsd_arena;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
+	if (!config_opt_safety_checks) {
+		return false;
+	}
+
+	/*
+	 * Eagerly detect double free and sized dealloc bugs for large sizes.
+	 * The cost is low enough (as edata will be accessed anyway) to be
+	 * enabled all the time.
+	 */
+	if (unlikely(edata == NULL
+	        || edata_state_get(edata) != extent_state_active)) {
+		safety_check_fail(
+		    "Invalid deallocation detected: "
+		    "pages being freed (%p) not currently active, "
+		    "possibly caused by double free bugs.",
+		    ptr);
+		return true;
+	}
+	if (unlikely(input_size != edata_usize_get(edata)
+	        || input_size > SC_LARGE_MAXCLASS)) {
+		safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
+		    /* true_size */ edata_usize_get(edata), input_size);
+		return true;
+	}
+
+	return false;
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -46,48 +89,56 @@ arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
 	assert(prof_info != NULL);
 
 	edata_t *edata = NULL;
-	bool is_slab;
+	bool     is_slab;
 
 	/* Static check. */
 	if (alloc_ctx == NULL) {
-		edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global,
+		edata = emap_edata_lookup(
-		    ptr);
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
 		is_slab = edata_slab_get(edata);
 	} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
-		edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global,
+		edata = emap_edata_lookup(
-		    ptr);
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
 	}
 
 	if (unlikely(!is_slab)) {
 		/* edata must have been initialized at this point. */
 		assert(edata != NULL);
+		size_t usize = (alloc_ctx == NULL)
+		    ? edata_usize_get(edata)
+		    : emap_alloc_ctx_usize_get(alloc_ctx);
+		if (reset_recent
+		    && large_dalloc_safety_checks(edata, ptr, usize)) {
+			prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
+			return;
+		}
 		large_prof_info_get(tsd, edata, prof_info, reset_recent);
 	} else {
-		prof_info->alloc_tctx = (prof_tctx_t *)(uintptr_t)1U;
+		prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
 		/*
 		 * No need to set other fields in prof_info; they will never be
-		 * accessed if (uintptr_t)alloc_tctx == (uintptr_t)1U.
+		 * accessed if alloc_tctx == PROF_TCTX_SENTINEL.
 		 */
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void
-arena_prof_tctx_reset(tsd_t *tsd, const void *ptr,
+arena_prof_tctx_reset(
-    emap_alloc_ctx_t *alloc_ctx) {
+    tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
 	/* Static check. */
 	if (alloc_ctx == NULL) {
-		edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd),
+		edata_t *edata = emap_edata_lookup(
-		    &arena_emap_global, ptr);
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
 		if (unlikely(!edata_slab_get(edata))) {
 			large_prof_tctx_reset(edata);
 		}
 	} else {
 		if (unlikely(!alloc_ctx->slab)) {
-			edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd),
+			edata_t *edata = emap_edata_lookup(
-			    &arena_emap_global, ptr);
+			    tsd_tsdn(tsd), &arena_emap_global, ptr);
 			large_prof_tctx_reset(edata);
 		}
 	}
@@ -98,16 +149,16 @@ arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global,
+	edata_t *edata = emap_edata_lookup(
-	    ptr);
+	    tsd_tsdn(tsd), &arena_emap_global, ptr);
 	assert(!edata_slab_get(edata));
 
 	large_prof_tctx_reset(edata);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-arena_prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx,
+arena_prof_info_set(
-    size_t size) {
+    tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
 	cassert(config_prof);
 
 	assert(!edata_slab_get(edata));
@@ -130,8 +181,9 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
 	 * use a single ticker for all of them.
 	 */
 	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
-	uint64_t *prng_state = tsd_prng_statep_get(tsd);
+	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
-	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks))) {
+	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
+	        tsd_reentrancy_level_get(tsd) > 0))) {
 		arena_decay(tsdn, arena, false, false);
 	}
 }
@@ -143,23 +195,24 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
 
 JEMALLOC_ALWAYS_INLINE void *
 arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
-    tcache_t *tcache, bool slow_path) {
+    bool slab, tcache_t *tcache, bool slow_path) {
 	assert(!tsdn_null(tsdn) || tcache == NULL);
 
 	if (likely(tcache != NULL)) {
-		if (likely(size <= SC_SMALL_MAXCLASS)) {
+		if (likely(slab)) {
-			return tcache_alloc_small(tsdn_tsd(tsdn), arena,
+			assert(sz_can_use_slab(size));
-			    tcache, size, ind, zero, slow_path);
+			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		} else if (likely(ind < tcache_nbins_get(tcache->tcache_slow)
+		               && !tcache_bin_disabled(ind, &tcache->bins[ind],
+		                   tcache->tcache_slow))) {
+			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
 		}
-		if (likely(size <= tcache_maxclass)) {
+		/* (size > tcache_max) case falls through. */
-			return tcache_alloc_large(tsdn_tsd(tsdn), arena,
-			    tcache, size, ind, zero, slow_path);
-		}
-		/* (size > tcache_maxclass) case falls through. */
-		assert(size > tcache_maxclass);
 	}
 
-	return arena_malloc_hard(tsdn, arena, size, ind, zero);
+	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
 }
 
 JEMALLOC_ALWAYS_INLINE arena_t *
@@ -176,7 +229,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) {
 	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
 	assert(alloc_ctx.szind != SC_NSIZES);
 
-	return sz_index2size(alloc_ctx.szind);
+	return emap_alloc_ctx_usize_get(&alloc_ctx);
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
@@ -191,8 +244,8 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
 	 */
 
 	emap_full_alloc_ctx_t full_alloc_ctx;
-	bool missing = emap_full_alloc_ctx_try_lookup(tsdn, &arena_emap_global,
+	bool                  missing = emap_full_alloc_ctx_try_lookup(
-	    ptr, &full_alloc_ctx);
+            tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
 	if (missing) {
 		return 0;
 	}
@@ -207,46 +260,24 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
 
 	assert(full_alloc_ctx.szind != SC_NSIZES);
 
-	return sz_index2size(full_alloc_ctx.szind);
+	return edata_usize_get(full_alloc_ctx.edata);
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-large_dalloc_safety_checks(edata_t *edata, void *ptr, szind_t szind) {
-	if (!config_opt_safety_checks) {
-		return false;
-	}
-
-	/*
-	 * Eagerly detect double free and sized dealloc bugs for large sizes.
-	 * The cost is low enough (as edata will be accessed anyway) to be
-	 * enabled all the time.
-	 */
-	if (unlikely(edata == NULL ||
-	    edata_state_get(edata) != extent_state_active)) {
-		safety_check_fail("Invalid deallocation detected: "
-		    "pages being freed (%p) not currently active, "
-		    "possibly caused by double free bugs.",
-		    (uintptr_t)edata_addr_get(edata));
-		return true;
-	}
-	size_t input_size = sz_index2size(szind);
-	if (unlikely(input_size != edata_usize_get(edata))) {
-		safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
-		    /* true_size */ edata_usize_get(edata), input_size);
-		return true;
-	}
-
-	return false;
 }
 
 static inline void
-arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) {
+arena_dalloc_large_no_tcache(
+    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
+	/*
+	 * szind is still needed in this function mainly becuase
+	 * szind < SC_NBINS determines not only if this is a small alloc,
+	 * but also if szind is valid (an inactive extent would have
+	 * szind == SC_NSIZES).
+	 */
 	if (config_prof && unlikely(szind < SC_NBINS)) {
 		arena_dalloc_promoted(tsdn, ptr, NULL, true);
 	} else {
-		edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global,
+		edata_t *edata = emap_edata_lookup(
-		    ptr);
+		    tsdn, &arena_emap_global, ptr);
-		if (large_dalloc_safety_checks(edata, ptr, szind)) {
+		if (large_dalloc_safety_checks(edata, ptr, usize)) {
 			/* See the comment in isfree. */
 			return;
 		}
@@ -262,42 +293,76 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
 	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
 
 	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global,
+		edata_t *edata = emap_edata_lookup(
-		    ptr);
+		    tsdn, &arena_emap_global, ptr);
 		assert(alloc_ctx.szind == edata_szind_get(edata));
 		assert(alloc_ctx.szind < SC_NSIZES);
 		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
 	}
 
 	if (likely(alloc_ctx.slab)) {
 		/* Small allocation. */
 		arena_dalloc_small(tsdn, ptr);
 	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind);
+		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void
 arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
-    bool slow_path) {
+    size_t usize, bool slow_path) {
-	if (szind < nhbins) {
+	assert(!tsdn_null(tsdn) && tcache != NULL);
-		if (config_prof && unlikely(szind < SC_NBINS)) {
+	bool is_sample_promoted = config_prof && szind < SC_NBINS;
-			arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
+	if (unlikely(is_sample_promoted)) {
-		} else {
+		arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
-			tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind,
-			    slow_path);
-		}
 	} else {
-		edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global,
+		if (szind < tcache_nbins_get(tcache->tcache_slow)
-		    ptr);
+		    && !tcache_bin_disabled(
-		if (large_dalloc_safety_checks(edata, ptr, szind)) {
+		        szind, &tcache->bins[szind], tcache->tcache_slow)) {
-			/* See the comment in isfree. */
+			tcache_dalloc_large(
-			return;
+			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
+		} else {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			if (large_dalloc_safety_checks(edata, ptr, usize)) {
+				/* See the comment in isfree. */
+				return;
+			}
+			large_dalloc(tsdn, edata);
 		}
-		large_dalloc(tsdn, edata);
 	}
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
+	if (!config_debug) {
+		return false;
+	}
+	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	szind_t    binind = edata_szind_get(edata);
+	div_info_t div_info = arena_binind_div_info[binind];
+	/*
+	 * Calls the internal function bin_slab_regind_impl because the
+	 * safety check does not require a lock.
+	 */
+	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
+	slab_data_t      *slab_data = edata_slab_data_get(edata);
+	const bin_info_t *bin_info = &bin_infos[binind];
+	assert(edata_nfree_get(edata) < bin_info->nregs);
+	if (unlikely(!bitmap_get(
+	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
+		safety_check_fail(
+		    "Invalid deallocation detected: the pointer being freed (%p) not "
+		    "currently active, possibly caused by double free bugs.\n",
+		    ptr);
+		return true;
+	}
+	return false;
+}
+
 JEMALLOC_ALWAYS_INLINE void
 arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
     emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
@@ -313,26 +378,31 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
 	if (caller_alloc_ctx != NULL) {
 		alloc_ctx = *caller_alloc_ctx;
 	} else {
-		util_assume(!tsdn_null(tsdn));
+		util_assume(tsdn != NULL);
-		emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr,
+		emap_alloc_ctx_lookup(
-		    &alloc_ctx);
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
 	}
 
 	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global,
+		edata_t *edata = emap_edata_lookup(
-		    ptr);
+		    tsdn, &arena_emap_global, ptr);
 		assert(alloc_ctx.szind == edata_szind_get(edata));
 		assert(alloc_ctx.szind < SC_NSIZES);
 		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
 	}
 
 	if (likely(alloc_ctx.slab)) {
 		/* Small allocation. */
-		tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr,
+		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-		    alloc_ctx.szind, slow_path);
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
 	} else {
 		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    slow_path);
+		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
 	}
 }
 
@@ -347,21 +417,22 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
 		 * There is no risk of being confused by a promoted sampled
 		 * object, so base szind and slab on the given size.
 		 */
-		alloc_ctx.szind = sz_size2index(size);
+		szind_t szind = sz_size2index(size);
-		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
+		emap_alloc_ctx_init(
+		    &alloc_ctx, szind, (szind < SC_NBINS), size);
 	}
 
 	if ((config_prof && opt_prof) || config_debug) {
-		emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr,
+		emap_alloc_ctx_lookup(
-		    &alloc_ctx);
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
 
 		assert(alloc_ctx.szind == sz_size2index(size));
 		assert((config_prof && opt_prof)
 		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
 
 		if (config_debug) {
-			edata_t *edata = emap_edata_lookup(tsdn,
+			edata_t *edata = emap_edata_lookup(
-			    &arena_emap_global, ptr);
+			    tsdn, &arena_emap_global, ptr);
 			assert(alloc_ctx.szind == edata_szind_get(edata));
 			assert(alloc_ctx.slab == edata_slab_get(edata));
 		}
@@ -371,7 +442,8 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
 		/* Small allocation. */
 		arena_dalloc_small(tsdn, ptr);
 	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind);
+		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
 	}
 }
 
@@ -391,9 +463,10 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
 	if (config_prof && opt_prof) {
 		if (caller_alloc_ctx == NULL) {
 			/* Uncommon case and should be a static check. */
-			emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr,
+			emap_alloc_ctx_lookup(
-			    &alloc_ctx);
+			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
 			assert(alloc_ctx.szind == sz_size2index(size));
+			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
 		} else {
 			alloc_ctx = *caller_alloc_ctx;
 		}
@@ -407,30 +480,37 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
 	}
 
 	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global,
+		edata_t *edata = emap_edata_lookup(
-		    ptr);
+		    tsdn, &arena_emap_global, ptr);
 		assert(alloc_ctx.szind == edata_szind_get(edata));
 		assert(alloc_ctx.slab == edata_slab_get(edata));
+		emap_alloc_ctx_init(
+		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
 	}
 
 	if (likely(alloc_ctx.slab)) {
 		/* Small allocation. */
-		tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr,
+		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-		    alloc_ctx.szind, slow_path);
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
 	} else {
 		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    slow_path);
+		    sz_s2u(size), slow_path);
 	}
 }
 
 static inline void
-arena_cache_oblivious_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
+arena_cache_oblivious_randomize(
-    size_t alignment) {
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
 	assert(edata_base_get(edata) == edata_addr_get(edata));
 
 	if (alignment < PAGE) {
-		unsigned lg_range = LG_PAGE -
+		unsigned lg_range = LG_PAGE
-		    lg_floor(CACHELINE_CEILING(alignment));
+		    - lg_floor(CACHELINE_CEILING(alignment));
 		size_t r;
 		if (!tsdn_null(tsdn)) {
 			tsd_t *tsd = tsdn_tsd(tsdn);
@@ -440,110 +520,18 @@ arena_cache_oblivious_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
 			uint64_t stack_value = (uint64_t)(uintptr_t)&r;
 			r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
 		}
-		uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
+		uintptr_t random_offset = ((uintptr_t)r)
-		    lg_range);
+		    << (LG_PAGE - lg_range);
-		edata->e_addr = (void *)((uintptr_t)edata->e_addr +
+		edata->e_addr = (void *)((byte_t *)edata->e_addr
-		    random_offset);
+		    + random_offset);
-		assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment) ==
+		assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
-		    edata->e_addr);
+		    == edata->e_addr);
-	}
-}
-
-/*
- * The dalloc bin info contains just the information that the common paths need
- * during tcache flushes.  By force-inlining these paths, and using local copies
- * of data (so that the compiler knows it's constant), we avoid a whole bunch of
- * redundant loads and stores by leaving this information in registers.
- */
-typedef struct arena_dalloc_bin_locked_info_s arena_dalloc_bin_locked_info_t;
-struct arena_dalloc_bin_locked_info_s {
-	div_info_t div_info;
-	uint32_t nregs;
-	uint64_t ndalloc;
-};
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_slab_regind(arena_dalloc_bin_locked_info_t *info, szind_t binind,
-    edata_t *slab, const void *ptr) {
-	size_t diff, regind;
-
-	/* Freeing a pointer outside the slab can cause assertion failure. */
-	assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab));
-	assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab));
-	/* Freeing an interior pointer can cause assertion failure. */
-	assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)) %
-	    (uintptr_t)bin_infos[binind].reg_size == 0);
-
-	diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab));
-
-	/* Avoid doing division with a variable divisor. */
-	regind = div_compute(&info->div_info, diff);
-
-	assert(regind < bin_infos[binind].nregs);
-
-	return regind;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc_bin_locked_begin(arena_dalloc_bin_locked_info_t *info,
-    szind_t binind) {
-	info->div_info = arena_binind_div_info[binind];
-	info->nregs = bin_infos[binind].nregs;
-	info->ndalloc = 0;
-}
-
-/*
- * Does the deallocation work associated with freeing a single pointer (a
- * "step") in between a arena_dalloc_bin_locked begin and end call.
- *
- * Returns true if arena_slab_dalloc must be called on slab.  Doesn't do
- * stats updates, which happen during finish (this lets running counts get left
- * in a register).
- */
-JEMALLOC_ALWAYS_INLINE bool
-arena_dalloc_bin_locked_step(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
-    arena_dalloc_bin_locked_info_t *info, szind_t binind, edata_t *slab,
-    void *ptr) {
-	const bin_info_t *bin_info = &bin_infos[binind];
-	size_t regind = arena_slab_regind(info, binind, slab, ptr);
-	slab_data_t *slab_data = edata_slab_data_get(slab);
-
-	assert(edata_nfree_get(slab) < bin_info->nregs);
-	/* Freeing an unallocated pointer can cause assertion failure. */
-	assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));
-
-	bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);
-	edata_nfree_inc(slab);
-
-	if (config_stats) {
-		info->ndalloc++;
-	}
-
-	unsigned nfree = edata_nfree_get(slab);
-	if (nfree == bin_info->nregs) {
-		arena_dalloc_bin_locked_handle_newly_empty(tsdn, arena, slab,
-		    bin);
-		return true;
-	} else if (nfree == 1 && slab != bin->slabcur) {
-		arena_dalloc_bin_locked_handle_newly_nonempty(tsdn, arena, slab,
-		    bin);
-	}
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc_bin_locked_finish(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
-    arena_dalloc_bin_locked_info_t *info) {
-	if (config_stats) {
-		bin->stats.ndalloc += info->ndalloc;
-		assert(bin->stats.curregs >= (size_t)info->ndalloc);
-		bin->stats.curregs -= (size_t)info->ndalloc;
 	}
 }
 
 static inline bin_t *
 arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
-	bin_t *shard0 = (bin_t *)((uintptr_t)arena + arena_bin_offsets[binind]);
+	bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
 	return shard0 + binshard;
 }
 

dep/jemalloc/include/jemalloc/internal/arena_stats.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_ARENA_STATS_H
 #define JEMALLOC_INTERNAL_ARENA_STATS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/lockedint.h"
 #include "jemalloc/internal/mutex.h"
@@ -13,28 +14,34 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 typedef struct arena_stats_large_s arena_stats_large_t;
 struct arena_stats_large_s {
 	/*
-	 * Total number of allocation/deallocation requests served directly by
+	 * Total number of large allocation/deallocation requests served directly
-	 * the arena.
+	 * by the arena.
 	 */
-	locked_u64_t	nmalloc;
+	locked_u64_t nmalloc;
-	locked_u64_t	ndalloc;
+	locked_u64_t ndalloc;
+
+	/*
+	 * Total large active bytes (allocated - deallocated) served directly
+	 * by the arena.
+	 */
+	locked_u64_t active_bytes;
 
 	/*
 	 * Number of allocation requests that correspond to this size class.
 	 * This includes requests served by tcache, though tcache only
 	 * periodically merges into this counter.
 	 */
-	locked_u64_t	nrequests; /* Partially derived. */
+	locked_u64_t nrequests; /* Partially derived. */
 	/*
 	 * Number of tcache fills / flushes for large (similarly, periodically
 	 * merged).  Note that there is no large tcache batch-fill currently
 	 * (i.e. only fill 1 at a time); however flush may be batched.
 	 */
-	locked_u64_t	nfills; /* Partially derived. */
+	locked_u64_t nfills;   /* Partially derived. */
-	locked_u64_t	nflushes; /* Partially derived. */
+	locked_u64_t nflushes; /* Partially derived. */
 
 	/* Current number of allocations of this size class. */
-	size_t		curlextents; /* Derived. */
+	size_t curlextents; /* Derived. */
 };
 
 /*
@@ -50,38 +57,40 @@ struct arena_stats_s {
 	 * resident includes the base stats -- that's why it lives here and not
 	 * in pa_shard_stats_t.
 	 */
-	size_t			base; /* Derived. */
+	size_t base;           /* Derived. */
-	size_t			resident; /* Derived. */
+	size_t metadata_edata; /* Derived. */
-	size_t			metadata_thp; /* Derived. */
+	size_t metadata_rtree; /* Derived. */
-	size_t			mapped; /* Derived. */
+	size_t resident;       /* Derived. */
+	size_t metadata_thp;   /* Derived. */
+	size_t mapped;         /* Derived. */
 
-	atomic_zu_t		internal;
+	atomic_zu_t internal;
 
-	size_t			allocated_large; /* Derived. */
+	size_t   allocated_large; /* Derived. */
-	uint64_t		nmalloc_large; /* Derived. */
+	uint64_t nmalloc_large;   /* Derived. */
-	uint64_t		ndalloc_large; /* Derived. */
+	uint64_t ndalloc_large;   /* Derived. */
-	uint64_t		nfills_large; /* Derived. */
+	uint64_t nfills_large;    /* Derived. */
-	uint64_t		nflushes_large; /* Derived. */
+	uint64_t nflushes_large;  /* Derived. */
-	uint64_t		nrequests_large; /* Derived. */
+	uint64_t nrequests_large; /* Derived. */
 
 	/*
 	 * The stats logically owned by the pa_shard in the same arena.  This
 	 * lives here only because it's convenient for the purposes of the ctl
 	 * module -- it only knows about the single arena_stats.
 	 */
-	pa_shard_stats_t	pa_shard_stats;
+	pa_shard_stats_t pa_shard_stats;
 
 	/* Number of bytes cached in tcache associated with this arena. */
-	size_t			tcache_bytes; /* Derived. */
+	size_t tcache_bytes;         /* Derived. */
-	size_t			tcache_stashed_bytes; /* Derived. */
+	size_t tcache_stashed_bytes; /* Derived. */
 
 	mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
 
 	/* One element for each large size class. */
-	arena_stats_large_t	lstats[SC_NSIZES - SC_NBINS];
+	arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
 
 	/* Arena uptime. */
-	nstime_t		uptime;
+	nstime_t uptime;
 };
 
 static inline bool
@@ -92,7 +101,7 @@ arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) {
 		}
 	}
 	if (LOCKEDINT_MTX_INIT(arena_stats->mtx, "arena_stats",
-	    WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
+	        WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
 		return true;
 	}
 	/* Memory is zeroed, so there is no need to clear stats. */
@@ -106,8 +115,8 @@ arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
 	arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS];
 	locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx),
 	    &lstats->nrequests, nrequests);
-	locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx),
+	locked_inc_u64(
-	    &lstats->nflushes, 1);
+	    tsdn, LOCKEDINT_MTX(arena_stats->mtx), &lstats->nflushes, 1);
 	LOCKEDINT_MTX_UNLOCK(tsdn, arena_stats->mtx);
 }
 

dep/jemalloc/include/jemalloc/internal/arena_structs.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_H
 #define JEMALLOC_INTERNAL_ARENA_STRUCTS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_stats.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bin.h"
@@ -31,20 +32,20 @@ struct arena_s {
 	 *
 	 * Synchronization: atomic.
 	 */
-	atomic_u_t		nthreads[2];
+	atomic_u_t nthreads[2];
 
 	/* Next bin shard for binding new threads. Synchronization: atomic. */
-	atomic_u_t		binshard_next;
+	atomic_u_t binshard_next;
 
 	/*
 	 * When percpu_arena is enabled, to amortize the cost of reading /
 	 * updating the current CPU id, track the most recent thread accessing
 	 * this arena, and only read CPU if there is a mismatch.
 	 */
-	tsdn_t		*last_thd;
+	tsdn_t *last_thd;
 
 	/* Synchronization: internal. */
-	arena_stats_t		stats;
+	arena_stats_t stats;
 
 	/*
 	 * Lists of tcaches and cache_bin_array_descriptors for extant threads
@@ -53,28 +54,28 @@ struct arena_s {
 	 *
 	 * Synchronization: tcache_ql_mtx.
 	 */
-	ql_head(tcache_slow_t)			tcache_ql;
+	ql_head(tcache_slow_t) tcache_ql;
-	ql_head(cache_bin_array_descriptor_t)	cache_bin_array_descriptor_ql;
+	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
-	malloc_mutex_t				tcache_ql_mtx;
+	malloc_mutex_t tcache_ql_mtx;
 
 	/*
 	 * Represents a dss_prec_t, but atomically.
 	 *
 	 * Synchronization: atomic.
 	 */
-	atomic_u_t		dss_prec;
+	atomic_u_t dss_prec;
 
 	/*
 	 * Extant large allocations.
 	 *
 	 * Synchronization: large_mtx.
 	 */
-	edata_list_active_t	large;
+	edata_list_active_t large;
 	/* Synchronizes all large allocation/update/deallocation. */
-	malloc_mutex_t		large_mtx;
+	malloc_mutex_t large_mtx;
 
 	/* The page-level allocator shard this arena uses. */
-	pa_shard_t		pa_shard;
+	pa_shard_t pa_shard;
 
 	/*
 	 * A cached copy of base->ind.  This can get accessed on hot paths;
@@ -87,15 +88,24 @@ struct arena_s {
 	 *
 	 * Synchronization: internal.
 	 */
-	base_t			*base;
+	base_t *base;
 	/* Used to determine uptime.  Read-only after initialization. */
-	nstime_t		create_time;
+	nstime_t create_time;
+
+	/* The name of the arena. */
+	char name[ARENA_NAME_LEN];
 
 	/*
 	 * The arena is allocated alongside its bins; really this is a
 	 * dynamically sized array determined by the binshard settings.
+	 * Enforcing cacheline-alignment to minimize the number of cachelines
+	 * touched on the hot paths.
 	 */
-	bin_t			bins[0];
+	JEMALLOC_WARN_ON_USAGE(
+	    "Do not use this field directly. "
+	    "Use `arena_get_bin` instead.")
+	JEMALLOC_ALIGNED(CACHELINE)
+	bin_t all_bins[0];
 };
 
 #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */

dep/jemalloc/include/jemalloc/internal/arena_types.h

@@ -1,39 +1,41 @@
 #ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H
 #define JEMALLOC_INTERNAL_ARENA_TYPES_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/sc.h"
 
 /* Default decay times in milliseconds. */
-#define DIRTY_DECAY_MS_DEFAULT	ZD(10 * 1000)
+#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
-#define MUZZY_DECAY_MS_DEFAULT	(0)
+#define MUZZY_DECAY_MS_DEFAULT (0)
 /* Number of event ticks between time checks. */
-#define ARENA_DECAY_NTICKS_PER_UPDATE	1000
+#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
+/* Maximum length of the arena name. */
+#define ARENA_NAME_LEN 32
 
-typedef struct arena_decay_s arena_decay_t;
 typedef struct arena_s arena_t;
 
 typedef enum {
-	percpu_arena_mode_names_base   = 0, /* Used for options processing. */
+	percpu_arena_mode_names_base = 0, /* Used for options processing. */
 
 	/*
 	 * *_uninit are used only during bootstrapping, and must correspond
 	 * to initialized variant plus percpu_arena_mode_enabled_base.
 	 */
-	percpu_arena_uninit            = 0,
+	percpu_arena_uninit = 0,
-	per_phycpu_arena_uninit        = 1,
+	per_phycpu_arena_uninit = 1,
 
 	/* All non-disabled modes must come after percpu_arena_disabled. */
-	percpu_arena_disabled          = 2,
+	percpu_arena_disabled = 2,
 
-	percpu_arena_mode_names_limit  = 3, /* Used for options processing. */
+	percpu_arena_mode_names_limit = 3, /* Used for options processing. */
 	percpu_arena_mode_enabled_base = 3,
 
-	percpu_arena                   = 3,
+	percpu_arena = 3,
-	per_phycpu_arena               = 4  /* Hyper threads share arena. */
+	per_phycpu_arena = 4 /* Hyper threads share arena. */
 } percpu_arena_mode_t;
 
-#define PERCPU_ARENA_ENABLED(m)	((m) >= percpu_arena_mode_enabled_base)
+#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
-#define PERCPU_ARENA_DEFAULT	percpu_arena_disabled
+#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
 
 /*
  * When allocation_size >= oversize_threshold, use the dedicated huge arena

dep/jemalloc/include/jemalloc/internal/assert.h

@@ -1,3 +1,4 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/util.h"
 
@@ -6,51 +7,57 @@
  * assertion failure.
  */
 #ifndef assert
-#define assert(e) do {							\
+#	define assert(e)                                                            \
-	if (unlikely(config_debug && !(e))) {				\
+		do {                                                                 \
-		malloc_printf(						\
+			if (unlikely(config_debug && !(e))) {                        \
-		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
+				malloc_printf(                                       \
-		    __FILE__, __LINE__, #e);				\
+				    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \
-		abort();						\
+				    __FILE__, __LINE__, #e);                         \
-	}								\
+				abort();                                             \
-} while (0)
+			}                                                            \
+		} while (0)
 #endif
 
 #ifndef not_reached
-#define not_reached() do {						\
+#	define not_reached()                                                        \
-	if (config_debug) {						\
+		do {                                                                 \
-		malloc_printf(						\
+			if (config_debug) {                                          \
-		    "<jemalloc>: %s:%d: Unreachable code reached\n",	\
+				malloc_printf(                                       \
-		    __FILE__, __LINE__);				\
+				    "<jemalloc>: %s:%d: Unreachable code reached\n", \
-		abort();						\
+				    __FILE__, __LINE__);                             \
-	}								\
+				abort();                                             \
-	unreachable();							\
+			}                                                            \
-} while (0)
+			unreachable();                                               \
+		} while (0)
 #endif
 
 #ifndef not_implemented
-#define not_implemented() do {						\
+#	define not_implemented()                                              \
-	if (config_debug) {						\
+		do {                                                           \
-		malloc_printf("<jemalloc>: %s:%d: Not implemented\n",	\
+			if (config_debug) {                                    \
-		    __FILE__, __LINE__);				\
+				malloc_printf(                                 \
-		abort();						\
+				    "<jemalloc>: %s:%d: Not implemented\n",    \
-	}								\
+				    __FILE__, __LINE__);                       \
-} while (0)
+				abort();                                       \
+			}                                                      \
+		} while (0)
 #endif
 
 #ifndef assert_not_implemented
-#define assert_not_implemented(e) do {					\
+#	define assert_not_implemented(e)                                      \
-	if (unlikely(config_debug && !(e))) {				\
+		do {                                                           \
-		not_implemented();					\
+			if (unlikely(config_debug && !(e))) {                  \
-	}								\
+				not_implemented();                             \
-} while (0)
+			}                                                      \
+		} while (0)
 #endif
 
 /* Use to assert a particular configuration, e.g., cassert(config_debug). */
 #ifndef cassert
-#define cassert(c) do {							\
+#	define cassert(c)                                                     \
-	if (unlikely(!(c))) {						\
+		do {                                                           \
-		not_reached();						\
+			if (unlikely(!(c))) {                                  \
-	}								\
+				not_reached();                                 \
-} while (0)
+			}                                                      \
+		} while (0)
 #endif

dep/jemalloc/include/jemalloc/internal/atomic.h

@@ -1,27 +1,29 @@
 #ifndef JEMALLOC_INTERNAL_ATOMIC_H
 #define JEMALLOC_INTERNAL_ATOMIC_H
 
-#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
+#include "jemalloc/internal/jemalloc_preamble.h"
 
 #define JEMALLOC_U8_ATOMICS
 #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS)
-#  include "jemalloc/internal/atomic_gcc_atomic.h"
+#	include "jemalloc/internal/atomic_gcc_atomic.h"
-#  if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS)
+#	if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS)
-#    undef JEMALLOC_U8_ATOMICS
+#		undef JEMALLOC_U8_ATOMICS
-#  endif
+#	endif
 #elif defined(JEMALLOC_GCC_SYNC_ATOMICS)
-#  include "jemalloc/internal/atomic_gcc_sync.h"
+#	include "jemalloc/internal/atomic_gcc_sync.h"
-#  if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS)
+#	if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS)
-#    undef JEMALLOC_U8_ATOMICS
+#		undef JEMALLOC_U8_ATOMICS
-#  endif
+#	endif
 #elif defined(_MSC_VER)
-#  include "jemalloc/internal/atomic_msvc.h"
+#	include "jemalloc/internal/atomic_msvc.h"
 #elif defined(JEMALLOC_C11_ATOMICS)
-#  include "jemalloc/internal/atomic_c11.h"
+#	include "jemalloc/internal/atomic_c11.h"
 #else
-#  error "Don't have atomics implemented on this platform."
+#	error "Don't have atomics implemented on this platform."
 #endif
 
+#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
+
 /*
  * This header gives more or less a backport of C11 atomics. The user can write
  * JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_sizeof_type); to generate
@@ -54,22 +56,19 @@
 /*
  * Another convenience -- simple atomic helper functions.
  */
-#define JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(type, short_type,	\
+#define JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(type, short_type, lg_size)      \
-    lg_size)								\
+	JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size)               \
-    JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size)		\
+	ATOMIC_INLINE void atomic_load_add_store_##short_type(                 \
-    ATOMIC_INLINE void							\
+	    atomic_##short_type##_t *a, type inc) {                            \
-    atomic_load_add_store_##short_type(atomic_##short_type##_t *a,	\
+		type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED);     \
-	type inc) {							\
+		type newval = oldval + inc;                                    \
-	    type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED);	\
+		atomic_store_##short_type(a, newval, ATOMIC_RELAXED);          \
-	    type newval = oldval + inc;					\
+	}                                                                      \
-	    atomic_store_##short_type(a, newval, ATOMIC_RELAXED);	\
+	ATOMIC_INLINE void atomic_load_sub_store_##short_type(                 \
-	}								\
+	    atomic_##short_type##_t *a, type inc) {                            \
-    ATOMIC_INLINE void							\
+		type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED);     \
-    atomic_load_sub_store_##short_type(atomic_##short_type##_t *a,	\
+		type newval = oldval - inc;                                    \
-	type inc) {							\
+		atomic_store_##short_type(a, newval, ATOMIC_RELAXED);          \
-	    type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED);	\
-	    type newval = oldval - inc;					\
-	    atomic_store_##short_type(a, newval, ATOMIC_RELAXED);	\
 	}
 
 /*
@@ -77,7 +76,7 @@
  * fact.
  */
 #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
-#  define JEMALLOC_ATOMIC_U64
+#	define JEMALLOC_ATOMIC_U64
 #endif
 
 JEMALLOC_GENERATE_ATOMICS(void *, p, LG_SIZEOF_PTR)
@@ -90,6 +89,8 @@ JEMALLOC_GENERATE_ATOMICS(bool, b, 0)
 
 JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(unsigned, u, LG_SIZEOF_INT)
 
+JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(int, i, LG_SIZEOF_INT)
+
 JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR)
 
 JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR)

dep/jemalloc/include/jemalloc/internal/atomic_c11.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_ATOMIC_C11_H
 #define JEMALLOC_INTERNAL_ATOMIC_C11_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include <stdatomic.h>
 
 #define ATOMIC_INIT(...) ATOMIC_VAR_INIT(__VA_ARGS__)
@@ -14,6 +15,7 @@
 
 #define atomic_fence atomic_thread_fence
 
+/* clang-format off */
 #define JEMALLOC_GENERATE_ATOMICS(type, short_type,			\
     /* unused */ lg_size)						\
 typedef _Atomic(type) atomic_##short_type##_t;				\
@@ -58,40 +60,35 @@ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a,	\
 	return atomic_compare_exchange_strong_explicit(a, expected,	\
 	    desired, success_mo, failure_mo);				\
 }
+/* clang-format on */
 
 /*
  * Integral types have some special operations available that non-integral ones
  * lack.
  */
-#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, 		\
+#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size)  \
-    /* unused */ lg_size)						\
+	JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)      \
-JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)	\
+                                                                               \
-									\
+	ATOMIC_INLINE type atomic_fetch_add_##short_type(                      \
-ATOMIC_INLINE type							\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-atomic_fetch_add_##short_type(atomic_##short_type##_t *a,		\
+		return atomic_fetch_add_explicit(a, val, mo);                  \
-    type val, atomic_memory_order_t mo) {				\
+	}                                                                      \
-	return atomic_fetch_add_explicit(a, val, mo);			\
+                                                                               \
-}									\
+	ATOMIC_INLINE type atomic_fetch_sub_##short_type(                      \
-									\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-ATOMIC_INLINE type							\
+		return atomic_fetch_sub_explicit(a, val, mo);                  \
-atomic_fetch_sub_##short_type(atomic_##short_type##_t *a,		\
+	}                                                                      \
-    type val, atomic_memory_order_t mo) {				\
+	ATOMIC_INLINE type atomic_fetch_and_##short_type(                      \
-	return atomic_fetch_sub_explicit(a, val, mo);			\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-}									\
+		return atomic_fetch_and_explicit(a, val, mo);                  \
-ATOMIC_INLINE type							\
+	}                                                                      \
-atomic_fetch_and_##short_type(atomic_##short_type##_t *a,		\
+	ATOMIC_INLINE type atomic_fetch_or_##short_type(                       \
-    type val, atomic_memory_order_t mo) {				\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-	return atomic_fetch_and_explicit(a, val, mo);			\
+		return atomic_fetch_or_explicit(a, val, mo);                   \
-}									\
+	}                                                                      \
-ATOMIC_INLINE type							\
+	ATOMIC_INLINE type atomic_fetch_xor_##short_type(                      \
-atomic_fetch_or_##short_type(atomic_##short_type##_t *a,		\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-    type val, atomic_memory_order_t mo) {				\
+		return atomic_fetch_xor_explicit(a, val, mo);                  \
-	return atomic_fetch_or_explicit(a, val, mo);			\
+	}
-}									\
-ATOMIC_INLINE type							\
-atomic_fetch_xor_##short_type(atomic_##short_type##_t *a,		\
-    type val, atomic_memory_order_t mo) {				\
-	return atomic_fetch_xor_explicit(a, val, mo);			\
-}
 
 #endif /* JEMALLOC_INTERNAL_ATOMIC_C11_H */

dep/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h

@@ -1,9 +1,13 @@
 #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H
 #define JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/assert.h"
 
-#define ATOMIC_INIT(...) {__VA_ARGS__}
+#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
+
+#define ATOMIC_INIT(...)                                                       \
+	{ __VA_ARGS__ }
 
 typedef enum {
 	atomic_memory_order_relaxed,
@@ -36,94 +40,82 @@ atomic_fence(atomic_memory_order_t mo) {
 	__atomic_thread_fence(atomic_enum_to_builtin(mo));
 }
 
-#define JEMALLOC_GENERATE_ATOMICS(type, short_type,			\
+#define JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)      \
-    /* unused */ lg_size)						\
+	typedef struct {                                                       \
-typedef struct {							\
+		type repr;                                                     \
-	type repr;							\
+	} atomic_##short_type##_t;                                             \
-} atomic_##short_type##_t;						\
+                                                                               \
-									\
+	ATOMIC_INLINE type atomic_load_##short_type(                           \
-ATOMIC_INLINE type							\
+	    const atomic_##short_type##_t *a, atomic_memory_order_t mo) {      \
-atomic_load_##short_type(const atomic_##short_type##_t *a,		\
+		type result;                                                   \
-    atomic_memory_order_t mo) {						\
+		__atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo));  \
-	type result;							\
+		return result;                                                 \
-	__atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo));	\
+	}                                                                      \
-	return result;							\
+                                                                               \
-}									\
+	ATOMIC_INLINE void atomic_store_##short_type(                          \
-									\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-ATOMIC_INLINE void							\
+		__atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo));    \
-atomic_store_##short_type(atomic_##short_type##_t *a, type val,		\
+	}                                                                      \
-    atomic_memory_order_t mo) {						\
+                                                                               \
-	__atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo));	\
+	ATOMIC_INLINE type atomic_exchange_##short_type(                       \
-}									\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-									\
+		type result;                                                   \
-ATOMIC_INLINE type							\
+		__atomic_exchange(                                             \
-atomic_exchange_##short_type(atomic_##short_type##_t *a, type val,	\
+		    &a->repr, &val, &result, atomic_enum_to_builtin(mo));      \
-    atomic_memory_order_t mo) {						\
+		return result;                                                 \
-	type result;							\
+	}                                                                      \
-	__atomic_exchange(&a->repr, &val, &result,			\
+                                                                               \
-	    atomic_enum_to_builtin(mo));				\
+	ATOMIC_INLINE bool atomic_compare_exchange_weak_##short_type(          \
-	return result;							\
+	    atomic_##short_type##_t *a, UNUSED type *expected, type desired,   \
-}									\
+	    atomic_memory_order_t success_mo,                                  \
-									\
+	    atomic_memory_order_t failure_mo) {                                \
-ATOMIC_INLINE bool							\
+		return __atomic_compare_exchange(&a->repr, expected, &desired, \
-atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a,	\
+		    true, atomic_enum_to_builtin(success_mo),                  \
-    UNUSED type *expected, type desired,				\
+		    atomic_enum_to_builtin(failure_mo));                       \
-    atomic_memory_order_t success_mo,					\
+	}                                                                      \
-    atomic_memory_order_t failure_mo) {					\
+                                                                               \
-	return __atomic_compare_exchange(&a->repr, expected, &desired,	\
+	ATOMIC_INLINE bool atomic_compare_exchange_strong_##short_type(        \
-	    true, atomic_enum_to_builtin(success_mo),			\
+	    atomic_##short_type##_t *a, UNUSED type *expected, type desired,   \
-	    atomic_enum_to_builtin(failure_mo));			\
+	    atomic_memory_order_t success_mo,                                  \
-}									\
+	    atomic_memory_order_t failure_mo) {                                \
-									\
+		return __atomic_compare_exchange(&a->repr, expected, &desired, \
-ATOMIC_INLINE bool							\
+		    false, atomic_enum_to_builtin(success_mo),                 \
-atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a,	\
+		    atomic_enum_to_builtin(failure_mo));                       \
-    UNUSED type *expected, type desired,				\
+	}
-    atomic_memory_order_t success_mo,					\
-    atomic_memory_order_t failure_mo) {					\
-	return __atomic_compare_exchange(&a->repr, expected, &desired,	\
-	    false,							\
-	    atomic_enum_to_builtin(success_mo),				\
-	    atomic_enum_to_builtin(failure_mo));			\
-}
 
+#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size)  \
+	JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)      \
+                                                                               \
+	ATOMIC_INLINE type atomic_fetch_add_##short_type(                      \
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
+		return __atomic_fetch_add(                                     \
+		    &a->repr, val, atomic_enum_to_builtin(mo));                \
+	}                                                                      \
+                                                                               \
+	ATOMIC_INLINE type atomic_fetch_sub_##short_type(                      \
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
+		return __atomic_fetch_sub(                                     \
+		    &a->repr, val, atomic_enum_to_builtin(mo));                \
+	}                                                                      \
+                                                                               \
+	ATOMIC_INLINE type atomic_fetch_and_##short_type(                      \
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
+		return __atomic_fetch_and(                                     \
+		    &a->repr, val, atomic_enum_to_builtin(mo));                \
+	}                                                                      \
+                                                                               \
+	ATOMIC_INLINE type atomic_fetch_or_##short_type(                       \
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
+		return __atomic_fetch_or(                                      \
+		    &a->repr, val, atomic_enum_to_builtin(mo));                \
+	}                                                                      \
+                                                                               \
+	ATOMIC_INLINE type atomic_fetch_xor_##short_type(                      \
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
+		return __atomic_fetch_xor(                                     \
+		    &a->repr, val, atomic_enum_to_builtin(mo));                \
+	}
 
-#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type,			\
+#undef ATOMIC_INLINE
-    /* unused */ lg_size)						\
-JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)	\
-									\
-ATOMIC_INLINE type							\
-atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val,	\
-    atomic_memory_order_t mo) {						\
-	return __atomic_fetch_add(&a->repr, val,			\
-	    atomic_enum_to_builtin(mo));				\
-}									\
-									\
-ATOMIC_INLINE type							\
-atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val,	\
-    atomic_memory_order_t mo) {						\
-	return __atomic_fetch_sub(&a->repr, val,			\
-	    atomic_enum_to_builtin(mo));				\
-}									\
-									\
-ATOMIC_INLINE type							\
-atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val,	\
-    atomic_memory_order_t mo) {						\
-	return __atomic_fetch_and(&a->repr, val,			\
-	    atomic_enum_to_builtin(mo));				\
-}									\
-									\
-ATOMIC_INLINE type							\
-atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val,	\
-    atomic_memory_order_t mo) {						\
-	return __atomic_fetch_or(&a->repr, val,				\
-	    atomic_enum_to_builtin(mo));				\
-}									\
-									\
-ATOMIC_INLINE type							\
-atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val,	\
-    atomic_memory_order_t mo) {						\
-	return __atomic_fetch_xor(&a->repr, val,			\
-	    atomic_enum_to_builtin(mo));				\
-}
 
 #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H */

dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h

@@ -1,7 +1,12 @@
 #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H
 #define JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H
 
-#define ATOMIC_INIT(...) {__VA_ARGS__}
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
+
+#define ATOMIC_INIT(...)                                                       \
+	{ __VA_ARGS__ }
 
 typedef enum {
 	atomic_memory_order_relaxed,
@@ -25,13 +30,13 @@ atomic_fence(atomic_memory_order_t mo) {
 		return;
 	}
 	asm volatile("" ::: "memory");
-#  if defined(__i386__) || defined(__x86_64__)
+#if defined(__i386__) || defined(__x86_64__)
 	/* This is implicit on x86. */
-#  elif defined(__ppc64__)
+#elif defined(__ppc64__)
 	asm volatile("lwsync");
-#  elif defined(__ppc__)
+#elif defined(__ppc__)
 	asm volatile("sync");
-#  elif defined(__sparc__) && defined(__arch64__)
+#elif defined(__sparc__) && defined(__arch64__)
 	if (mo == atomic_memory_order_acquire) {
 		asm volatile("membar #LoadLoad | #LoadStore");
 	} else if (mo == atomic_memory_order_release) {
@@ -39,9 +44,9 @@ atomic_fence(atomic_memory_order_t mo) {
 	} else {
 		asm volatile("membar #LoadLoad | #LoadStore | #StoreStore");
 	}
-#  else
+#else
 	__sync_synchronize();
-#  endif
+#endif
 	asm volatile("" ::: "memory");
 }
 
@@ -64,25 +69,25 @@ atomic_fence(atomic_memory_order_t mo) {
 
 ATOMIC_INLINE void
 atomic_pre_sc_load_fence() {
-#  if defined(__i386__) || defined(__x86_64__) ||			\
+#if defined(__i386__) || defined(__x86_64__)                                   \
-    (defined(__sparc__) && defined(__arch64__))
+    || (defined(__sparc__) && defined(__arch64__))
 	atomic_fence(atomic_memory_order_relaxed);
-#  else
+#else
 	atomic_fence(atomic_memory_order_seq_cst);
-#  endif
+#endif
 }
 
 ATOMIC_INLINE void
 atomic_post_sc_store_fence() {
-#  if defined(__i386__) || defined(__x86_64__) ||			\
+#if defined(__i386__) || defined(__x86_64__)                                   \
-    (defined(__sparc__) && defined(__arch64__))
+    || (defined(__sparc__) && defined(__arch64__))
 	atomic_fence(atomic_memory_order_seq_cst);
-#  else
+#else
 	atomic_fence(atomic_memory_order_relaxed);
-#  endif
+#endif
-
 }
 
+/* clang-format off */
 #define JEMALLOC_GENERATE_ATOMICS(type, short_type,			\
     /* unused */ lg_size)						\
 typedef struct {							\
@@ -157,39 +162,36 @@ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a,	\
 		return false;						\
 	}								\
 }
+/* clang-format on */
 
-#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type,			\
+#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size)  \
-    /* unused */ lg_size)						\
+	JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)      \
-JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)	\
+                                                                               \
-									\
+	ATOMIC_INLINE type atomic_fetch_add_##short_type(                      \
-ATOMIC_INLINE type							\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val,	\
+		return __sync_fetch_and_add(&a->repr, val);                    \
-    atomic_memory_order_t mo) {						\
+	}                                                                      \
-	return __sync_fetch_and_add(&a->repr, val);			\
+                                                                               \
-}									\
+	ATOMIC_INLINE type atomic_fetch_sub_##short_type(                      \
-									\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-ATOMIC_INLINE type							\
+		return __sync_fetch_and_sub(&a->repr, val);                    \
-atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val,	\
+	}                                                                      \
-    atomic_memory_order_t mo) {						\
+                                                                               \
-	return __sync_fetch_and_sub(&a->repr, val);			\
+	ATOMIC_INLINE type atomic_fetch_and_##short_type(                      \
-}									\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-									\
+		return __sync_fetch_and_and(&a->repr, val);                    \
-ATOMIC_INLINE type							\
+	}                                                                      \
-atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val,	\
+                                                                               \
-    atomic_memory_order_t mo) {						\
+	ATOMIC_INLINE type atomic_fetch_or_##short_type(                       \
-	return __sync_fetch_and_and(&a->repr, val);			\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-}									\
+		return __sync_fetch_and_or(&a->repr, val);                     \
-									\
+	}                                                                      \
-ATOMIC_INLINE type							\
+                                                                               \
-atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val,	\
+	ATOMIC_INLINE type atomic_fetch_xor_##short_type(                      \
-    atomic_memory_order_t mo) {						\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-	return __sync_fetch_and_or(&a->repr, val);			\
+		return __sync_fetch_and_xor(&a->repr, val);                    \
-}									\
+	}
-									\
+
-ATOMIC_INLINE type							\
+#undef ATOMIC_INLINE
-atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val,	\
-    atomic_memory_order_t mo) {						\
-	return __sync_fetch_and_xor(&a->repr, val);			\
-}
 
 #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H */

dep/jemalloc/include/jemalloc/internal/atomic_msvc.h

@@ -1,7 +1,12 @@
 #ifndef JEMALLOC_INTERNAL_ATOMIC_MSVC_H
 #define JEMALLOC_INTERNAL_ATOMIC_MSVC_H
 
-#define ATOMIC_INIT(...) {__VA_ARGS__}
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
+
+#define ATOMIC_INIT(...)                                                       \
+	{ __VA_ARGS__ }
 
 typedef enum {
 	atomic_memory_order_relaxed,
@@ -11,109 +16,106 @@ typedef enum {
 	atomic_memory_order_seq_cst
 } atomic_memory_order_t;
 
-typedef char atomic_repr_0_t;
+typedef char    atomic_repr_0_t;
-typedef short atomic_repr_1_t;
+typedef short   atomic_repr_1_t;
-typedef long atomic_repr_2_t;
+typedef long    atomic_repr_2_t;
 typedef __int64 atomic_repr_3_t;
 
 ATOMIC_INLINE void
 atomic_fence(atomic_memory_order_t mo) {
 	_ReadWriteBarrier();
-#  if defined(_M_ARM) || defined(_M_ARM64)
+#if defined(_M_ARM) || defined(_M_ARM64)
 	/* ARM needs a barrier for everything but relaxed. */
 	if (mo != atomic_memory_order_relaxed) {
 		MemoryBarrier();
 	}
-#  elif defined(_M_IX86) || defined (_M_X64)
+#elif defined(_M_IX86) || defined(_M_X64)
 	/* x86 needs a barrier only for seq_cst. */
 	if (mo == atomic_memory_order_seq_cst) {
 		MemoryBarrier();
 	}
-#  else
+#else
-#  error "Don't know how to create atomics for this platform for MSVC."
+#	error "Don't know how to create atomics for this platform for MSVC."
-#  endif
+#endif
 	_ReadWriteBarrier();
 }
 
-#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_ ## lg_size ## _t
+#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_##lg_size##_t
 
 #define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b)
-#define ATOMIC_RAW_CONCAT(a, b) a ## b
+#define ATOMIC_RAW_CONCAT(a, b) a##b
 
-#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) ATOMIC_CONCAT(	\
+#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size)                            \
-    base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size))
+	ATOMIC_CONCAT(base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size))
 
-#define ATOMIC_INTERLOCKED_SUFFIX(lg_size)				\
+#define ATOMIC_INTERLOCKED_SUFFIX(lg_size)                                     \
-    ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size)
+	ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size)
 
 #define ATOMIC_INTERLOCKED_SUFFIX_0 8
 #define ATOMIC_INTERLOCKED_SUFFIX_1 16
 #define ATOMIC_INTERLOCKED_SUFFIX_2
 #define ATOMIC_INTERLOCKED_SUFFIX_3 64
 
-#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size)		\
+#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size)                   \
-typedef struct {							\
+	typedef struct {                                                       \
-	ATOMIC_INTERLOCKED_REPR(lg_size) repr;				\
+		ATOMIC_INTERLOCKED_REPR(lg_size) repr;                         \
-} atomic_##short_type##_t;						\
+	} atomic_##short_type##_t;                                             \
-									\
+                                                                               \
-ATOMIC_INLINE type							\
+	ATOMIC_INLINE type atomic_load_##short_type(                           \
-atomic_load_##short_type(const atomic_##short_type##_t *a,		\
+	    const atomic_##short_type##_t *a, atomic_memory_order_t mo) {      \
-    atomic_memory_order_t mo) {						\
+		ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr;                \
-	ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr;			\
+		if (mo != atomic_memory_order_relaxed) {                       \
-	if (mo != atomic_memory_order_relaxed) {			\
+			atomic_fence(atomic_memory_order_acquire);             \
-		atomic_fence(atomic_memory_order_acquire);		\
+		}                                                              \
-	}								\
+		return (type)ret;                                              \
-	return (type) ret;						\
+	}                                                                      \
-}									\
+                                                                               \
-									\
+	ATOMIC_INLINE void atomic_store_##short_type(                          \
-ATOMIC_INLINE void							\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-atomic_store_##short_type(atomic_##short_type##_t *a,			\
+		if (mo != atomic_memory_order_relaxed) {                       \
-    type val, atomic_memory_order_t mo) {				\
+			atomic_fence(atomic_memory_order_release);             \
-	if (mo != atomic_memory_order_relaxed) {			\
+		}                                                              \
-		atomic_fence(atomic_memory_order_release);		\
+		a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size))val;               \
-	}								\
+		if (mo == atomic_memory_order_seq_cst) {                       \
-	a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size)) val;		\
+			atomic_fence(atomic_memory_order_seq_cst);             \
-	if (mo == atomic_memory_order_seq_cst) {			\
+		}                                                              \
-		atomic_fence(atomic_memory_order_seq_cst);		\
+	}                                                                      \
-	}								\
+                                                                               \
-}									\
+	ATOMIC_INLINE type atomic_exchange_##short_type(                       \
-									\
+	    atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) {  \
-ATOMIC_INLINE type							\
+		return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange,     \
-atomic_exchange_##short_type(atomic_##short_type##_t *a, type val,	\
+		    lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
-    atomic_memory_order_t mo) {						\
+	}                                                                      \
-	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange,	\
+                                                                               \
-	    lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);	\
+	ATOMIC_INLINE bool atomic_compare_exchange_weak_##short_type(          \
-}									\
+	    atomic_##short_type##_t *a, type *expected, type desired,          \
-									\
+	    atomic_memory_order_t success_mo,                                  \
-ATOMIC_INLINE bool							\
+	    atomic_memory_order_t failure_mo) {                                \
-atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a,	\
+		ATOMIC_INTERLOCKED_REPR(lg_size)                               \
-    type *expected, type desired, atomic_memory_order_t success_mo,	\
+		e = (ATOMIC_INTERLOCKED_REPR(lg_size)) * expected;             \
-    atomic_memory_order_t failure_mo) {					\
+		ATOMIC_INTERLOCKED_REPR(lg_size)                               \
-	ATOMIC_INTERLOCKED_REPR(lg_size) e =				\
+		d = (ATOMIC_INTERLOCKED_REPR(lg_size))desired;                 \
-	    (ATOMIC_INTERLOCKED_REPR(lg_size))*expected;		\
+		ATOMIC_INTERLOCKED_REPR(lg_size)                               \
-	ATOMIC_INTERLOCKED_REPR(lg_size) d =				\
+		old = ATOMIC_INTERLOCKED_NAME(                                 \
-	    (ATOMIC_INTERLOCKED_REPR(lg_size))desired;			\
+		    _InterlockedCompareExchange, lg_size)(&a->repr, d, e);     \
-	ATOMIC_INTERLOCKED_REPR(lg_size) old =				\
+		if (old == e) {                                                \
-	    ATOMIC_INTERLOCKED_NAME(_InterlockedCompareExchange, 	\
+			return true;                                           \
-		lg_size)(&a->repr, d, e);				\
+		} else {                                                       \
-	if (old == e) {							\
+			*expected = (type)old;                                 \
-		return true;						\
+			return false;                                          \
-	} else {							\
+		}                                                              \
-		*expected = (type)old;					\
+	}                                                                      \
-		return false;						\
+                                                                               \
-	}								\
+	ATOMIC_INLINE bool atomic_compare_exchange_strong_##short_type(        \
-}									\
+	    atomic_##short_type##_t *a, type *expected, type desired,          \
-									\
+	    atomic_memory_order_t success_mo,                                  \
-ATOMIC_INLINE bool							\
+	    atomic_memory_order_t failure_mo) {                                \
-atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a,	\
+		/* We implement the weak version with strong semantics. */     \
-    type *expected, type desired, atomic_memory_order_t success_mo,	\
+		return atomic_compare_exchange_weak_##short_type(              \
-    atomic_memory_order_t failure_mo) {					\
+		    a, expected, desired, success_mo, failure_mo);             \
-	/* We implement the weak version with strong semantics. */	\
+	}
-	return atomic_compare_exchange_weak_##short_type(a, expected,	\
-	    desired, success_mo, failure_mo);				\
-}
-
 
+/* clang-format off */
 #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size)	\
 JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size)			\
 									\
@@ -154,5 +156,8 @@ atomic_fetch_xor_##short_type(atomic_##short_type##_t *a,		\
 	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedXor, lg_size)(	\
 	    &a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);		\
 }
+/* clang-format on */
+
+#undef ATOMIC_INLINE
 
 #endif /* JEMALLOC_INTERNAL_ATOMIC_MSVC_H */

dep/jemalloc/include/jemalloc/internal/background_thread_externs.h

@@ -1,26 +1,31 @@
 #ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
 #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
 
-extern bool opt_background_thread;
+#include "jemalloc/internal/jemalloc_preamble.h"
-extern size_t opt_max_background_threads;
+#include "jemalloc/internal/background_thread_structs.h"
-extern malloc_mutex_t background_thread_lock;
+#include "jemalloc/internal/base.h"
-extern atomic_b_t background_thread_enabled_state;
+#include "jemalloc/internal/mutex.h"
-extern size_t n_background_threads;
+
-extern size_t max_background_threads;
+extern bool                      opt_background_thread;
+extern size_t                    opt_max_background_threads;
+extern malloc_mutex_t            background_thread_lock;
+extern atomic_b_t                background_thread_enabled_state;
+extern size_t                    n_background_threads;
+extern size_t                    max_background_threads;
 extern background_thread_info_t *background_thread_info;
 
 bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
 bool background_threads_enable(tsd_t *tsd);
 bool background_threads_disable(tsd_t *tsd);
-bool background_thread_is_started(background_thread_info_t* info);
+bool background_thread_is_started(background_thread_info_t *info);
-void background_thread_wakeup_early(background_thread_info_t *info,
+void background_thread_wakeup_early(
-    nstime_t *remaining_sleep);
+    background_thread_info_t *info, nstime_t *remaining_sleep);
 void background_thread_prefork0(tsdn_t *tsdn);
 void background_thread_prefork1(tsdn_t *tsdn);
 void background_thread_postfork_parent(tsdn_t *tsdn);
 void background_thread_postfork_child(tsdn_t *tsdn);
-bool background_thread_stats_read(tsdn_t *tsdn,
+bool background_thread_stats_read(
-    background_thread_stats_t *stats);
+    tsdn_t *tsdn, background_thread_stats_t *stats);
 void background_thread_ctl_init(tsdn_t *tsdn);
 
 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER

dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h

@@ -1,15 +1,25 @@
 #ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
 #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_inlines_a.h"
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/background_thread_externs.h"
+
 JEMALLOC_ALWAYS_INLINE bool
 background_thread_enabled(void) {
 	return atomic_load_b(&background_thread_enabled_state, ATOMIC_RELAXED);
 }
 
+JEMALLOC_ALWAYS_INLINE void
+background_thread_enabled_set_impl(bool state) {
+	atomic_store_b(&background_thread_enabled_state, state, ATOMIC_RELAXED);
+}
+
 JEMALLOC_ALWAYS_INLINE void
 background_thread_enabled_set(tsdn_t *tsdn, bool state) {
 	malloc_mutex_assert_owner(tsdn, &background_thread_lock);
-	atomic_store_b(&background_thread_enabled_state, state, ATOMIC_RELAXED);
+	background_thread_enabled_set_impl(state);
 }
 
 JEMALLOC_ALWAYS_INLINE background_thread_info_t *
@@ -26,14 +36,14 @@ background_thread_info_get(size_t ind) {
 JEMALLOC_ALWAYS_INLINE uint64_t
 background_thread_wakeup_time_get(background_thread_info_t *info) {
 	uint64_t next_wakeup = nstime_ns(&info->next_wakeup);
-	assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE) ==
+	assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE)
-	    (next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP));
+	    == (next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP));
 	return next_wakeup;
 }
 
 JEMALLOC_ALWAYS_INLINE void
-background_thread_wakeup_time_set(tsdn_t *tsdn, background_thread_info_t *info,
+background_thread_wakeup_time_set(
-    uint64_t wakeup_time) {
+    tsdn_t *tsdn, background_thread_info_t *info, uint64_t wakeup_time) {
 	malloc_mutex_assert_owner(tsdn, &info->mtx);
 	atomic_store_b(&info->indefinite_sleep,
 	    wakeup_time == BACKGROUND_THREAD_INDEFINITE_SLEEP, ATOMIC_RELEASE);

dep/jemalloc/include/jemalloc/internal/background_thread_structs.h

@@ -1,10 +1,13 @@
 #ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
 #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/mutex.h"
+
 /* This file really combines "structs" and "types", but only transitionally. */
 
 #if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
-#  define JEMALLOC_PTHREAD_CREATE_WRAPPER
+#	define JEMALLOC_PTHREAD_CREATE_WRAPPER
 #endif
 
 #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX
@@ -32,33 +35,33 @@ typedef enum {
 struct background_thread_info_s {
 #ifdef JEMALLOC_BACKGROUND_THREAD
 	/* Background thread is pthread specific. */
-	pthread_t		thread;
+	pthread_t      thread;
-	pthread_cond_t		cond;
+	pthread_cond_t cond;
 #endif
-	malloc_mutex_t		mtx;
+	malloc_mutex_t            mtx;
-	background_thread_state_t	state;
+	background_thread_state_t state;
 	/* When true, it means no wakeup scheduled. */
-	atomic_b_t		indefinite_sleep;
+	atomic_b_t indefinite_sleep;
 	/* Next scheduled wakeup time (absolute time in ns). */
-	nstime_t		next_wakeup;
+	nstime_t next_wakeup;
 	/*
 	 *  Since the last background thread run, newly added number of pages
 	 *  that need to be purged by the next wakeup.  This is adjusted on
 	 *  epoch advance, and is used to determine whether we should signal the
 	 *  background thread to wake up earlier.
 	 */
-	size_t			npages_to_purge_new;
+	size_t npages_to_purge_new;
 	/* Stats: total number of runs since started. */
-	uint64_t		tot_n_runs;
+	uint64_t tot_n_runs;
 	/* Stats: total sleep time since started. */
-	nstime_t		tot_sleep_time;
+	nstime_t tot_sleep_time;
 };
 typedef struct background_thread_info_s background_thread_info_t;
 
 struct background_thread_stats_s {
-	size_t num_threads;
+	size_t            num_threads;
-	uint64_t num_runs;
+	uint64_t          num_runs;
-	nstime_t run_interval;
+	nstime_t          run_interval;
 	mutex_prof_data_t max_counter_per_bg_thd;
 };
 typedef struct background_thread_stats_s background_thread_stats_t;

dep/jemalloc/include/jemalloc/internal/base.h

@@ -1,12 +1,19 @@
 #ifndef JEMALLOC_INTERNAL_BASE_H
 #define JEMALLOC_INTERNAL_BASE_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/ehooks.h"
 #include "jemalloc/internal/mutex.h"
 
+/*
+ * Alignment when THP is not enabled.  Set to constant 2M in case the HUGEPAGE
+ * value is unexpected high (which would cause VM over-reservation).
+ */
+#define BASE_BLOCK_MIN_ALIGN ((size_t)2 << 20)
+
 enum metadata_thp_mode_e {
-	metadata_thp_disabled   = 0,
+	metadata_thp_disabled = 0,
 	/*
 	 * Lazily enable hugepage for metadata. To avoid high RSS caused by THP
 	 * + low usage arena (i.e. THP becomes a significant percentage), the
@@ -15,16 +22,15 @@ enum metadata_thp_mode_e {
 	 * arena), "auto" behaves the same as "always", i.e. madvise hugepage
 	 * right away.
 	 */
-	metadata_thp_auto       = 1,
+	metadata_thp_auto = 1,
-	metadata_thp_always     = 2,
+	metadata_thp_always = 2,
 	metadata_thp_mode_limit = 3
 };
 typedef enum metadata_thp_mode_e metadata_thp_mode_t;
 
 #define METADATA_THP_DEFAULT metadata_thp_disabled
 extern metadata_thp_mode_t opt_metadata_thp;
-extern const char *metadata_thp_mode_names[];
+extern const char *const   metadata_thp_mode_names[];
-
 
 /* Embedded at the beginning of every block of base-managed virtual memory. */
 typedef struct base_block_s base_block_t;
@@ -72,8 +78,13 @@ struct base_s {
 	/* Heap of extents that track unused trailing space within blocks. */
 	edata_heap_t avail[SC_NSIZES];
 
+	/* Contains reusable base edata (used by tcache_stacks currently). */
+	edata_avail_t edata_avail;
+
 	/* Stats, only maintained if config_stats. */
 	size_t allocated;
+	size_t edata_allocated;
+	size_t rtree_allocated;
 	size_t resident;
 	size_t mapped;
 	/* Number of THP regions touched. */
@@ -91,20 +102,24 @@ metadata_thp_enabled(void) {
 }
 
 base_t *b0get(void);
-base_t *base_new(tsdn_t *tsdn, unsigned ind,
+base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
-    const extent_hooks_t *extent_hooks, bool metadata_use_hooks);
+    bool metadata_use_hooks);
-void base_delete(tsdn_t *tsdn, base_t *base);
+void    base_delete(tsdn_t *tsdn, base_t *base);
-ehooks_t *base_ehooks_get(base_t *base);
+ehooks_t       *base_ehooks_get(base_t *base);
-ehooks_t *base_ehooks_get_for_metadata(base_t *base);
+ehooks_t       *base_ehooks_get_for_metadata(base_t *base);
-extent_hooks_t *base_extent_hooks_set(base_t *base,
+extent_hooks_t *base_extent_hooks_set(
-    extent_hooks_t *extent_hooks);
+    base_t *base, extent_hooks_t *extent_hooks);
-void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);
+void    *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);
 edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base);
-void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated,
+void    *base_alloc_rtree(tsdn_t *tsdn, base_t *base, size_t size);
-    size_t *resident, size_t *mapped, size_t *n_thp);
+void    *b0_alloc_tcache_stack(tsdn_t *tsdn, size_t size);
-void base_prefork(tsdn_t *tsdn, base_t *base);
+void     b0_dalloc_tcache_stack(tsdn_t *tsdn, void *tcache_stack);
-void base_postfork_parent(tsdn_t *tsdn, base_t *base);
+void     base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated,
-void base_postfork_child(tsdn_t *tsdn, base_t *base);
+        size_t *edata_allocated, size_t *rtree_allocated, size_t *resident,
-bool base_boot(tsdn_t *tsdn);
+        size_t *mapped, size_t *n_thp);
+void     base_prefork(tsdn_t *tsdn, base_t *base);
+void     base_postfork_parent(tsdn_t *tsdn, base_t *base);
+void     base_postfork_child(tsdn_t *tsdn, base_t *base);
+bool     base_boot(tsdn_t *tsdn);
 
 #endif /* JEMALLOC_INTERNAL_BASE_H */

dep/jemalloc/include/jemalloc/internal/bin.h

@@ -1,6 +1,8 @@
 #ifndef JEMALLOC_INTERNAL_BIN_H
 #define JEMALLOC_INTERNAL_BIN_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/bin_info.h"
 #include "jemalloc/internal/bin_stats.h"
 #include "jemalloc/internal/bin_types.h"
 #include "jemalloc/internal/edata.h"
@@ -14,13 +16,13 @@
 typedef struct bin_s bin_t;
 struct bin_s {
 	/* All operations on bin_t fields require lock ownership. */
-	malloc_mutex_t		lock;
+	malloc_mutex_t lock;
 
 	/*
 	 * Bin statistics.  These get touched every time the lock is acquired,
 	 * so put them close by in the hopes of getting some cache locality.
 	 */
-	bin_stats_t	stats;
+	bin_stats_t stats;
 
 	/*
 	 * Current slab being used to service allocations of this bin's size
@@ -28,17 +30,17 @@ struct bin_s {
 	 * slabcur is reassigned, the previous slab must be deallocated or
 	 * inserted into slabs_{nonfull,full}.
 	 */
-	edata_t			*slabcur;
+	edata_t *slabcur;
 
 	/*
 	 * Heap of non-full slabs.  This heap is used to assure that new
 	 * allocations come from the non-full slab that is oldest/lowest in
 	 * memory.
 	 */
-	edata_heap_t		slabs_nonfull;
+	edata_heap_t slabs_nonfull;
 
 	/* List used to track full slabs. */
-	edata_list_active_t	slabs_full;
+	edata_list_active_t slabs_full;
 };
 
 /* A set of sharded bins of the same size class. */
@@ -48,7 +50,7 @@ struct bins_s {
 	bin_t *bin_shards;
 };
 
-void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]);
+void bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]);
 bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size,
     size_t end_size, size_t nshards);
 
@@ -60,6 +62,43 @@ void bin_prefork(tsdn_t *tsdn, bin_t *bin);
 void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin);
 void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
 
+/* Slab region allocation. */
+void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info);
+void  bin_slab_reg_alloc_batch(
+     edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs);
+
+/* Slab list management. */
+void     bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab);
+void     bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab);
+edata_t *bin_slabs_nonfull_tryget(bin_t *bin);
+void     bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab);
+void     bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab);
+
+/* Slab association / demotion. */
+void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin);
+void bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
+
+/* Deallocation helpers (called under bin lock). */
+void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin);
+void bin_dalloc_locked_handle_newly_empty(
+    tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
+void bin_dalloc_locked_handle_newly_nonempty(
+    tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
+
+/* Slabcur refill and allocation. */
+void  bin_refill_slabcur_with_fresh_slab(tsdn_t *tsdn, bin_t *bin,
+    szind_t binind, edata_t *fresh_slab);
+void *bin_malloc_with_fresh_slab(tsdn_t *tsdn, bin_t *bin,
+    szind_t binind, edata_t *fresh_slab);
+bool  bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto,
+    bin_t *bin);
+void *bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin,
+    szind_t binind);
+
+/* Bin selection. */
+bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
+    unsigned *binshard_p);
+
 /* Stats. */
 static inline void
 bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {

dep/jemalloc/include/jemalloc/internal/bin_info.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_BIN_INFO_H
 #define JEMALLOC_INTERNAL_BIN_INFO_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bitmap.h"
 
 /*
@@ -25,22 +26,22 @@
 typedef struct bin_info_s bin_info_t;
 struct bin_info_s {
 	/* Size of regions in a slab for this bin's size class. */
-	size_t			reg_size;
+	size_t reg_size;
 
 	/* Total size of a slab for this bin's size class. */
-	size_t			slab_size;
+	size_t slab_size;
 
 	/* Total number of regions in a slab for this bin's size class. */
-	uint32_t		nregs;
+	uint32_t nregs;
 
 	/* Number of sharded bins in each arena for this size class. */
-	uint32_t		n_shards;
+	uint32_t n_shards;
 
 	/*
 	 * Metadata used to manipulate bitmaps for slabs associated with this
 	 * bin.
 	 */
-	bitmap_info_t		bitmap_info;
+	bitmap_info_t bitmap_info;
 };
 
 extern bin_info_t bin_infos[SC_NBINS];

dep/jemalloc/include/jemalloc/internal/bin_inlines.h

@@ -0,0 +1,112 @@
+#ifndef JEMALLOC_INTERNAL_BIN_INLINES_H
+#define JEMALLOC_INTERNAL_BIN_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/bin.h"
+#include "jemalloc/internal/bin_info.h"
+#include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/edata.h"
+#include "jemalloc/internal/sc.h"
+
+/*
+ * The dalloc bin info contains just the information that the common paths need
+ * during tcache flushes.  By force-inlining these paths, and using local copies
+ * of data (so that the compiler knows it's constant), we avoid a whole bunch of
+ * redundant loads and stores by leaving this information in registers.
+ */
+typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
+struct bin_dalloc_locked_info_s {
+	div_info_t div_info;
+	uint32_t   nregs;
+	uint64_t   ndalloc;
+};
+
+/* Find the region index of a pointer within a slab. */
+JEMALLOC_ALWAYS_INLINE size_t
+bin_slab_regind_impl(
+    div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
+	size_t diff, regind;
+
+	/* Freeing a pointer outside the slab can cause assertion failure. */
+	assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab));
+	assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab));
+	/* Freeing an interior pointer can cause assertion failure. */
+	assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab))
+	        % (uintptr_t)bin_infos[binind].reg_size
+	    == 0);
+
+	diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab));
+
+	/* Avoid doing division with a variable divisor. */
+	regind = div_compute(div_info, diff);
+	assert(regind < bin_infos[binind].nregs);
+	return regind;
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind,
+    edata_t *slab, const void *ptr) {
+	size_t regind = bin_slab_regind_impl(
+	    &info->div_info, binind, slab, ptr);
+	return regind;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+bin_dalloc_locked_begin(
+    bin_dalloc_locked_info_t *info, szind_t binind) {
+	info->div_info = arena_binind_div_info[binind];
+	info->nregs = bin_infos[binind].nregs;
+	info->ndalloc = 0;
+}
+
+/*
+ * Does the deallocation work associated with freeing a single pointer (a
+ * "step") in between a bin_dalloc_locked begin and end call.
+ *
+ * Returns true if arena_slab_dalloc must be called on slab.  Doesn't do
+ * stats updates, which happen during finish (this lets running counts get left
+ * in a register).
+ */
+JEMALLOC_ALWAYS_INLINE bool
+bin_dalloc_locked_step(tsdn_t *tsdn, bool is_auto, bin_t *bin,
+    bin_dalloc_locked_info_t *info, szind_t binind, edata_t *slab,
+    void *ptr) {
+	const bin_info_t *bin_info = &bin_infos[binind];
+	size_t            regind = bin_slab_regind(info, binind, slab, ptr);
+	slab_data_t      *slab_data = edata_slab_data_get(slab);
+
+	assert(edata_nfree_get(slab) < bin_info->nregs);
+	/* Freeing an unallocated pointer can cause assertion failure. */
+	assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));
+
+	bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);
+	edata_nfree_inc(slab);
+
+	if (config_stats) {
+		info->ndalloc++;
+	}
+
+	unsigned nfree = edata_nfree_get(slab);
+	if (nfree == bin_info->nregs) {
+		bin_dalloc_locked_handle_newly_empty(
+		    tsdn, is_auto, slab, bin);
+		return true;
+	} else if (nfree == 1 && slab != bin->slabcur) {
+		bin_dalloc_locked_handle_newly_nonempty(
+		    tsdn, is_auto, slab, bin);
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+bin_dalloc_locked_finish(tsdn_t *tsdn, bin_t *bin,
+    bin_dalloc_locked_info_t *info) {
+	if (config_stats) {
+		bin->stats.ndalloc += info->ndalloc;
+		assert(bin->stats.curregs >= (size_t)info->ndalloc);
+		bin->stats.curregs -= (size_t)info->ndalloc;
+	}
+}
+
+#endif /* JEMALLOC_INTERNAL_BIN_INLINES_H */

dep/jemalloc/include/jemalloc/internal/bin_stats.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_BIN_STATS_H
 #define JEMALLOC_INTERNAL_BIN_STATS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/mutex_prof.h"
 
 typedef struct bin_stats_s bin_stats_t;
@@ -11,47 +12,47 @@ struct bin_stats_s {
 	 * many times, resulting many increments to nrequests, but only one
 	 * each to nmalloc and ndalloc.
 	 */
-	uint64_t	nmalloc;
+	uint64_t nmalloc;
-	uint64_t	ndalloc;
+	uint64_t ndalloc;
 
 	/*
 	 * Number of allocation requests that correspond to the size of this
 	 * bin.  This includes requests served by tcache, though tcache only
 	 * periodically merges into this counter.
 	 */
-	uint64_t	nrequests;
+	uint64_t nrequests;
 
 	/*
 	 * Current number of regions of this size class, including regions
 	 * currently cached by tcache.
 	 */
-	size_t		curregs;
+	size_t curregs;
 
 	/* Number of tcache fills from this bin. */
-	uint64_t	nfills;
+	uint64_t nfills;
 
 	/* Number of tcache flushes to this bin. */
-	uint64_t	nflushes;
+	uint64_t nflushes;
 
 	/* Total number of slabs created for this bin's size class. */
-	uint64_t	nslabs;
+	uint64_t nslabs;
 
 	/*
 	 * Total number of slabs reused by extracting them from the slabs heap
 	 * for this bin's size class.
 	 */
-	uint64_t	reslabs;
+	uint64_t reslabs;
 
 	/* Current number of slabs in this bin. */
-	size_t		curslabs;
+	size_t curslabs;
 
 	/* Current size of nonfull slabs heap in this bin. */
-	size_t		nonfull_slabs;
+	size_t nonfull_slabs;
 };
 
 typedef struct bin_stats_data_s bin_stats_data_t;
 struct bin_stats_data_s {
-	bin_stats_t stats_data;
+	bin_stats_t       stats_data;
 	mutex_prof_data_t mutex_data;
 };
 #endif /* JEMALLOC_INTERNAL_BIN_STATS_H */

dep/jemalloc/include/jemalloc/internal/bin_types.h

@@ -1,13 +1,17 @@
 #ifndef JEMALLOC_INTERNAL_BIN_TYPES_H
 #define JEMALLOC_INTERNAL_BIN_TYPES_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/sc.h"
 
 #define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
 #define N_BIN_SHARDS_DEFAULT 1
 
 /* Used in TSD static initializer only. Real init in arena_bind(). */
-#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}}
+#define TSD_BINSHARDS_ZERO_INITIALIZER                                         \
+	{                                                                      \
+		{ UINT8_MAX }                                                  \
+	}
 
 typedef struct tsd_binshards_s tsd_binshards_t;
 struct tsd_binshards_s {

dep/jemalloc/include/jemalloc/internal/bit_util.h

@@ -1,12 +1,13 @@
 #ifndef JEMALLOC_INTERNAL_BIT_UTIL_H
 #define JEMALLOC_INTERNAL_BIT_UTIL_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/assert.h"
 
 /* Sanity check. */
-#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
+#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL)      \
     || !defined(JEMALLOC_INTERNAL_FFS)
-#  error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
+#	error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
 #endif
 
 /*
@@ -34,6 +35,7 @@ ffs_u(unsigned x) {
 	return JEMALLOC_INTERNAL_FFS(x) - 1;
 }
 
+/* clang-format off */
 #define DO_FLS_SLOW(x, suffix) do {					\
 	util_assume(x != 0);						\
 	x |= (x >> 1);							\
@@ -57,6 +59,7 @@ ffs_u(unsigned x) {
 	}								\
 	return ffs_##suffix(x) - 1;					\
 } while(0)
+/* clang-format on */
 
 static inline unsigned
 fls_llu_slow(unsigned long long x) {
@@ -107,16 +110,19 @@ fls_u(unsigned x) {
 }
 #elif defined(_MSC_VER)
 
-#if LG_SIZEOF_PTR == 3
+#	if LG_SIZEOF_PTR == 3
-#define DO_BSR64(bit, x) _BitScanReverse64(&bit, x)
+#		define DO_BSR64(bit, x) _BitScanReverse64(&bit, x)
-#else
+#	else
 /*
  * This never actually runs; we're just dodging a compiler error for the
  * never-taken branch where sizeof(void *) == 8.
  */
-#define DO_BSR64(bit, x) bit = 0; unreachable()
+#		define DO_BSR64(bit, x)                                       \
-#endif
+			bit = 0;                                               \
+			unreachable()
+#	endif
 
+/* clang-format off */
 #define DO_FLS(x) do {							\
 	if (x == 0) {							\
 		return 8 * sizeof(x);					\
@@ -143,6 +149,7 @@ fls_u(unsigned x) {
 	}								\
 	unreachable();							\
 } while (0)
+/* clang-format on */
 
 static inline unsigned
 fls_llu(unsigned long long x) {
@@ -159,8 +166,8 @@ fls_u(unsigned x) {
 	DO_FLS(x);
 }
 
-#undef DO_FLS
+#	undef DO_FLS
-#undef DO_BSR64
+#	undef DO_BSR64
 #else
 
 static inline unsigned
@@ -180,14 +187,15 @@ fls_u(unsigned x) {
 #endif
 
 #if LG_SIZEOF_LONG_LONG > 3
-#  error "Haven't implemented popcount for 16-byte ints."
+#	error "Haven't implemented popcount for 16-byte ints."
 #endif
 
+/* clang-format off */
 #define DO_POPCOUNT(x, type) do {					\
 	/*								\
 	 * Algorithm from an old AMD optimization reference manual.	\
 	 * We're putting a little bit more work than you might expect	\
-	 * into the no-instrinsic case, since we only support the	\
+	 * into the no-intrinsic case, since we only support the	\
 	 * GCC intrinsics spelling of popcount (for now).  Detecting	\
 	 * whether or not the popcount builtin is actually useable in	\
 	 * MSVC is nontrivial.						\
@@ -204,7 +212,7 @@ fls_u(unsigned x) {
 	 * That costs an extra mask relative to this, though.		\
 	 */								\
 	x = x - ((x >> 1) & (0x55U * bmul));				\
-	/* Replace each 4 bits with their sideays sum.  0x3 = 0b0011. */\
+	/* Replace each 4 bits with their sideways sum.  0x3 = 0b0011. */\
 	x = (x & (bmul * 0x33U)) + ((x >> 2) & (bmul * 0x33U));		\
 	/*								\
 	 * Replace each 8 bits with their sideways sum.  Note that we	\
@@ -226,6 +234,7 @@ fls_u(unsigned x) {
 	x >>= ((sizeof(x) - 1) * 8);					\
 	return (unsigned)x;						\
 } while(0)
+/* clang-format on */
 
 static inline unsigned
 popcount_u_slow(unsigned bitmap) {
@@ -277,7 +286,7 @@ popcount_llu(unsigned long long bitmap) {
  */
 
 static inline size_t
-cfs_lu(unsigned long* bitmap) {
+cfs_lu(unsigned long *bitmap) {
 	util_assume(*bitmap != 0);
 	size_t bit = ffs_lu(*bitmap);
 	*bitmap ^= ZU(1) << bit;
@@ -293,7 +302,7 @@ ffs_zu(size_t x) {
 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
 	return ffs_llu(x);
 #else
-#error No implementation for size_t ffs()
+#	error No implementation for size_t ffs()
 #endif
 }
 
@@ -306,11 +315,10 @@ fls_zu(size_t x) {
 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
 	return fls_llu(x);
 #else
-#error No implementation for size_t fls()
+#	error No implementation for size_t fls()
 #endif
 }
 
-
 static inline unsigned
 ffs_u64(uint64_t x) {
 #if LG_SIZEOF_LONG == 3
@@ -318,7 +326,7 @@ ffs_u64(uint64_t x) {
 #elif LG_SIZEOF_LONG_LONG == 3
 	return ffs_llu(x);
 #else
-#error No implementation for 64-bit ffs()
+#	error No implementation for 64-bit ffs()
 #endif
 }
 
@@ -329,7 +337,7 @@ fls_u64(uint64_t x) {
 #elif LG_SIZEOF_LONG_LONG == 3
 	return fls_llu(x);
 #else
-#error No implementation for 64-bit fls()
+#	error No implementation for 64-bit fls()
 #endif
 }
 
@@ -338,9 +346,8 @@ ffs_u32(uint32_t x) {
 #if LG_SIZEOF_INT == 2
 	return ffs_u(x);
 #else
-#error No implementation for 32-bit ffs()
+#	error No implementation for 32-bit ffs()
 #endif
-	return ffs_u(x);
 }
 
 static inline unsigned
@@ -348,9 +355,8 @@ fls_u32(uint32_t x) {
 #if LG_SIZEOF_INT == 2
 	return fls_u(x);
 #else
-#error No implementation for 32-bit fls()
+#	error No implementation for 32-bit fls()
 #endif
-	return fls_u(x);
 }
 
 static inline uint64_t
@@ -370,7 +376,7 @@ pow2_ceil_u64(uint64_t x) {
 static inline uint32_t
 pow2_ceil_u32(uint32_t x) {
 	if (unlikely(x <= 1)) {
-	    return x;
+		return x;
 	}
 	size_t msb_on_index = fls_u32(x - 1);
 	/* As above. */
@@ -408,13 +414,16 @@ lg_ceil(size_t x) {
 #define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1))
 #define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2))
 #define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4))
-#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8))
+#define LG_FLOOR_16(x)                                                         \
-#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16))
+	(x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8))
-#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32))
+#define LG_FLOOR_32(x)                                                         \
+	(x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16))
+#define LG_FLOOR_64(x)                                                         \
+	(x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32))
 #if LG_SIZEOF_PTR == 2
-#  define LG_FLOOR(x) LG_FLOOR_32((x))
+#	define LG_FLOOR(x) LG_FLOOR_32((x))
 #else
-#  define LG_FLOOR(x) LG_FLOOR_64((x))
+#	define LG_FLOOR(x) LG_FLOOR_64((x))
 #endif
 
 #define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1))

dep/jemalloc/include/jemalloc/internal/bitmap.h

@@ -1,26 +1,27 @@
 #ifndef JEMALLOC_INTERNAL_BITMAP_H
 #define JEMALLOC_INTERNAL_BITMAP_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/sc.h"
 
 typedef unsigned long bitmap_t;
-#define LG_SIZEOF_BITMAP	LG_SIZEOF_LONG
+#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG
 
 /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
 #if SC_LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES)
 /* Maximum bitmap bit count is determined by maximum regions per slab. */
-#  define LG_BITMAP_MAXBITS	SC_LG_SLAB_MAXREGS
+#	define LG_BITMAP_MAXBITS SC_LG_SLAB_MAXREGS
 #else
 /* Maximum bitmap bit count is determined by number of extent size classes. */
-#  define LG_BITMAP_MAXBITS	LG_CEIL(SC_NSIZES)
+#	define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES)
 #endif
-#define BITMAP_MAXBITS		(ZU(1) << LG_BITMAP_MAXBITS)
+#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
 
 /* Number of bits per group. */
-#define LG_BITMAP_GROUP_NBITS		(LG_SIZEOF_BITMAP + 3)
+#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3)
-#define BITMAP_GROUP_NBITS		(1U << LG_BITMAP_GROUP_NBITS)
+#define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS)
-#define BITMAP_GROUP_NBITS_MASK		(BITMAP_GROUP_NBITS-1)
+#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS - 1)
 
 /*
  * Do some analysis on how big the bitmap is before we use a tree.  For a brute
@@ -28,67 +29,64 @@ typedef unsigned long bitmap_t;
  * use a tree instead.
  */
 #if LG_BITMAP_MAXBITS - LG_BITMAP_GROUP_NBITS > 3
-#  define BITMAP_USE_TREE
+#	define BITMAP_USE_TREE
 #endif
 
 /* Number of groups required to store a given number of bits. */
-#define BITMAP_BITS2GROUPS(nbits)					\
+#define BITMAP_BITS2GROUPS(nbits)                                              \
-    (((nbits) + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS)
+	(((nbits) + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS)
 
 /*
  * Number of groups required at a particular level for a given number of bits.
  */
-#define BITMAP_GROUPS_L0(nbits)						\
+#define BITMAP_GROUPS_L0(nbits) BITMAP_BITS2GROUPS(nbits)
-    BITMAP_BITS2GROUPS(nbits)
+#define BITMAP_GROUPS_L1(nbits) BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits))
-#define BITMAP_GROUPS_L1(nbits)						\
+#define BITMAP_GROUPS_L2(nbits)                                                \
-    BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits))
+	BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))
-#define BITMAP_GROUPS_L2(nbits)						\
+#define BITMAP_GROUPS_L3(nbits)                                                \
-    BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))
+	BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(                                 \
-#define BITMAP_GROUPS_L3(nbits)						\
+	    BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))))
-    BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(		\
+#define BITMAP_GROUPS_L4(nbits)                                                \
-	BITMAP_BITS2GROUPS((nbits)))))
+	BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(              \
-#define BITMAP_GROUPS_L4(nbits)						\
+	    BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))))
-    BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(		\
-	BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))))
 
 /*
  * Assuming the number of levels, number of groups required for a given number
  * of bits.
  */
-#define BITMAP_GROUPS_1_LEVEL(nbits)					\
+#define BITMAP_GROUPS_1_LEVEL(nbits) BITMAP_GROUPS_L0(nbits)
-    BITMAP_GROUPS_L0(nbits)
+#define BITMAP_GROUPS_2_LEVEL(nbits)                                           \
-#define BITMAP_GROUPS_2_LEVEL(nbits)					\
+	(BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits))
-    (BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits))
+#define BITMAP_GROUPS_3_LEVEL(nbits)                                           \
-#define BITMAP_GROUPS_3_LEVEL(nbits)					\
+	(BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits))
-    (BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits))
+#define BITMAP_GROUPS_4_LEVEL(nbits)                                           \
-#define BITMAP_GROUPS_4_LEVEL(nbits)					\
+	(BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits))
-    (BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits))
+#define BITMAP_GROUPS_5_LEVEL(nbits)                                           \
-#define BITMAP_GROUPS_5_LEVEL(nbits)					\
+	(BITMAP_GROUPS_4_LEVEL(nbits) + BITMAP_GROUPS_L4(nbits))
-    (BITMAP_GROUPS_4_LEVEL(nbits) + BITMAP_GROUPS_L4(nbits))
 
 /*
  * Maximum number of groups required to support LG_BITMAP_MAXBITS.
  */
 #ifdef BITMAP_USE_TREE
 
-#if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS
+#	if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS
-#  define BITMAP_GROUPS(nbits)	BITMAP_GROUPS_1_LEVEL(nbits)
+#		define BITMAP_GROUPS(nbits) BITMAP_GROUPS_1_LEVEL(nbits)
-#  define BITMAP_GROUPS_MAX	BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS)
+#		define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS)
-#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2
+#	elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2
-#  define BITMAP_GROUPS(nbits)	BITMAP_GROUPS_2_LEVEL(nbits)
+#		define BITMAP_GROUPS(nbits) BITMAP_GROUPS_2_LEVEL(nbits)
-#  define BITMAP_GROUPS_MAX	BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS)
+#		define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS)
-#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3
+#	elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3
-#  define BITMAP_GROUPS(nbits)	BITMAP_GROUPS_3_LEVEL(nbits)
+#		define BITMAP_GROUPS(nbits) BITMAP_GROUPS_3_LEVEL(nbits)
-#  define BITMAP_GROUPS_MAX	BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS)
+#		define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS)
-#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4
+#	elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4
-#  define BITMAP_GROUPS(nbits)	BITMAP_GROUPS_4_LEVEL(nbits)
+#		define BITMAP_GROUPS(nbits) BITMAP_GROUPS_4_LEVEL(nbits)
-#  define BITMAP_GROUPS_MAX	BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS)
+#		define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS)
-#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5
+#	elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5
-#  define BITMAP_GROUPS(nbits)	BITMAP_GROUPS_5_LEVEL(nbits)
+#		define BITMAP_GROUPS(nbits) BITMAP_GROUPS_5_LEVEL(nbits)
-#  define BITMAP_GROUPS_MAX	BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS)
+#		define BITMAP_GROUPS_MAX BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS)
-#else
+#	else
-#  error "Unsupported bitmap size"
+#		error "Unsupported bitmap size"
-#endif
+#	endif
 
 /*
  * Maximum number of levels possible.  This could be statically computed based
@@ -104,42 +102,53 @@ typedef unsigned long bitmap_t;
  * unused trailing entries in bitmap_info_t structures; the bitmaps themselves
  * are not impacted.
  */
-#define BITMAP_MAX_LEVELS	5
+#	define BITMAP_MAX_LEVELS 5
 
-#define BITMAP_INFO_INITIALIZER(nbits) {				\
+#	define BITMAP_INFO_INITIALIZER(nbits)                                 \
-	/* nbits. */							\
+		{                                                              \
-	nbits,								\
+			/* nbits. */                                           \
-	/* nlevels. */							\
+			nbits, /* nlevels. */                                  \
-	(BITMAP_GROUPS_L0(nbits) > BITMAP_GROUPS_L1(nbits)) +		\
+			    (BITMAP_GROUPS_L0(nbits)                           \
-	    (BITMAP_GROUPS_L1(nbits) > BITMAP_GROUPS_L2(nbits)) +	\
+			        > BITMAP_GROUPS_L1(nbits))                     \
-	    (BITMAP_GROUPS_L2(nbits) > BITMAP_GROUPS_L3(nbits)) +	\
+			    + (BITMAP_GROUPS_L1(nbits)                         \
-	    (BITMAP_GROUPS_L3(nbits) > BITMAP_GROUPS_L4(nbits)) + 1,	\
+			        > BITMAP_GROUPS_L2(nbits))                     \
-	/* levels. */							\
+			    + (BITMAP_GROUPS_L2(nbits)                         \
-	{								\
+			        > BITMAP_GROUPS_L3(nbits))                     \
-		{0},							\
+			    + (BITMAP_GROUPS_L3(nbits)                         \
-		{BITMAP_GROUPS_L0(nbits)},				\
+			        > BITMAP_GROUPS_L4(nbits))                     \
-		{BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)},	\
+			    + 1, /* levels. */                                 \
-		{BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) +	\
+			{                                                      \
-		    BITMAP_GROUPS_L0(nbits)},				\
+				{0}, {BITMAP_GROUPS_L0(nbits)},                \
-		{BITMAP_GROUPS_L3(nbits) + BITMAP_GROUPS_L2(nbits) +	\
+				    {BITMAP_GROUPS_L1(nbits)                   \
-		    BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)},	\
+				        + BITMAP_GROUPS_L0(nbits)},            \
-		{BITMAP_GROUPS_L4(nbits) + BITMAP_GROUPS_L3(nbits) +	\
+				    {BITMAP_GROUPS_L2(nbits)                   \
-		     BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits)	\
+				        + BITMAP_GROUPS_L1(nbits)              \
-		     + BITMAP_GROUPS_L0(nbits)}				\
+				        + BITMAP_GROUPS_L0(nbits)},            \
-	}								\
+				    {BITMAP_GROUPS_L3(nbits)                   \
-}
+				        + BITMAP_GROUPS_L2(nbits)              \
+				        + BITMAP_GROUPS_L1(nbits)              \
+				        + BITMAP_GROUPS_L0(nbits)},            \
+				{                                              \
+					BITMAP_GROUPS_L4(nbits)                \
+					    + BITMAP_GROUPS_L3(nbits)          \
+					    + BITMAP_GROUPS_L2(nbits)          \
+					    + BITMAP_GROUPS_L1(nbits)          \
+					    + BITMAP_GROUPS_L0(nbits)          \
+				}                                              \
+			}                                                      \
+		}
 
 #else /* BITMAP_USE_TREE */
 
-#define BITMAP_GROUPS(nbits)	BITMAP_BITS2GROUPS(nbits)
+#	define BITMAP_GROUPS(nbits) BITMAP_BITS2GROUPS(nbits)
-#define BITMAP_GROUPS_MAX	BITMAP_BITS2GROUPS(BITMAP_MAXBITS)
+#	define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS)
 
-#define BITMAP_INFO_INITIALIZER(nbits) {				\
+#	define BITMAP_INFO_INITIALIZER(nbits)                                 \
-	/* nbits. */							\
+		{                                                              \
-	nbits,								\
+			/* nbits. */                                           \
-	/* ngroups. */							\
+			nbits, /* ngroups. */                                  \
-	BITMAP_BITS2GROUPS(nbits)					\
+			    BITMAP_BITS2GROUPS(nbits)                          \
-}
+		}
 
 #endif /* BITMAP_USE_TREE */
 
@@ -160,21 +169,21 @@ typedef struct bitmap_info_s {
 	 * Only the first (nlevels+1) elements are used, and levels are ordered
 	 * bottom to top (e.g. the bottom level is stored in levels[0]).
 	 */
-	bitmap_level_t levels[BITMAP_MAX_LEVELS+1];
+	bitmap_level_t levels[BITMAP_MAX_LEVELS + 1];
-#else /* BITMAP_USE_TREE */
+#else  /* BITMAP_USE_TREE */
 	/* Number of groups necessary for nbits. */
 	size_t ngroups;
 #endif /* BITMAP_USE_TREE */
 } bitmap_info_t;
 
-void bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
+void   bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
-void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
+void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
 size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
 bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 #ifdef BITMAP_USE_TREE
-	size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
+	size_t   rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
 	bitmap_t rg = bitmap[rgoff];
 	/* The bitmap is full iff the root group is 0. */
 	return (rg == 0);
@@ -192,7 +201,7 @@ bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 
 static inline bool
 bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
-	size_t goff;
+	size_t   goff;
 	bitmap_t g;
 
 	assert(bit < binfo->nbits);
@@ -203,9 +212,9 @@ bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
 
 static inline void
 bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
-	size_t goff;
+	size_t    goff;
 	bitmap_t *gp;
-	bitmap_t g;
+	bitmap_t  g;
 
 	assert(bit < binfo->nbits);
 	assert(!bitmap_get(bitmap, binfo, bit));
@@ -244,12 +253,13 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
 #ifdef BITMAP_USE_TREE
 	size_t bit = 0;
 	for (unsigned level = binfo->nlevels; level--;) {
-		size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level +
+		size_t   lg_bits_per_group = (LG_BITMAP_GROUP_NBITS
-		    1));
+                    * (level + 1));
-		bitmap_t group = bitmap[binfo->levels[level].group_offset + (bit
+		bitmap_t group = bitmap[binfo->levels[level].group_offset
-		    >> lg_bits_per_group)];
+		    + (bit >> lg_bits_per_group)];
-		unsigned group_nmask = (unsigned)(((min_bit > bit) ? (min_bit -
+		unsigned group_nmask =
-		    bit) : 0) >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS));
+		    (unsigned)(((min_bit > bit) ? (min_bit - bit) : 0)
+		        >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS));
 		assert(group_nmask <= BITMAP_GROUP_NBITS);
 		bitmap_t group_mask = ~((1LU << group_nmask) - 1);
 		bitmap_t group_masked = group & group_mask;
@@ -272,25 +282,28 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
 			}
 			return bitmap_ffu(bitmap, binfo, sib_base);
 		}
-		bit += ((size_t)ffs_lu(group_masked)) <<
+		bit += ((size_t)ffs_lu(group_masked))
-		    (lg_bits_per_group - LG_BITMAP_GROUP_NBITS);
+		    << (lg_bits_per_group - LG_BITMAP_GROUP_NBITS);
 	}
 	assert(bit >= min_bit);
 	assert(bit < binfo->nbits);
 	return bit;
 #else
-	size_t i = min_bit >> LG_BITMAP_GROUP_NBITS;
+	size_t   i = min_bit >> LG_BITMAP_GROUP_NBITS;
-	bitmap_t g = bitmap[i] & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK))
+	bitmap_t g = bitmap[i]
-	    - 1);
+	    & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK)) - 1);
 	size_t bit;
-	do {
+	while (1) {
 		if (g != 0) {
 			bit = ffs_lu(g);
 			return (i << LG_BITMAP_GROUP_NBITS) + bit;
 		}
 		i++;
+		if (i >= binfo->ngroups) {
+			break;
+		}
 		g = bitmap[i];
-	} while (i < binfo->ngroups);
+	}
 	return binfo->nbits;
 #endif
 }
@@ -298,7 +311,7 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
 /* sfu: set first unset. */
 static inline size_t
 bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) {
-	size_t bit;
+	size_t   bit;
 	bitmap_t g;
 	unsigned i;
 
@@ -328,9 +341,9 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 
 static inline void
 bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
-	size_t goff;
+	size_t      goff;
-	bitmap_t *gp;
+	bitmap_t   *gp;
-	bitmap_t g;
+	bitmap_t    g;
 	UNUSED bool propagate;
 
 	assert(bit < binfo->nbits);

dep/jemalloc/include/jemalloc/internal/buf_writer.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_BUF_WRITER_H
 #define JEMALLOC_INTERNAL_BUF_WRITER_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/tsd_types.h"
+
 /*
  * Note: when using the buffered writer, cbopaque is passed to write_cb only
  * when the buffer is flushed.  It would make a difference if cbopaque points
@@ -12,21 +16,21 @@
 
 typedef struct {
 	write_cb_t *write_cb;
-	void *cbopaque;
+	void       *cbopaque;
-	char *buf;
+	char       *buf;
-	size_t buf_size;
+	size_t      buf_size;
-	size_t buf_end;
+	size_t      buf_end;
-	bool internal_buf;
+	bool        internal_buf;
 } buf_writer_t;
 
-bool buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer,
+bool       buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer,
-    write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len);
+          write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len);
-void buf_writer_flush(buf_writer_t *buf_writer);
+void       buf_writer_flush(buf_writer_t *buf_writer);
 write_cb_t buf_writer_cb;
-void buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
+void       buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
 
-typedef ssize_t (read_cb_t)(void *read_cbopaque, void *buf, size_t limit);
+typedef ssize_t(read_cb_t)(void *read_cbopaque, void *buf, size_t limit);
-void buf_writer_pipe(buf_writer_t *buf_writer, read_cb_t *read_cb,
+void buf_writer_pipe(
-    void *read_cbopaque);
+    buf_writer_t *buf_writer, read_cb_t *read_cb, void *read_cbopaque);
 
 #endif /* JEMALLOC_INTERNAL_BUF_WRITER_H */

dep/jemalloc/include/jemalloc/internal/cache_bin.h

@@ -1,7 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_CACHE_BIN_H
 #define JEMALLOC_INTERNAL_CACHE_BIN_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sz.h"
 
 /*
@@ -20,16 +23,20 @@
  */
 typedef uint16_t cache_bin_sz_t;
 
+#define JUNK_ADDR ((uintptr_t)0x7a7a7a7a7a7a7a7aULL)
 /*
  * Leave a noticeable mark pattern on the cache bin stack boundaries, in case a
  * bug starts leaking those.  Make it look like the junk pattern but be distinct
  * from it.
  */
-static const uintptr_t cache_bin_preceding_junk =
+static const uintptr_t cache_bin_preceding_junk = JUNK_ADDR;
-    (uintptr_t)0x7a7a7a7a7a7a7a7aULL;
+/* Note: JUNK_ADDR vs. JUNK_ADDR + 1 -- this tells you which pointer leaked. */
-/* Note: a7 vs. 7a above -- this tells you which pointer leaked. */
+static const uintptr_t cache_bin_trailing_junk = JUNK_ADDR + 1;
-static const uintptr_t cache_bin_trailing_junk =
+/*
-    (uintptr_t)0xa7a7a7a7a7a7a7a7ULL;
+ * A pointer used to initialize a fake stack_head for disabled small bins
+ * so that the enabled/disabled assessment does not rely on ncached_max.
+ */
+extern const uintptr_t disabled_bin;
 
 /*
  * That implies the following value, for the maximum number of items in any
@@ -38,8 +45,8 @@ static const uintptr_t cache_bin_trailing_junk =
  *   1 << (sizeof(cache_bin_sz_t) * 8)
  * bytes spread across pointer sized objects to get the maximum.
  */
-#define CACHE_BIN_NCACHED_MAX (((size_t)1 << sizeof(cache_bin_sz_t) * 8) \
+#define CACHE_BIN_NCACHED_MAX                                                  \
-    / sizeof(void *) - 1)
+	(((size_t)1 << sizeof(cache_bin_sz_t) * 8) / sizeof(void *) - 1)
 
 /*
  * This lives inside the cache_bin (for locality reasons), and is initialized
@@ -101,7 +108,7 @@ struct cache_bin_s {
 	 * Since the stack grows down, this is a higher address than
 	 * low_bits_full.
 	 */
-	uint16_t low_bits_low_water;
+	cache_bin_sz_t low_bits_low_water;
 
 	/*
 	 * The low bits of the value that stack_head will take on when the array
@@ -112,7 +119,7 @@ struct cache_bin_s {
 	 * Recall that since the stack grows down, this is the lowest available
 	 * address in the array for caching.  Only adjusted when stashing items.
 	 */
-	uint16_t low_bits_full;
+	cache_bin_sz_t low_bits_full;
 
 	/*
 	 * The low bits of the value that stack_head will take on when the array
@@ -121,7 +128,10 @@ struct cache_bin_s {
 	 * The stack grows down -- this is one past the highest address in the
 	 * array.  Immutable after initialization.
 	 */
-	uint16_t low_bits_empty;
+	cache_bin_sz_t low_bits_empty;
+
+	/* The maximum number of cached items in the bin. */
+	cache_bin_info_t bin_info;
 };
 
 /*
@@ -142,8 +152,8 @@ struct cache_bin_array_descriptor_s {
 };
 
 static inline void
-cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor,
+cache_bin_array_descriptor_init(
-    cache_bin_t *bins) {
+    cache_bin_array_descriptor_t *descriptor, cache_bin_t *bins) {
 	ql_elm_new(descriptor, link);
 	descriptor->bins = bins;
 }
@@ -168,10 +178,41 @@ cache_bin_nonfast_aligned(const void *ptr) {
 	return ((uintptr_t)ptr & san_cache_bin_nonfast_mask) == 0;
 }
 
+static inline const void *
+cache_bin_disabled_bin_stack(void) {
+	return &disabled_bin;
+}
+
+/*
+ * If a cache bin was zero initialized (either because it lives in static or
+ * thread-local storage, or was memset to 0), this function indicates whether or
+ * not cache_bin_init was called on it.
+ */
+static inline bool
+cache_bin_still_zero_initialized(cache_bin_t *bin) {
+	return bin->stack_head == NULL;
+}
+
+static inline bool
+cache_bin_disabled(cache_bin_t *bin) {
+	bool disabled = (bin->stack_head == cache_bin_disabled_bin_stack());
+	if (disabled) {
+		assert((uintptr_t)(*bin->stack_head) == JUNK_ADDR);
+	}
+	return disabled;
+}
+
+/* Gets ncached_max without asserting that the bin is enabled. */
+static inline cache_bin_sz_t
+cache_bin_ncached_max_get_unsafe(cache_bin_t *bin) {
+	return bin->bin_info.ncached_max;
+}
+
 /* Returns ncached_max: Upper limit on ncached. */
 static inline cache_bin_sz_t
-cache_bin_info_ncached_max(cache_bin_info_t *info) {
+cache_bin_ncached_max_get(cache_bin_t *bin) {
-	return info->ncached_max;
+	assert(!cache_bin_disabled(bin));
+	return cache_bin_ncached_max_get_unsafe(bin);
 }
 
 /*
@@ -181,7 +222,8 @@ cache_bin_info_ncached_max(cache_bin_info_t *info) {
  * with later.
  */
 static inline void
-cache_bin_assert_earlier(cache_bin_t *bin, uint16_t earlier, uint16_t later) {
+cache_bin_assert_earlier(
+    cache_bin_t *bin, cache_bin_sz_t earlier, cache_bin_sz_t later) {
 	if (earlier > later) {
 		assert(bin->low_bits_full > bin->low_bits_empty);
 	}
@@ -193,28 +235,19 @@ cache_bin_assert_earlier(cache_bin_t *bin, uint16_t earlier, uint16_t later) {
  * Does difference calculations that handle wraparound correctly.  Earlier must
  * be associated with the position earlier in memory.
  */
-static inline uint16_t
+static inline cache_bin_sz_t
-cache_bin_diff(cache_bin_t *bin, uint16_t earlier, uint16_t later, bool racy) {
+cache_bin_diff(cache_bin_t *bin, cache_bin_sz_t earlier, cache_bin_sz_t later) {
-	/*
+	cache_bin_assert_earlier(bin, earlier, later);
-	 * When it's racy, bin->low_bits_full can be modified concurrently. It
-	 * can cross the uint16_t max value and become less than
-	 * bin->low_bits_empty at the time of the check.
-	 */
-	if (!racy) {
-		cache_bin_assert_earlier(bin, earlier, later);
-	}
 	return later - earlier;
 }
 
 /*
  * Number of items currently cached in the bin, without checking ncached_max.
- * We require specifying whether or not the request is racy or not (i.e. whether
- * or not concurrent modifications are possible).
  */
 static inline cache_bin_sz_t
-cache_bin_ncached_get_internal(cache_bin_t *bin, bool racy) {
+cache_bin_ncached_get_internal(cache_bin_t *bin) {
 	cache_bin_sz_t diff = cache_bin_diff(bin,
-	    (uint16_t)(uintptr_t)bin->stack_head, bin->low_bits_empty, racy);
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head, bin->low_bits_empty);
 	cache_bin_sz_t n = diff / sizeof(void *);
 	/*
 	 * We have undefined behavior here; if this function is called from the
@@ -225,7 +258,7 @@ cache_bin_ncached_get_internal(cache_bin_t *bin, bool racy) {
 	 * fast paths.  This should still be "safe" in the sense of generating
 	 * the correct assembly for the foreseeable future, though.
 	 */
-	assert(n == 0 || *(bin->stack_head) != NULL || racy);
+	assert(n == 0 || *(bin->stack_head) != NULL);
 	return n;
 }
 
@@ -235,10 +268,9 @@ cache_bin_ncached_get_internal(cache_bin_t *bin, bool racy) {
  * possible.
  */
 static inline cache_bin_sz_t
-cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_ncached_get_local(cache_bin_t *bin) {
-	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin,
+	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
-	    /* racy */ false);
+	assert(n <= cache_bin_ncached_max_get(bin));
-	assert(n <= cache_bin_info_ncached_max(info));
 	return n;
 }
 
@@ -253,10 +285,9 @@ cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
 static inline void **
 cache_bin_empty_position_get(cache_bin_t *bin) {
 	cache_bin_sz_t diff = cache_bin_diff(bin,
-	    (uint16_t)(uintptr_t)bin->stack_head, bin->low_bits_empty,
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head, bin->low_bits_empty);
-	    /* racy */ false);
+	byte_t        *empty_bits = (byte_t *)bin->stack_head + diff;
-	uintptr_t empty_bits = (uintptr_t)bin->stack_head + diff;
+	void         **ret = (void **)empty_bits;
-	void **ret = (void **)empty_bits;
 
 	assert(ret >= bin->stack_head);
 
@@ -273,10 +304,10 @@ cache_bin_empty_position_get(cache_bin_t *bin) {
  * multithreaded environment. Currently concurrent access happens only during
  * arena statistics collection.
  */
-static inline uint16_t
+static inline cache_bin_sz_t
-cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_low_bits_low_bound_get(cache_bin_t *bin) {
-	return (uint16_t)bin->low_bits_empty -
+	return (cache_bin_sz_t)bin->low_bits_empty
-	    info->ncached_max * sizeof(void *);
+	    - cache_bin_ncached_max_get(bin) * sizeof(void *);
 }
 
 /*
@@ -285,9 +316,9 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
  * A pointer to the position with the lowest address of the backing array.
  */
 static inline void **
-cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_low_bound_get(cache_bin_t *bin) {
-	cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info);
+	cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
-	void **ret = cache_bin_empty_position_get(bin) - ncached_max;
+	void         **ret = cache_bin_empty_position_get(bin) - ncached_max;
 	assert(ret <= bin->stack_head);
 
 	return ret;
@@ -298,8 +329,8 @@ cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
  * batch fill a nonempty cache bin.
  */
 static inline void
-cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_assert_empty(cache_bin_t *bin) {
-	assert(cache_bin_ncached_get_local(bin, info) == 0);
+	assert(cache_bin_ncached_get_local(bin) == 0);
 	assert(cache_bin_empty_position_get(bin) == bin->stack_head);
 }
 
@@ -310,18 +341,19 @@ cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) {
  */
 static inline cache_bin_sz_t
 cache_bin_low_water_get_internal(cache_bin_t *bin) {
-	return cache_bin_diff(bin, bin->low_bits_low_water,
+	return cache_bin_diff(bin, bin->low_bits_low_water, bin->low_bits_empty)
-	    bin->low_bits_empty, /* racy */ false) / sizeof(void *);
+	    / sizeof(void *);
 }
 
 /* Returns the numeric value of low water in [0, ncached]. */
 static inline cache_bin_sz_t
-cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_low_water_get(cache_bin_t *bin) {
 	cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin);
-	assert(low_water <= cache_bin_info_ncached_max(info));
+	assert(low_water <= cache_bin_ncached_max_get(bin));
-	assert(low_water <= cache_bin_ncached_get_local(bin, info));
+	assert(low_water <= cache_bin_ncached_get_local(bin));
 
-	cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head,
+	cache_bin_assert_earlier(bin,
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head,
 	    bin->low_bits_low_water);
 
 	return low_water;
@@ -333,12 +365,14 @@ cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) {
  */
 static inline void
 cache_bin_low_water_set(cache_bin_t *bin) {
-	bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head;
+	assert(!cache_bin_disabled(bin));
+	bin->low_bits_low_water = (cache_bin_sz_t)(uintptr_t)bin->stack_head;
 }
 
 static inline void
 cache_bin_low_water_adjust(cache_bin_t *bin) {
-	if (cache_bin_ncached_get_internal(bin, /* racy */ false)
+	assert(!cache_bin_disabled(bin));
+	if (cache_bin_ncached_get_internal(bin)
 	    < cache_bin_low_water_get_internal(bin)) {
 		cache_bin_low_water_set(bin);
 	}
@@ -358,9 +392,9 @@ cache_bin_alloc_impl(cache_bin_t *bin, bool *success, bool adjust_low_water) {
 	 * This may read from the empty position; however the loaded value won't
 	 * be used.  It's safe because the stack has one more slot reserved.
 	 */
-	void *ret = *bin->stack_head;
+	void          *ret = *bin->stack_head;
-	uint16_t low_bits = (uint16_t)(uintptr_t)bin->stack_head;
+	cache_bin_sz_t low_bits = (cache_bin_sz_t)(uintptr_t)bin->stack_head;
-	void **new_head = bin->stack_head + 1;
+	void         **new_head = bin->stack_head + 1;
 
 	/*
 	 * Note that the low water mark is at most empty; if we pass this check,
@@ -382,7 +416,7 @@ cache_bin_alloc_impl(cache_bin_t *bin, bool *success, bool adjust_low_water) {
 	 */
 	if (likely(low_bits != bin->low_bits_empty)) {
 		bin->stack_head = new_head;
-		bin->low_bits_low_water = (uint16_t)(uintptr_t)new_head;
+		bin->low_bits_low_water = (cache_bin_sz_t)(uintptr_t)new_head;
 		*success = true;
 		return ret;
 	}
@@ -410,8 +444,7 @@ cache_bin_alloc(cache_bin_t *bin, bool *success) {
 
 JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
 cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
-	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin,
+	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
-	    /* racy */ false);
 	if (n > num) {
 		n = (cache_bin_sz_t)num;
 	}
@@ -424,7 +457,37 @@ cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
 
 JEMALLOC_ALWAYS_INLINE bool
 cache_bin_full(cache_bin_t *bin) {
-	return ((uint16_t)(uintptr_t)bin->stack_head == bin->low_bits_full);
+	return (
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head == bin->low_bits_full);
+}
+
+/*
+ * Scans the allocated area of the cache_bin for the given pointer up to limit.
+ * Fires safety_check_fail if the ptr is found and returns true.
+ */
+JEMALLOC_ALWAYS_INLINE bool
+cache_bin_dalloc_safety_checks(cache_bin_t *bin, void *ptr) {
+	if (!config_debug || opt_debug_double_free_max_scan == 0) {
+		return false;
+	}
+
+	cache_bin_sz_t ncached = cache_bin_ncached_get_internal(bin);
+	unsigned       max_scan = opt_debug_double_free_max_scan < ncached
+	          ? opt_debug_double_free_max_scan
+	          : ncached;
+
+	void **cur = bin->stack_head;
+	void **limit = cur + max_scan;
+	for (; cur < limit; cur++) {
+		if (*cur == ptr) {
+			safety_check_fail(
+			    "Invalid deallocation detected: double free of "
+			    "pointer %p\n",
+			    ptr);
+			return true;
+		}
+	}
+	return false;
 }
 
 /*
@@ -436,10 +499,14 @@ cache_bin_dalloc_easy(cache_bin_t *bin, void *ptr) {
 		return false;
 	}
 
+	if (unlikely(cache_bin_dalloc_safety_checks(bin, ptr))) {
+		return true;
+	}
+
 	bin->stack_head--;
 	*bin->stack_head = ptr;
 	cache_bin_assert_earlier(bin, bin->low_bits_full,
-	    (uint16_t)(uintptr_t)bin->stack_head);
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head);
 
 	return true;
 }
@@ -452,11 +519,12 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
 	}
 
 	/* Stash at the full position, in the [full, head) range. */
-	uint16_t low_bits_head = (uint16_t)(uintptr_t)bin->stack_head;
+	cache_bin_sz_t low_bits_head = (cache_bin_sz_t)(uintptr_t)
+	                                   bin->stack_head;
 	/* Wraparound handled as well. */
-	uint16_t diff = cache_bin_diff(bin, bin->low_bits_full, low_bits_head,
+	cache_bin_sz_t diff = cache_bin_diff(
-	    /* racy */ false);
+	    bin, bin->low_bits_full, low_bits_head);
-	*(void **)((uintptr_t)bin->stack_head - diff) = ptr;
+	*(void **)((byte_t *)bin->stack_head - diff) = ptr;
 
 	assert(!cache_bin_full(bin));
 	bin->low_bits_full += sizeof(void *);
@@ -465,67 +533,101 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
 	return true;
 }
 
-/*
+/* Get the number of stashed pointers. */
- * Get the number of stashed pointers.
- *
- * When called from a thread not owning the TLS (i.e. racy = true), it's
- * important to keep in mind that 'bin->stack_head' and 'bin->low_bits_full' can
- * be modified concurrently and almost none assertions about their values can be
- * made.
- */
 JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
-cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info,
+cache_bin_nstashed_get_internal(cache_bin_t *bin) {
-    bool racy) {
+	cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
-	cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info);
+	cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
-	uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin,
+	    bin);
-	    info);
 
-	cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound,
+	cache_bin_sz_t n = cache_bin_diff(
-	    bin->low_bits_full, racy) / sizeof(void *);
+	                       bin, low_bits_low_bound, bin->low_bits_full)
+	    / sizeof(void *);
 	assert(n <= ncached_max);
-
+	if (config_debug && n != 0) {
-	if (!racy) {
 		/* Below are for assertions only. */
-		void **low_bound = cache_bin_low_bound_get(bin, info);
+		void **low_bound = cache_bin_low_bound_get(bin);
 
-		assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound);
+		assert(
+		    (cache_bin_sz_t)(uintptr_t)low_bound == low_bits_low_bound);
 		void *stashed = *(low_bound + n - 1);
-		bool aligned = cache_bin_nonfast_aligned(stashed);
+		bool  aligned = cache_bin_nonfast_aligned(stashed);
 #ifdef JEMALLOC_JET
 		/* Allow arbitrary pointers to be stashed in tests. */
 		aligned = true;
 #endif
-		assert(n == 0 || (stashed != NULL && aligned));
+		assert(stashed != NULL && aligned);
 	}
 
 	return n;
 }
 
 JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
-cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_nstashed_get_local(cache_bin_t *bin) {
-	cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info,
+	cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin);
-	    /* racy */ false);
+	assert(n <= cache_bin_ncached_max_get(bin));
-	assert(n <= cache_bin_info_ncached_max(info));
 	return n;
 }
 
 /*
  * Obtain a racy view of the number of items currently in the cache bin, in the
  * presence of possible concurrent modifications.
+ *
+ * Note that this is the only racy function in this header.  Any other functions
+ * are assumed to be non-racy.  The "racy" term here means accessed from another
+ * thread (that is not the owner of the specific cache bin).  This only happens
+ * when gathering stats (read-only).  The only change because of the racy
+ * condition is that assertions based on mutable fields are omitted.
+ *
+ * It's important to keep in mind that 'bin->stack_head' and
+ * 'bin->low_bits_full' can be modified concurrently and almost no assertions
+ * about their values can be made.
+ *
+ * This function should not call other utility functions because the racy
+ * condition may cause unexpected / undefined behaviors in unverified utility
+ * functions.  Currently, this function calls two utility functions
+ * cache_bin_ncached_max_get and cache_bin_low_bits_low_bound_get because
+ * they help access values that will not be concurrently modified.
  */
 static inline void
-cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
+cache_bin_nitems_get_remote(
-    cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) {
+    cache_bin_t *bin, cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) {
-	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin, /* racy */ true);
+	/* Racy version of cache_bin_ncached_get_internal. */
-	assert(n <= cache_bin_info_ncached_max(info));
+	cache_bin_sz_t diff = bin->low_bits_empty
+	    - (cache_bin_sz_t)(uintptr_t)bin->stack_head;
+	cache_bin_sz_t n = diff / sizeof(void *);
 	*ncached = n;
 
-	n = cache_bin_nstashed_get_internal(bin, info, /* racy */ true);
+	/* Racy version of cache_bin_nstashed_get_internal. */
-	assert(n <= cache_bin_info_ncached_max(info));
+	cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
+	    bin);
+	n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *);
 	*nstashed = n;
-	/* Note that cannot assert ncached + nstashed <= ncached_max (racy). */
+	/*
+	 * Note that cannot assert anything regarding ncached_max because
+	 * it can be configured on the fly and is thus racy.
+	 */
 }
 
+/*
+ * For small bins, used to calculate how many items to fill at a time.
+ * The final nfill is calculated by (ncached_max >> (base - offset)).
+ */
+typedef struct cache_bin_fill_ctl_s cache_bin_fill_ctl_t;
+struct cache_bin_fill_ctl_s {
+	uint8_t base;
+	uint8_t offset;
+};
+
+/*
+ * Limit how many items can be flushed in a batch (Which is the upper bound
+ * for the nflush parameter in tcache_bin_flush_impl()).
+ * This is to avoid stack overflow when we do batch edata look up, which
+ * reserves a nflush * sizeof(emap_batch_lookup_result_t) stack variable.
+ */
+#define CACHE_BIN_NFLUSH_BATCH_MAX                                             \
+	((VARIABLE_ARRAY_SIZE_MAX >> LG_SIZEOF_PTR) - 1)
+
 /*
  * Filling and flushing are done in batch, on arrays of void *s.  For filling,
  * the arrays go forward, and can be accessed with ordinary array arithmetic.
@@ -546,7 +648,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
 typedef struct cache_bin_ptr_array_s cache_bin_ptr_array_t;
 struct cache_bin_ptr_array_s {
 	cache_bin_sz_t n;
-	void **ptr;
+	void         **ptr;
 };
 
 /*
@@ -558,18 +660,18 @@ struct cache_bin_ptr_array_s {
  * representations is easy (since they'll require an alloca in the calling
  * frame).
  */
-#define CACHE_BIN_PTR_ARRAY_DECLARE(name, nval)				\
+#define CACHE_BIN_PTR_ARRAY_DECLARE(name, nval)                                \
-    cache_bin_ptr_array_t name;						\
+	cache_bin_ptr_array_t name;                                            \
-    name.n = (nval)
+	name.n = (nval)
 
 /*
  * Start a fill.  The bin must be empty, and This must be followed by a
  * finish_fill call before doing any alloc/dalloc operations on the bin.
  */
 static inline void
-cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info,
+cache_bin_init_ptr_array_for_fill(
-    cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) {
+    cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) {
-	cache_bin_assert_empty(bin, info);
+	cache_bin_assert_empty(bin);
 	arr->ptr = cache_bin_empty_position_get(bin) - nfill;
 }
 
@@ -579,15 +681,19 @@ cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info,
  * case of OOM.
  */
 static inline void
-cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info,
+cache_bin_finish_fill(
-    cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) {
+    cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) {
-	cache_bin_assert_empty(bin, info);
+	cache_bin_assert_empty(bin);
 	void **empty_position = cache_bin_empty_position_get(bin);
 	if (nfilled < arr->n) {
 		memmove(empty_position - nfilled, empty_position - arr->n,
 		    nfilled * sizeof(void *));
 	}
 	bin->stack_head = empty_position - nfilled;
+	/* Reset the bin stats as it's merged during fill. */
+	if (config_stats) {
+		bin->tstats.nrequests = 0;
+	}
 }
 
 /*
@@ -595,55 +701,61 @@ cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info,
  * everything we give them.
  */
 static inline void
-cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_info_t *info,
+cache_bin_init_ptr_array_for_flush(
-    cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) {
+    cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) {
 	arr->ptr = cache_bin_empty_position_get(bin) - nflush;
-	assert(cache_bin_ncached_get_local(bin, info) == 0
+	assert(cache_bin_ncached_get_local(bin) == 0 || *arr->ptr != NULL);
-	    || *arr->ptr != NULL);
 }
 
 static inline void
-cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info,
+cache_bin_finish_flush(
-    cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) {
+    cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) {
-	unsigned rem = cache_bin_ncached_get_local(bin, info) - nflushed;
+	unsigned rem = cache_bin_ncached_get_local(bin) - nflushed;
-	memmove(bin->stack_head + nflushed, bin->stack_head,
+	memmove(
-	    rem * sizeof(void *));
+	    bin->stack_head + nflushed, bin->stack_head, rem * sizeof(void *));
-	bin->stack_head = bin->stack_head + nflushed;
+	bin->stack_head += nflushed;
 	cache_bin_low_water_adjust(bin);
+	/* Reset the bin stats as it's merged during flush. */
+	if (config_stats) {
+		bin->tstats.nrequests = 0;
+	}
 }
 
 static inline void
 cache_bin_init_ptr_array_for_stashed(cache_bin_t *bin, szind_t binind,
-    cache_bin_info_t *info, cache_bin_ptr_array_t *arr,
+    cache_bin_ptr_array_t *arr, cache_bin_sz_t nstashed) {
-    cache_bin_sz_t nstashed) {
 	assert(nstashed > 0);
-	assert(cache_bin_nstashed_get_local(bin, info) == nstashed);
+	assert(cache_bin_nstashed_get_local(bin) == nstashed);
 
-	void **low_bound = cache_bin_low_bound_get(bin, info);
+	void **low_bound = cache_bin_low_bound_get(bin);
 	arr->ptr = low_bound;
 	assert(*arr->ptr != NULL);
 }
 
 static inline void
-cache_bin_finish_flush_stashed(cache_bin_t *bin, cache_bin_info_t *info) {
+cache_bin_finish_flush_stashed(cache_bin_t *bin) {
-	void **low_bound = cache_bin_low_bound_get(bin, info);
+	void **low_bound = cache_bin_low_bound_get(bin);
 
 	/* Reset the bin local full position. */
 	bin->low_bits_full = (uint16_t)(uintptr_t)low_bound;
-	assert(cache_bin_nstashed_get_local(bin, info) == 0);
+	assert(cache_bin_nstashed_get_local(bin) == 0);
+	/* Reset the bin stats as it's merged during flush. */
+	if (config_stats) {
+		bin->tstats.nrequests = 0;
+	}
 }
 
 /*
  * Initialize a cache_bin_info to represent up to the given number of items in
  * the cache_bins it is associated with.
  */
-void cache_bin_info_init(cache_bin_info_t *bin_info,
+void cache_bin_info_init(
-    cache_bin_sz_t ncached_max);
+    cache_bin_info_t *bin_info, cache_bin_sz_t ncached_max);
 /*
  * Given an array of initialized cache_bin_info_ts, determine how big an
  * allocation is required to initialize a full set of cache_bin_ts.
  */
-void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
+void cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos,
     size_t *size, size_t *alignment);
 
 /*
@@ -653,18 +765,13 @@ void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
  * cache_bin_postincrement.  *alloc_cur will then point immediately past the end
  * of the allocation.
  */
-void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos,
+void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos,
     void *alloc, size_t *cur_offset);
-void cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos,
+void cache_bin_postincrement(void *alloc, size_t *cur_offset);
-    void *alloc, size_t *cur_offset);
+void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
-void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
     size_t *cur_offset);
+void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max);
 
-/*
+bool cache_bin_stack_use_thp(void);
- * If a cache bin was zero initialized (either because it lives in static or
- * thread-local storage, or was memset to 0), this function indicates whether or
- * not cache_bin_init was called on it.
- */
-bool cache_bin_still_zero_initialized(cache_bin_t *bin);
 
 #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */

dep/jemalloc/include/jemalloc/internal/ckh.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_CKH_H
 #define JEMALLOC_INTERNAL_CKH_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/tsd.h"
 
 /* Cuckoo hashing implementation.  Skip to the end for the interface. */
@@ -21,8 +22,8 @@
 #define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
 
 /* Typedefs to allow easy function pointer passing. */
-typedef void ckh_hash_t (const void *, size_t[2]);
+typedef void ckh_hash_t(const void *, size_t[2]);
-typedef bool ckh_keycomp_t (const void *, const void *);
+typedef bool ckh_keycomp_t(const void *, const void *);
 
 /* Hash table cell. */
 typedef struct {
@@ -55,7 +56,7 @@ typedef struct {
 	unsigned lg_curbuckets;
 
 	/* Hash and comparison functions. */
-	ckh_hash_t *hash;
+	ckh_hash_t    *hash;
 	ckh_keycomp_t *keycomp;
 
 	/* Hash table with 2^lg_curbuckets buckets. */
@@ -88,8 +89,8 @@ bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
  * the key and value, and doesn't do any lifetime management.
  */
 bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
-bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
+bool ckh_remove(
-    void **data);
+    tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data);
 bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
 
 /* Some useful hash and comparison functions for strings and pointers. */

dep/jemalloc/include/jemalloc/internal/conf.h

@@ -0,0 +1,23 @@
+#ifndef JEMALLOC_INTERNAL_CONF_H
+#define JEMALLOC_INTERNAL_CONF_H
+
+#include "jemalloc/internal/sc.h"
+
+void malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
+    char readlink_buf[PATH_MAX + 1]);
+void malloc_abort_invalid_conf(void);
+
+#ifdef JEMALLOC_JET
+extern bool had_conf_error;
+
+bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
+    char const **v_p, size_t *vlen_p);
+void conf_error(
+    const char *msg, const char *k, size_t klen, const char *v, size_t vlen);
+bool conf_handle_bool(const char *v, size_t vlen, bool *result);
+bool conf_handle_signed(const char *v, size_t vlen, intmax_t min, intmax_t max,
+    bool check_min, bool check_max, bool clip, intmax_t *result);
+bool conf_handle_char_p(const char *v, size_t vlen, char *dest, size_t dest_sz);
+#endif
+
+#endif /* JEMALLOC_INTERNAL_CONF_H */

dep/jemalloc/include/jemalloc/internal/counter.h

@@ -1,12 +1,14 @@
 #ifndef JEMALLOC_INTERNAL_COUNTER_H
 #define JEMALLOC_INTERNAL_COUNTER_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/lockedint.h"
 #include "jemalloc/internal/mutex.h"
 
 typedef struct counter_accum_s {
 	LOCKEDINT_MTX_DECLARE(mtx)
 	locked_u64_t accumbytes;
-	uint64_t interval;
+	uint64_t     interval;
 } counter_accum_t;
 
 JEMALLOC_ALWAYS_INLINE bool

dep/jemalloc/include/jemalloc/internal/ctl.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_CTL_H
 #define JEMALLOC_INTERNAL_CTL_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_stats.h"
+#include "jemalloc/internal/background_thread_structs.h"
+#include "jemalloc/internal/bin_stats.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex_prof.h"
@@ -9,50 +13,52 @@
 #include "jemalloc/internal/stats.h"
 
 /* Maximum ctl tree depth. */
-#define CTL_MAX_DEPTH	7
+#define CTL_MAX_DEPTH 7
+#define CTL_MULTI_SETTING_MAX_LEN 1000
 
 typedef struct ctl_node_s {
 	bool named;
 } ctl_node_t;
 
 typedef struct ctl_named_node_s {
-	ctl_node_t node;
+	ctl_node_t  node;
 	const char *name;
 	/* If (nchildren == 0), this is a terminal node. */
-	size_t nchildren;
+	size_t            nchildren;
 	const ctl_node_t *children;
-	int (*ctl)(tsd_t *, const size_t *, size_t, void *, size_t *, void *,
+	int (*ctl)(
-	    size_t);
+	    tsd_t *, const size_t *, size_t, void *, size_t *, void *, size_t);
 } ctl_named_node_t;
 
 typedef struct ctl_indexed_node_s {
 	struct ctl_node_s node;
-	const ctl_named_node_t *(*index)(tsdn_t *, const size_t *, size_t,
+	const ctl_named_node_t *(*index)(
-	    size_t);
+	    tsdn_t *, const size_t *, size_t, size_t);
 } ctl_indexed_node_t;
 
 typedef struct ctl_arena_stats_s {
 	arena_stats_t astats;
 
 	/* Aggregate stats for small size classes, based on bin stats. */
-	size_t allocated_small;
+	size_t   allocated_small;
 	uint64_t nmalloc_small;
 	uint64_t ndalloc_small;
 	uint64_t nrequests_small;
 	uint64_t nfills_small;
 	uint64_t nflushes_small;
 
-	bin_stats_data_t bstats[SC_NBINS];
+	bin_stats_data_t    bstats[SC_NBINS];
 	arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
-	pac_estats_t estats[SC_NPSIZES];
+	pac_estats_t        estats[SC_NPSIZES];
-	hpa_shard_stats_t hpastats;
+	hpa_shard_stats_t   hpastats;
-	sec_stats_t secstats;
 } ctl_arena_stats_t;
 
 typedef struct ctl_stats_s {
 	size_t allocated;
 	size_t active;
 	size_t metadata;
+	size_t metadata_edata;
+	size_t metadata_rtree;
 	size_t metadata_thp;
 	size_t resident;
 	size_t mapped;
@@ -65,17 +71,17 @@ typedef struct ctl_stats_s {
 typedef struct ctl_arena_s ctl_arena_t;
 struct ctl_arena_s {
 	unsigned arena_ind;
-	bool initialized;
+	bool     initialized;
 	ql_elm(ctl_arena_t) destroyed_link;
 
 	/* Basic stats, supported even if !config_stats. */
-	unsigned nthreads;
+	unsigned    nthreads;
 	const char *dss;
-	ssize_t dirty_decay_ms;
+	ssize_t     dirty_decay_ms;
-	ssize_t muzzy_decay_ms;
+	ssize_t     muzzy_decay_ms;
-	size_t pactive;
+	size_t      pactive;
-	size_t pdirty;
+	size_t      pdirty;
-	size_t pmuzzy;
+	size_t      pmuzzy;
 
 	/* NULL if !config_stats. */
 	ctl_arena_stats_t *astats;
@@ -100,60 +106,67 @@ int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
 int ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp);
 int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
     size_t *oldlenp, void *newp, size_t newlen);
-int ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
+int ctl_mibnametomib(
-    size_t *miblenp);
+    tsd_t *tsd, size_t *mib, size_t miblen, const char *name, size_t *miblenp);
-int ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
+int  ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
-    size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
+     size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
 bool ctl_boot(void);
 void ctl_prefork(tsdn_t *tsdn);
 void ctl_postfork_parent(tsdn_t *tsdn);
 void ctl_postfork_child(tsdn_t *tsdn);
 void ctl_mtx_assert_held(tsdn_t *tsdn);
 
-#define xmallctl(name, oldp, oldlenp, newp, newlen) do {		\
+#define xmallctl(name, oldp, oldlenp, newp, newlen)                            \
-	if (je_mallctl(name, oldp, oldlenp, newp, newlen)		\
+	do {                                                                   \
-	    != 0) {							\
+		if (je_mallctl(name, oldp, oldlenp, newp, newlen) != 0) {      \
-		malloc_printf(						\
+			malloc_printf(                                         \
-		    "<jemalloc>: Failure in xmallctl(\"%s\", ...)\n",	\
+			    "<jemalloc>: Failure in xmallctl(\"%s\", ...)\n",  \
-		    name);						\
+			    name);                                             \
-		abort();						\
+			abort();                                               \
-	}								\
+		}                                                              \
-} while (0)
+	} while (0)
 
-#define xmallctlnametomib(name, mibp, miblenp) do {			\
+#define xmallctlnametomib(name, mibp, miblenp)                                 \
-	if (je_mallctlnametomib(name, mibp, miblenp) != 0) {		\
+	do {                                                                   \
-		malloc_printf("<jemalloc>: Failure in "			\
+		if (je_mallctlnametomib(name, mibp, miblenp) != 0) {           \
-		    "xmallctlnametomib(\"%s\", ...)\n", name);		\
+			malloc_printf(                                         \
-		abort();						\
+			    "<jemalloc>: Failure in "                          \
-	}								\
+			    "xmallctlnametomib(\"%s\", ...)\n",                \
-} while (0)
+			    name);                                             \
+			abort();                                               \
+		}                                                              \
+	} while (0)
 
-#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do {	\
+#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen)                \
-	if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp,		\
+	do {                                                                   \
-	    newlen) != 0) {						\
+		if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen)  \
-		malloc_write(						\
+		    != 0) {                                                    \
-		    "<jemalloc>: Failure in xmallctlbymib()\n");	\
+			malloc_write(                                          \
-		abort();						\
+			    "<jemalloc>: Failure in xmallctlbymib()\n");       \
-	}								\
+			abort();                                               \
-} while (0)
+		}                                                              \
+	} while (0)
 
-#define xmallctlmibnametomib(mib, miblen, name, miblenp) do {		\
+#define xmallctlmibnametomib(mib, miblen, name, miblenp)                       \
-	if (ctl_mibnametomib(tsd_fetch(), mib, miblen, name, miblenp)	\
+	do {                                                                   \
-	    != 0) {							\
+		if (ctl_mibnametomib(tsd_fetch(), mib, miblen, name, miblenp)  \
-		malloc_write(						\
+		    != 0) {                                                    \
-		    "<jemalloc>: Failure in ctl_mibnametomib()\n");	\
+			malloc_write(                                          \
-		abort();						\
+			    "<jemalloc>: Failure in ctl_mibnametomib()\n");    \
-	}								\
+			abort();                                               \
-} while (0)
+		}                                                              \
+	} while (0)
 
-#define xmallctlbymibname(mib, miblen, name, miblenp, oldp, oldlenp,	\
+#define xmallctlbymibname(                                                     \
-    newp, newlen) do {							\
+    mib, miblen, name, miblenp, oldp, oldlenp, newp, newlen)                   \
-	if (ctl_bymibname(tsd_fetch(), mib, miblen, name, miblenp,	\
+	do {                                                                   \
-	    oldp, oldlenp, newp, newlen) != 0) {			\
+		if (ctl_bymibname(tsd_fetch(), mib, miblen, name, miblenp,     \
-		malloc_write(						\
+		        oldp, oldlenp, newp, newlen)                           \
-		    "<jemalloc>: Failure in ctl_bymibname()\n");	\
+		    != 0) {                                                    \
-		abort();						\
+			malloc_write(                                          \
-	}								\
+			    "<jemalloc>: Failure in ctl_bymibname()\n");       \
-} while (0)
+			abort();                                               \
+		}                                                              \
+	} while (0)
 
 #endif /* JEMALLOC_INTERNAL_CTL_H */

dep/jemalloc/include/jemalloc/internal/decay.h

@@ -1,9 +1,11 @@
 #ifndef JEMALLOC_INTERNAL_DECAY_H
 #define JEMALLOC_INTERNAL_DECAY_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/smoothstep.h"
 
-#define DECAY_UNBOUNDED_TIME_TO_PURGE ((uint64_t)-1)
+#define DECAY_UNBOUNDED_TIME_TO_PURGE ((uint64_t) - 1)
 
 /*
  * The decay_t computes the number of pages we should purge at any given time.
@@ -166,12 +168,12 @@ void decay_reinit(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms);
 /*
  * Compute how many of 'npages_new' pages we would need to purge in 'time'.
  */
-uint64_t decay_npages_purge_in(decay_t *decay, nstime_t *time,
+uint64_t decay_npages_purge_in(
-    size_t npages_new);
+    decay_t *decay, nstime_t *time, size_t npages_new);
 
 /* Returns true if the epoch advanced and there are pages to purge. */
-bool decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time,
+bool decay_maybe_advance_epoch(
-    size_t current_npages);
+    decay_t *decay, nstime_t *new_time, size_t current_npages);
 
 /*
  * Calculates wait time until a number of pages in the interval
@@ -180,7 +182,7 @@ bool decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time,
  * Returns number of nanoseconds or DECAY_UNBOUNDED_TIME_TO_PURGE in case of
  * indefinite wait.
  */
-uint64_t decay_ns_until_purge(decay_t *decay, size_t npages_current,
+uint64_t decay_ns_until_purge(
-    uint64_t npages_threshold);
+    decay_t *decay, size_t npages_current, uint64_t npages_threshold);
 
 #endif /* JEMALLOC_INTERNAL_DECAY_H */

dep/jemalloc/include/jemalloc/internal/div.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_DIV_H
 #define JEMALLOC_INTERNAL_DIV_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/assert.h"
 
 /*

dep/jemalloc/include/jemalloc/internal/ecache.h

@@ -1,15 +1,16 @@
 #ifndef JEMALLOC_INTERNAL_ECACHE_H
 #define JEMALLOC_INTERNAL_ECACHE_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/eset.h"
-#include "jemalloc/internal/san.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/san.h"
 
 typedef struct ecache_s ecache_t;
 struct ecache_s {
 	malloc_mutex_t mtx;
-	eset_t eset;
+	eset_t         eset;
-	eset_t guarded_eset;
+	eset_t         guarded_eset;
 	/* All stored extents must be in the same state. */
 	extent_state_t state;
 	/* The index of the ehooks the ecache is associated with. */
@@ -23,22 +24,22 @@ struct ecache_s {
 
 static inline size_t
 ecache_npages_get(ecache_t *ecache) {
-	return eset_npages_get(&ecache->eset) +
+	return eset_npages_get(&ecache->eset)
-	    eset_npages_get(&ecache->guarded_eset);
+	    + eset_npages_get(&ecache->guarded_eset);
 }
 
 /* Get the number of extents in the given page size index. */
 static inline size_t
 ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
-	return eset_nextents_get(&ecache->eset, ind) +
+	return eset_nextents_get(&ecache->eset, ind)
-	    eset_nextents_get(&ecache->guarded_eset, ind);
+	    + eset_nextents_get(&ecache->guarded_eset, ind);
 }
 
 /* Get the sum total bytes of the extents in the given page size index. */
 static inline size_t
 ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
-	return eset_nbytes_get(&ecache->eset, ind) +
+	return eset_nbytes_get(&ecache->eset, ind)
-	    eset_nbytes_get(&ecache->guarded_eset, ind);
+	    + eset_nbytes_get(&ecache->guarded_eset, ind);
 }
 
 static inline unsigned

dep/jemalloc/include/jemalloc/internal/edata.h

@@ -1,12 +1,14 @@
 #ifndef JEMALLOC_INTERNAL_EDATA_H
 #define JEMALLOC_INTERNAL_EDATA_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bin_info.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/hpdata.h"
 #include "jemalloc/internal/nstime.h"
 #include "jemalloc/internal/ph.h"
+#include "jemalloc/internal/prof_types.h"
 #include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/slab_data.h"
@@ -19,10 +21,18 @@
  */
 #define EDATA_ALIGNMENT 128
 
+/*
+ * Defines how many nodes visited when enumerating the heap to search for
+ * qualified extents.  More nodes visited may result in better choices at
+ * the cost of longer search time.  This size should not exceed 2^16 - 1
+ * because we use uint16_t for accessing the queue needed for enumeration.
+ */
+#define ESET_ENUMERATE_MAX_NUM 32
+
 enum extent_state_e {
-	extent_state_active   = 0,
+	extent_state_active = 0,
-	extent_state_dirty    = 1,
+	extent_state_dirty = 1,
-	extent_state_muzzy    = 2,
+	extent_state_muzzy = 2,
 	extent_state_retained = 3,
 	extent_state_transition = 4, /* States below are intermediate. */
 	extent_state_merging = 5,
@@ -32,7 +42,7 @@ typedef enum extent_state_e extent_state_t;
 
 enum extent_head_state_e {
 	EXTENT_NOT_HEAD,
-	EXTENT_IS_HEAD   /* See comments in ehooks_default_merge_impl(). */
+	EXTENT_IS_HEAD /* See comments in ehooks_default_merge_impl(). */
 };
 typedef enum extent_head_state_e extent_head_state_t;
 
@@ -40,25 +50,22 @@ typedef enum extent_head_state_e extent_head_state_t;
  * Which implementation of the page allocator interface, (PAI, defined in
  * pai.h) owns the given extent?
  */
-enum extent_pai_e {
+enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 };
-	EXTENT_PAI_PAC = 0,
-	EXTENT_PAI_HPA = 1
-};
 typedef enum extent_pai_e extent_pai_t;
 
 struct e_prof_info_s {
 	/* Time when this was allocated. */
-	nstime_t	e_prof_alloc_time;
+	nstime_t e_prof_alloc_time;
 	/* Allocation request size. */
-	size_t		e_prof_alloc_size;
+	size_t e_prof_alloc_size;
 	/* Points to a prof_tctx_t. */
-	atomic_p_t	e_prof_tctx;
+	atomic_p_t e_prof_tctx;
 	/*
 	 * Points to a prof_recent_t for the allocation; NULL
 	 * means the recent allocation record no longer exists.
 	 * Protected by prof_recent_alloc_mtx.
 	 */
-	atomic_p_t	e_prof_recent_alloc;
+	atomic_p_t e_prof_recent_alloc;
 };
 typedef struct e_prof_info_s e_prof_info_t;
 
@@ -75,20 +82,20 @@ typedef struct e_prof_info_s e_prof_info_t;
  */
 typedef struct edata_map_info_s edata_map_info_t;
 struct edata_map_info_s {
-	bool slab;
+	bool    slab;
 	szind_t szind;
 };
 
 typedef struct edata_cmp_summary_s edata_cmp_summary_t;
 struct edata_cmp_summary_s {
-	uint64_t sn;
+	uint64_t  sn;
 	uintptr_t addr;
 };
 
 /* Extent (span of pages).  Use accessor functions for e_* fields. */
 typedef struct edata_s edata_t;
-ph_structs(edata_avail, edata_t);
+ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM);
-ph_structs(edata_heap, edata_t);
+ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM);
 struct edata_s {
 	/*
 	 * Bitfield containing several fields:
@@ -139,55 +146,72 @@ struct edata_s {
 	 *
 	 * bin_shard: the shard of the bin from which this extent came.
 	 */
-	uint64_t		e_bits;
+	uint64_t e_bits;
-#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT))
+#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT)                         \
+	((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1))                   \
+	    << (CURRENT_FIELD_SHIFT))
 
-#define EDATA_BITS_ARENA_WIDTH  MALLOCX_ARENA_BITS
+#define EDATA_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS
-#define EDATA_BITS_ARENA_SHIFT  0
+#define EDATA_BITS_ARENA_SHIFT 0
-#define EDATA_BITS_ARENA_MASK  MASK(EDATA_BITS_ARENA_WIDTH, EDATA_BITS_ARENA_SHIFT)
+#define EDATA_BITS_ARENA_MASK                                                  \
+	MASK(EDATA_BITS_ARENA_WIDTH, EDATA_BITS_ARENA_SHIFT)
 
-#define EDATA_BITS_SLAB_WIDTH  1
+#define EDATA_BITS_SLAB_WIDTH 1
-#define EDATA_BITS_SLAB_SHIFT  (EDATA_BITS_ARENA_WIDTH + EDATA_BITS_ARENA_SHIFT)
+#define EDATA_BITS_SLAB_SHIFT (EDATA_BITS_ARENA_WIDTH + EDATA_BITS_ARENA_SHIFT)
-#define EDATA_BITS_SLAB_MASK  MASK(EDATA_BITS_SLAB_WIDTH, EDATA_BITS_SLAB_SHIFT)
+#define EDATA_BITS_SLAB_MASK MASK(EDATA_BITS_SLAB_WIDTH, EDATA_BITS_SLAB_SHIFT)
 
-#define EDATA_BITS_COMMITTED_WIDTH  1
+#define EDATA_BITS_COMMITTED_WIDTH 1
-#define EDATA_BITS_COMMITTED_SHIFT  (EDATA_BITS_SLAB_WIDTH + EDATA_BITS_SLAB_SHIFT)
+#define EDATA_BITS_COMMITTED_SHIFT                                             \
-#define EDATA_BITS_COMMITTED_MASK  MASK(EDATA_BITS_COMMITTED_WIDTH, EDATA_BITS_COMMITTED_SHIFT)
+	(EDATA_BITS_SLAB_WIDTH + EDATA_BITS_SLAB_SHIFT)
+#define EDATA_BITS_COMMITTED_MASK                                              \
+	MASK(EDATA_BITS_COMMITTED_WIDTH, EDATA_BITS_COMMITTED_SHIFT)
 
-#define EDATA_BITS_PAI_WIDTH  1
+#define EDATA_BITS_PAI_WIDTH 1
-#define EDATA_BITS_PAI_SHIFT  (EDATA_BITS_COMMITTED_WIDTH + EDATA_BITS_COMMITTED_SHIFT)
+#define EDATA_BITS_PAI_SHIFT                                                   \
-#define EDATA_BITS_PAI_MASK  MASK(EDATA_BITS_PAI_WIDTH, EDATA_BITS_PAI_SHIFT)
+	(EDATA_BITS_COMMITTED_WIDTH + EDATA_BITS_COMMITTED_SHIFT)
+#define EDATA_BITS_PAI_MASK MASK(EDATA_BITS_PAI_WIDTH, EDATA_BITS_PAI_SHIFT)
 
-#define EDATA_BITS_ZEROED_WIDTH  1
+#define EDATA_BITS_ZEROED_WIDTH 1
-#define EDATA_BITS_ZEROED_SHIFT  (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT)
+#define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT)
-#define EDATA_BITS_ZEROED_MASK  MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT)
+#define EDATA_BITS_ZEROED_MASK                                                 \
+	MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT)
 
-#define EDATA_BITS_GUARDED_WIDTH  1
+#define EDATA_BITS_GUARDED_WIDTH 1
-#define EDATA_BITS_GUARDED_SHIFT  (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT)
+#define EDATA_BITS_GUARDED_SHIFT                                               \
-#define EDATA_BITS_GUARDED_MASK  MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT)
+	(EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT)
+#define EDATA_BITS_GUARDED_MASK                                                \
+	MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT)
 
-#define EDATA_BITS_STATE_WIDTH  3
+#define EDATA_BITS_STATE_WIDTH 3
-#define EDATA_BITS_STATE_SHIFT  (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT)
+#define EDATA_BITS_STATE_SHIFT                                                 \
-#define EDATA_BITS_STATE_MASK  MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT)
+	(EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT)
+#define EDATA_BITS_STATE_MASK                                                  \
+	MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT)
 
-#define EDATA_BITS_SZIND_WIDTH  LG_CEIL(SC_NSIZES)
+#define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES)
-#define EDATA_BITS_SZIND_SHIFT  (EDATA_BITS_STATE_WIDTH + EDATA_BITS_STATE_SHIFT)
+#define EDATA_BITS_SZIND_SHIFT (EDATA_BITS_STATE_WIDTH + EDATA_BITS_STATE_SHIFT)
-#define EDATA_BITS_SZIND_MASK  MASK(EDATA_BITS_SZIND_WIDTH, EDATA_BITS_SZIND_SHIFT)
+#define EDATA_BITS_SZIND_MASK                                                  \
+	MASK(EDATA_BITS_SZIND_WIDTH, EDATA_BITS_SZIND_SHIFT)
 
-#define EDATA_BITS_NFREE_WIDTH  (SC_LG_SLAB_MAXREGS + 1)
+#define EDATA_BITS_NFREE_WIDTH (SC_LG_SLAB_MAXREGS + 1)
-#define EDATA_BITS_NFREE_SHIFT  (EDATA_BITS_SZIND_WIDTH + EDATA_BITS_SZIND_SHIFT)
+#define EDATA_BITS_NFREE_SHIFT (EDATA_BITS_SZIND_WIDTH + EDATA_BITS_SZIND_SHIFT)
-#define EDATA_BITS_NFREE_MASK  MASK(EDATA_BITS_NFREE_WIDTH, EDATA_BITS_NFREE_SHIFT)
+#define EDATA_BITS_NFREE_MASK                                                  \
+	MASK(EDATA_BITS_NFREE_WIDTH, EDATA_BITS_NFREE_SHIFT)
 
-#define EDATA_BITS_BINSHARD_WIDTH  6
+#define EDATA_BITS_BINSHARD_WIDTH 6
-#define EDATA_BITS_BINSHARD_SHIFT  (EDATA_BITS_NFREE_WIDTH + EDATA_BITS_NFREE_SHIFT)
+#define EDATA_BITS_BINSHARD_SHIFT                                              \
-#define EDATA_BITS_BINSHARD_MASK  MASK(EDATA_BITS_BINSHARD_WIDTH, EDATA_BITS_BINSHARD_SHIFT)
+	(EDATA_BITS_NFREE_WIDTH + EDATA_BITS_NFREE_SHIFT)
+#define EDATA_BITS_BINSHARD_MASK                                               \
+	MASK(EDATA_BITS_BINSHARD_WIDTH, EDATA_BITS_BINSHARD_SHIFT)
 
 #define EDATA_BITS_IS_HEAD_WIDTH 1
-#define EDATA_BITS_IS_HEAD_SHIFT  (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT)
+#define EDATA_BITS_IS_HEAD_SHIFT                                               \
-#define EDATA_BITS_IS_HEAD_MASK  MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
+	(EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT)
+#define EDATA_BITS_IS_HEAD_MASK                                                \
+	MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
 
 	/* Pointer to the extent that this structure is responsible for. */
-	void			*e_addr;
+	void *e_addr;
 
 	union {
 		/*
@@ -197,16 +221,16 @@ struct edata_s {
 		 *
 		 * ssssssss [...] ssssssss ssssnnnn nnnnnnnn
 		 */
-		size_t			e_size_esn;
+		size_t e_size_esn;
-	#define EDATA_SIZE_MASK	((size_t)~(PAGE-1))
+#define EDATA_SIZE_MASK ((size_t) ~(PAGE - 1))
-	#define EDATA_ESN_MASK		((size_t)PAGE-1)
+#define EDATA_ESN_MASK ((size_t)PAGE - 1)
 		/* Base extent size, which may not be a multiple of PAGE. */
-		size_t			e_bsize;
+		size_t e_bsize;
 	};
 
 	/*
 	 * If this edata is a user allocation from an HPA, it comes out of some
-	 * pageslab (we don't yet support huegpage allocations that don't fit
+	 * pageslab (we don't yet support hugepage allocations that don't fit
 	 * into pageslabs).  This tracks it.
 	 */
 	hpdata_t *e_ps;
@@ -222,7 +246,7 @@ struct edata_s {
 		 * List linkage used when the edata_t is active; either in
 		 * arena's large allocations or bin_t's slabs_full.
 		 */
-		ql_elm(edata_t)	ql_link_active;
+		ql_elm(edata_t) ql_link_active;
 		/*
 		 * Pairing heap linkage.  Used whenever the extent is inactive
 		 * (in the page allocators), or when it is active and in
@@ -230,7 +254,7 @@ struct edata_s {
 		 * extent and sitting in an edata_cache.
 		 */
 		union {
-			edata_heap_link_t heap_link;
+			edata_heap_link_t  heap_link;
 			edata_avail_link_t avail_link;
 		};
 	};
@@ -243,10 +267,10 @@ struct edata_s {
 		 */
 		ql_elm(edata_t) ql_link_inactive;
 		/* Small region slab metadata. */
-		slab_data_t	e_slab_data;
+		slab_data_t e_slab_data;
 
 		/* Profiling data, used for large objects. */
-		e_prof_info_t	e_prof_info;
+		e_prof_info_t e_prof_info;
 	};
 };
 
@@ -255,8 +279,8 @@ TYPED_LIST(edata_list_inactive, edata_t, ql_link_inactive)
 
 static inline unsigned
 edata_arena_ind_get(const edata_t *edata) {
-	unsigned arena_ind = (unsigned)((edata->e_bits &
+	unsigned arena_ind = (unsigned)((edata->e_bits & EDATA_BITS_ARENA_MASK)
-	    EDATA_BITS_ARENA_MASK) >> EDATA_BITS_ARENA_SHIFT);
+	    >> EDATA_BITS_ARENA_SHIFT);
 	assert(arena_ind < MALLOCX_ARENA_LIMIT);
 
 	return arena_ind;
@@ -264,8 +288,8 @@ edata_arena_ind_get(const edata_t *edata) {
 
 static inline szind_t
 edata_szind_get_maybe_invalid(const edata_t *edata) {
-	szind_t szind = (szind_t)((edata->e_bits & EDATA_BITS_SZIND_MASK) >>
+	szind_t szind = (szind_t)((edata->e_bits & EDATA_BITS_SZIND_MASK)
-	    EDATA_BITS_SZIND_SHIFT);
+	    >> EDATA_BITS_SZIND_SHIFT);
 	assert(szind <= SC_NSIZES);
 	return szind;
 }
@@ -279,13 +303,61 @@ edata_szind_get(const edata_t *edata) {
 
 static inline size_t
 edata_usize_get(const edata_t *edata) {
-	return sz_index2size(edata_szind_get(edata));
+	assert(edata != NULL);
+	/*
+	 * When sz_large_size_classes_disabled() is true, two cases:
+	 * 1. if usize_from_ind is not smaller than SC_LARGE_MINCLASS,
+	 * usize_from_size is accurate;
+	 * 2. otherwise, usize_from_ind is accurate.
+	 *
+	 * When sz_large_size_classes_disabled() is not true, the two should be the
+	 * same when usize_from_ind is not smaller than SC_LARGE_MINCLASS.
+	 *
+	 * Note sampled small allocs will be promoted.  Their extent size is
+	 * recorded in edata_size_get(edata), while their szind reflects the
+	 * true usize.  Thus, usize retrieved here is still accurate for
+	 * sampled small allocs.
+	 */
+	szind_t szind = edata_szind_get(edata);
+#ifdef JEMALLOC_JET
+	/*
+	 * Double free is invalid and results in undefined behavior.  However,
+	 * for double free tests to end gracefully, return an invalid usize
+	 * when szind shows the edata is not active, i.e., szind == SC_NSIZES.
+	 */
+	if (unlikely(szind == SC_NSIZES)) {
+		return SC_LARGE_MAXCLASS + 1;
+	}
+#endif
+
+	if (!sz_large_size_classes_disabled() || szind < SC_NBINS) {
+		size_t usize_from_ind = sz_index2size(szind);
+		if (!sz_large_size_classes_disabled()
+		    && usize_from_ind >= SC_LARGE_MINCLASS) {
+			size_t size = (edata->e_size_esn & EDATA_SIZE_MASK);
+			assert(size > sz_large_pad);
+			size_t usize_from_size = size - sz_large_pad;
+			assert(usize_from_ind == usize_from_size);
+		}
+		return usize_from_ind;
+	}
+
+	size_t size = (edata->e_size_esn & EDATA_SIZE_MASK);
+	assert(size > sz_large_pad);
+	size_t usize_from_size = size - sz_large_pad;
+	/*
+	 * no matter large size classes disabled or not, usize retrieved from
+	 * size is not accurate when smaller than SC_LARGE_MINCLASS.
+	 */
+	assert(usize_from_size >= SC_LARGE_MINCLASS);
+	return usize_from_size;
 }
 
 static inline unsigned
 edata_binshard_get(const edata_t *edata) {
-	unsigned binshard = (unsigned)((edata->e_bits &
+	unsigned binshard = (unsigned)((edata->e_bits
-	    EDATA_BITS_BINSHARD_MASK) >> EDATA_BITS_BINSHARD_SHIFT);
+	                                   & EDATA_BITS_BINSHARD_MASK)
+	    >> EDATA_BITS_BINSHARD_SHIFT);
 	assert(binshard < bin_infos[edata_szind_get(edata)].n_shards);
 	return binshard;
 }
@@ -297,58 +369,58 @@ edata_sn_get(const edata_t *edata) {
 
 static inline extent_state_t
 edata_state_get(const edata_t *edata) {
-	return (extent_state_t)((edata->e_bits & EDATA_BITS_STATE_MASK) >>
+	return (extent_state_t)((edata->e_bits & EDATA_BITS_STATE_MASK)
-	    EDATA_BITS_STATE_SHIFT);
+	    >> EDATA_BITS_STATE_SHIFT);
 }
 
 static inline bool
 edata_guarded_get(const edata_t *edata) {
-	return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) >>
+	return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK)
-	    EDATA_BITS_GUARDED_SHIFT);
+	    >> EDATA_BITS_GUARDED_SHIFT);
 }
 
 static inline bool
 edata_zeroed_get(const edata_t *edata) {
-	return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >>
+	return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK)
-	    EDATA_BITS_ZEROED_SHIFT);
+	    >> EDATA_BITS_ZEROED_SHIFT);
 }
 
 static inline bool
 edata_committed_get(const edata_t *edata) {
-	return (bool)((edata->e_bits & EDATA_BITS_COMMITTED_MASK) >>
+	return (bool)((edata->e_bits & EDATA_BITS_COMMITTED_MASK)
-	    EDATA_BITS_COMMITTED_SHIFT);
+	    >> EDATA_BITS_COMMITTED_SHIFT);
 }
 
 static inline extent_pai_t
 edata_pai_get(const edata_t *edata) {
-	return (extent_pai_t)((edata->e_bits & EDATA_BITS_PAI_MASK) >>
+	return (extent_pai_t)((edata->e_bits & EDATA_BITS_PAI_MASK)
-	    EDATA_BITS_PAI_SHIFT);
+	    >> EDATA_BITS_PAI_SHIFT);
 }
 
 static inline bool
 edata_slab_get(const edata_t *edata) {
-	return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK) >>
+	return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK)
-	    EDATA_BITS_SLAB_SHIFT);
+	    >> EDATA_BITS_SLAB_SHIFT);
 }
 
 static inline unsigned
 edata_nfree_get(const edata_t *edata) {
 	assert(edata_slab_get(edata));
-	return (unsigned)((edata->e_bits & EDATA_BITS_NFREE_MASK) >>
+	return (unsigned)((edata->e_bits & EDATA_BITS_NFREE_MASK)
-	    EDATA_BITS_NFREE_SHIFT);
+	    >> EDATA_BITS_NFREE_SHIFT);
 }
 
 static inline void *
 edata_base_get(const edata_t *edata) {
-	assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) ||
+	assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr)
-	    !edata_slab_get(edata));
+	    || !edata_slab_get(edata));
 	return PAGE_ADDR2BASE(edata->e_addr);
 }
 
 static inline void *
 edata_addr_get(const edata_t *edata) {
-	assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) ||
+	assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr)
-	    !edata_slab_get(edata));
+	    || !edata_slab_get(edata));
 	return edata->e_addr;
 }
 
@@ -375,19 +447,19 @@ edata_ps_get(const edata_t *edata) {
 
 static inline void *
 edata_before_get(const edata_t *edata) {
-	return (void *)((uintptr_t)edata_base_get(edata) - PAGE);
+	return (void *)((byte_t *)edata_base_get(edata) - PAGE);
 }
 
 static inline void *
 edata_last_get(const edata_t *edata) {
-	return (void *)((uintptr_t)edata_base_get(edata) +
+	return (void *)((byte_t *)edata_base_get(edata) + edata_size_get(edata)
-	    edata_size_get(edata) - PAGE);
+	    - PAGE);
 }
 
 static inline void *
 edata_past_get(const edata_t *edata) {
-	return (void *)((uintptr_t)edata_base_get(edata) +
+	return (
-	    edata_size_get(edata));
+	    void *)((byte_t *)edata_base_get(edata) + edata_size_get(edata));
 }
 
 static inline slab_data_t *
@@ -404,8 +476,8 @@ edata_slab_data_get_const(const edata_t *edata) {
 
 static inline prof_tctx_t *
 edata_prof_tctx_get(const edata_t *edata) {
-	return (prof_tctx_t *)atomic_load_p(&edata->e_prof_info.e_prof_tctx,
+	return (prof_tctx_t *)atomic_load_p(
-	    ATOMIC_ACQUIRE);
+	    &edata->e_prof_info.e_prof_tctx, ATOMIC_ACQUIRE);
 }
 
 static inline const nstime_t *
@@ -426,16 +498,16 @@ edata_prof_recent_alloc_get_dont_call_directly(const edata_t *edata) {
 
 static inline void
 edata_arena_ind_set(edata_t *edata, unsigned arena_ind) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_ARENA_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_ARENA_MASK)
-	    ((uint64_t)arena_ind << EDATA_BITS_ARENA_SHIFT);
+	    | ((uint64_t)arena_ind << EDATA_BITS_ARENA_SHIFT);
 }
 
 static inline void
 edata_binshard_set(edata_t *edata, unsigned binshard) {
 	/* The assertion assumes szind is set already. */
 	assert(binshard < bin_infos[edata_szind_get(edata)].n_shards);
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_BINSHARD_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_BINSHARD_MASK)
-	    ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT);
+	    | ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT);
 }
 
 static inline void
@@ -451,8 +523,8 @@ edata_size_set(edata_t *edata, size_t size) {
 
 static inline void
 edata_esn_set(edata_t *edata, size_t esn) {
-	edata->e_size_esn = (edata->e_size_esn & ~EDATA_ESN_MASK) | (esn &
+	edata->e_size_esn = (edata->e_size_esn & ~EDATA_ESN_MASK)
-	    EDATA_ESN_MASK);
+	    | (esn & EDATA_ESN_MASK);
 }
 
 static inline void
@@ -469,25 +541,26 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) {
 static inline void
 edata_szind_set(edata_t *edata, szind_t szind) {
 	assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_SZIND_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_SZIND_MASK)
-	    ((uint64_t)szind << EDATA_BITS_SZIND_SHIFT);
+	    | ((uint64_t)szind << EDATA_BITS_SZIND_SHIFT);
 }
 
 static inline void
 edata_nfree_set(edata_t *edata, unsigned nfree) {
 	assert(edata_slab_get(edata));
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_NFREE_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_NFREE_MASK)
-	    ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT);
+	    | ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT);
 }
 
 static inline void
 edata_nfree_binshard_set(edata_t *edata, unsigned nfree, unsigned binshard) {
 	/* The assertion assumes szind is set already. */
 	assert(binshard < bin_infos[edata_szind_get(edata)].n_shards);
-	edata->e_bits = (edata->e_bits &
+	edata->e_bits = (edata->e_bits
-	    (~EDATA_BITS_NFREE_MASK & ~EDATA_BITS_BINSHARD_MASK)) |
+	                    & (~EDATA_BITS_NFREE_MASK
-	    ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT) |
+	                        & ~EDATA_BITS_BINSHARD_MASK))
-	    ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT);
+	    | ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT)
+	    | ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT);
 }
 
 static inline void
@@ -515,38 +588,38 @@ edata_sn_set(edata_t *edata, uint64_t sn) {
 
 static inline void
 edata_state_set(edata_t *edata, extent_state_t state) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_STATE_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_STATE_MASK)
-	    ((uint64_t)state << EDATA_BITS_STATE_SHIFT);
+	    | ((uint64_t)state << EDATA_BITS_STATE_SHIFT);
 }
 
 static inline void
 edata_guarded_set(edata_t *edata, bool guarded) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK)
-	    ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT);
+	    | ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT);
 }
 
 static inline void
 edata_zeroed_set(edata_t *edata, bool zeroed) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK)
-	    ((uint64_t)zeroed << EDATA_BITS_ZEROED_SHIFT);
+	    | ((uint64_t)zeroed << EDATA_BITS_ZEROED_SHIFT);
 }
 
 static inline void
 edata_committed_set(edata_t *edata, bool committed) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_COMMITTED_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_COMMITTED_MASK)
-	    ((uint64_t)committed << EDATA_BITS_COMMITTED_SHIFT);
+	    | ((uint64_t)committed << EDATA_BITS_COMMITTED_SHIFT);
 }
 
 static inline void
 edata_pai_set(edata_t *edata, extent_pai_t pai) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_PAI_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_PAI_MASK)
-	    ((uint64_t)pai << EDATA_BITS_PAI_SHIFT);
+	    | ((uint64_t)pai << EDATA_BITS_PAI_SHIFT);
 }
 
 static inline void
 edata_slab_set(edata_t *edata, bool slab) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK)
-	    ((uint64_t)slab << EDATA_BITS_SLAB_SHIFT);
+	    | ((uint64_t)slab << EDATA_BITS_SLAB_SHIFT);
 }
 
 static inline void
@@ -565,22 +638,22 @@ edata_prof_alloc_size_set(edata_t *edata, size_t size) {
 }
 
 static inline void
-edata_prof_recent_alloc_set_dont_call_directly(edata_t *edata,
+edata_prof_recent_alloc_set_dont_call_directly(
-    prof_recent_t *recent_alloc) {
+    edata_t *edata, prof_recent_t *recent_alloc) {
 	atomic_store_p(&edata->e_prof_info.e_prof_recent_alloc, recent_alloc,
 	    ATOMIC_RELAXED);
 }
 
 static inline bool
 edata_is_head_get(edata_t *edata) {
-	return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >>
+	return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK)
-	    EDATA_BITS_IS_HEAD_SHIFT);
+	    >> EDATA_BITS_IS_HEAD_SHIFT);
 }
 
 static inline void
 edata_is_head_set(edata_t *edata, bool is_head) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) |
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK)
-	    ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT);
+	    | ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT);
 }
 
 static inline bool
@@ -619,7 +692,8 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
 }
 
 static inline void
-edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) {
+edata_binit(
+    edata_t *edata, void *addr, size_t bsize, uint64_t sn, bool reused) {
 	edata_arena_ind_set(edata, (1U << MALLOCX_ARENA_BITS) - 1);
 	edata_addr_set(edata, addr);
 	edata_bsize_set(edata, bsize);
@@ -627,7 +701,8 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) {
 	edata_szind_set(edata, SC_NSIZES);
 	edata_sn_set(edata, sn);
 	edata_state_set(edata, extent_state_active);
-	edata_guarded_set(edata, false);
+	/* See comments in base_edata_is_reused. */
+	edata_guarded_set(edata, reused);
 	edata_zeroed_set(edata, true);
 	edata_committed_set(edata, true);
 	/*
@@ -656,20 +731,47 @@ edata_ead_comp(const edata_t *a, const edata_t *b) {
 
 static inline edata_cmp_summary_t
 edata_cmp_summary_get(const edata_t *edata) {
-	return (edata_cmp_summary_t){edata_sn_get(edata),
+	edata_cmp_summary_t result;
-		(uintptr_t)edata_addr_get(edata)};
+	result.sn = edata_sn_get(edata);
+	result.addr = (uintptr_t)edata_addr_get(edata);
+	return result;
+}
+
+#ifdef JEMALLOC_HAVE_INT128
+JEMALLOC_ALWAYS_INLINE unsigned __int128
+edata_cmp_summary_encode(edata_cmp_summary_t src) {
+	return ((unsigned __int128)src.sn << 64) | src.addr;
 }
 
 static inline int
 edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) {
-	int ret;
+	unsigned __int128 a_encoded = edata_cmp_summary_encode(a);
-	ret = (a.sn > b.sn) - (a.sn < b.sn);
+	unsigned __int128 b_encoded = edata_cmp_summary_encode(b);
-	if (ret != 0) {
+	if (a_encoded < b_encoded)
-		return ret;
+		return -1;
-	}
+	if (a_encoded == b_encoded)
-	ret = (a.addr > b.addr) - (a.addr < b.addr);
+		return 0;
-	return ret;
+	return 1;
 }
+#else
+static inline int
+edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) {
+	/*
+	 * Logically, what we're doing here is comparing based on `.sn`, and
+	 * falling back to comparing on `.addr` in the case that `a.sn == b.sn`.
+	 * We accomplish this by multiplying the result of the `.sn` comparison
+	 * by 2, so that so long as it is not 0, it will dominate the `.addr`
+	 * comparison in determining the sign of the returned result value.
+	 * The justification for doing things this way is that this is
+	 * branchless - all of the branches that would be present in a
+	 * straightforward implementation are common cases, and thus the branch
+	 * prediction accuracy is not great. As a result, this implementation
+	 * is measurably faster (by around 30%).
+	 */
+	return (2 * ((a.sn > b.sn) - (a.sn < b.sn)))
+	    + ((a.addr > b.addr) - (a.addr < b.addr));
+}
+#endif
 
 static inline int
 edata_snad_comp(const edata_t *a, const edata_t *b) {
@@ -681,18 +783,13 @@ edata_snad_comp(const edata_t *a, const edata_t *b) {
 
 static inline int
 edata_esnead_comp(const edata_t *a, const edata_t *b) {
-	int ret;
+	/*
-
+	 * Similar to `edata_cmp_summary_comp`, we've opted for a
-	ret = edata_esn_comp(a, b);
+	 * branchless implementation for the sake of performance.
-	if (ret != 0) {
+	 */
-		return ret;
+	return (2 * edata_esn_comp(a, b)) + edata_ead_comp(a, b);
-	}
-
-	ret = edata_ead_comp(a, b);
-	return ret;
 }
 
-ph_proto(, edata_avail, edata_t)
+ph_proto(, edata_avail, edata_t) ph_proto(, edata_heap, edata_t)
-ph_proto(, edata_heap, edata_t)
 
 #endif /* JEMALLOC_INTERNAL_EDATA_H */

dep/jemalloc/include/jemalloc/internal/edata_cache.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_EDATA_CACHE_H
 #define JEMALLOC_INTERNAL_EDATA_CACHE_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 
 /* For tests only. */
@@ -14,13 +15,13 @@
 
 typedef struct edata_cache_s edata_cache_t;
 struct edata_cache_s {
-	edata_avail_t avail;
+	edata_avail_t  avail;
-	atomic_zu_t count;
+	atomic_zu_t    count;
 	malloc_mutex_t mtx;
-	base_t *base;
+	base_t        *base;
 };
 
-bool edata_cache_init(edata_cache_t *edata_cache, base_t *base);
+bool     edata_cache_init(edata_cache_t *edata_cache, base_t *base);
 edata_t *edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache);
 void edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata);
 
@@ -36,14 +37,14 @@ void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache);
 typedef struct edata_cache_fast_s edata_cache_fast_t;
 struct edata_cache_fast_s {
 	edata_list_inactive_t list;
-	edata_cache_t *fallback;
+	edata_cache_t        *fallback;
-	bool disabled;
+	bool                  disabled;
 };
 
 void edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback);
 edata_t *edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs);
-void edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs,
+void     edata_cache_fast_put(
-    edata_t *edata);
+        tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata);
 void edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs);
 
 #endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */

dep/jemalloc/include/jemalloc/internal/ehooks.h

@@ -1,8 +1,11 @@
 #ifndef JEMALLOC_INTERNAL_EHOOKS_H
 #define JEMALLOC_INTERNAL_EHOOKS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/extent_mmap.h"
+#include "jemalloc/internal/tsd.h"
+#include "jemalloc/internal/tsd_types.h"
 
 /*
  * This module is the internal interface to the extent hooks (both
@@ -43,17 +46,17 @@ extern const extent_hooks_t ehooks_default_extent_hooks;
  */
 void *ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size,
     size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
-bool ehooks_default_dalloc_impl(void *addr, size_t size);
+bool  ehooks_default_dalloc_impl(void *addr, size_t size);
-void ehooks_default_destroy_impl(void *addr, size_t size);
+void  ehooks_default_destroy_impl(void *addr, size_t size);
-bool ehooks_default_commit_impl(void *addr, size_t offset, size_t length);
+bool  ehooks_default_commit_impl(void *addr, size_t offset, size_t length);
-bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length);
+bool  ehooks_default_decommit_impl(void *addr, size_t offset, size_t length);
 #ifdef PAGES_CAN_PURGE_LAZY
 bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
 #endif
 #ifdef PAGES_CAN_PURGE_FORCED
 bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
-bool ehooks_default_split_impl();
+bool ehooks_default_split_impl(void);
 /*
  * Merge is the only default extent hook we declare -- see the comment in
  * ehooks_merge.
@@ -113,8 +116,8 @@ ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
 
 static inline bool
 ehooks_are_default(ehooks_t *ehooks) {
-	return ehooks_get_extent_hooks_ptr(ehooks) ==
+	return ehooks_get_extent_hooks_ptr(ehooks)
-	    &ehooks_default_extent_hooks;
+	    == &ehooks_default_extent_hooks;
 }
 
 /*
@@ -186,16 +189,15 @@ ehooks_debug_zero_check(void *addr, size_t size) {
 	}
 }
 
-
 static inline void *
 ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
     size_t alignment, bool *zero, bool *commit) {
-	bool orig_zero = *zero;
+	bool            orig_zero = *zero;
-	void *ret;
+	void           *ret;
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
 	if (extent_hooks == &ehooks_default_extent_hooks) {
-		ret = ehooks_default_alloc_impl(tsdn, new_addr, size,
+		ret = ehooks_default_alloc_impl(tsdn, new_addr, size, alignment,
-		    alignment, zero, commit, ehooks_ind_get(ehooks));
+		    zero, commit, ehooks_ind_get(ehooks));
 	} else {
 		ehooks_pre_reentrancy(tsdn);
 		ret = extent_hooks->alloc(extent_hooks, new_addr, size,
@@ -211,8 +213,8 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
 }
 
 static inline bool
-ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
+ehooks_dalloc(
-    bool committed) {
+    tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, bool committed) {
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
 	if (extent_hooks == &ehooks_default_extent_hooks) {
 		return ehooks_default_dalloc_impl(addr, size);
@@ -228,8 +230,8 @@ ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
 }
 
 static inline void
-ehooks_destroy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
+ehooks_destroy(
-    bool committed) {
+    tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, bool committed) {
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
 	if (extent_hooks == &ehooks_default_extent_hooks) {
 		ehooks_default_destroy_impl(addr, size);
@@ -247,15 +249,15 @@ static inline bool
 ehooks_commit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
     size_t offset, size_t length) {
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
-	bool err;
+	bool            err;
 	if (extent_hooks == &ehooks_default_extent_hooks) {
 		err = ehooks_default_commit_impl(addr, offset, length);
 	} else if (extent_hooks->commit == NULL) {
 		err = true;
 	} else {
 		ehooks_pre_reentrancy(tsdn);
-		err = extent_hooks->commit(extent_hooks, addr, size,
+		err = extent_hooks->commit(extent_hooks, addr, size, offset,
-		    offset, length, ehooks_ind_get(ehooks));
+		    length, ehooks_ind_get(ehooks));
 		ehooks_post_reentrancy(tsdn);
 	}
 	if (!err) {
@@ -381,7 +383,7 @@ ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size) {
 
 static inline bool
 ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
-	bool err;
+	bool            err;
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
 
 	if (extent_hooks == &ehooks_default_extent_hooks) {
@@ -396,7 +398,7 @@ ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
 
 static inline bool
 ehooks_unguard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
-	bool err;
+	bool            err;
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
 
 	if (extent_hooks == &ehooks_default_extent_hooks) {

dep/jemalloc/include/jemalloc/internal/emap.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_EMAP_H
 #define JEMALLOC_INTERNAL_EMAP_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/rtree.h"
 
@@ -9,9 +10,9 @@
  *     EMAP_DECLARE_RTREE_CTX;
  * in uses will avoid empty-statement warnings.
  */
-#define EMAP_DECLARE_RTREE_CTX						\
+#define EMAP_DECLARE_RTREE_CTX                                                 \
-    rtree_ctx_t rtree_ctx_fallback;					\
+	rtree_ctx_t  rtree_ctx_fallback;                                       \
-    rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback)
+	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback)
 
 typedef struct emap_s emap_t;
 struct emap_s {
@@ -19,26 +20,27 @@ struct emap_s {
 };
 
 /* Used to pass rtree lookup context down the path. */
-typedef struct emap_alloc_ctx_t emap_alloc_ctx_t;
+typedef struct emap_alloc_ctx_s emap_alloc_ctx_t;
-struct emap_alloc_ctx_t {
+struct emap_alloc_ctx_s {
+	size_t  usize;
 	szind_t szind;
-	bool slab;
+	bool    slab;
 };
 
 typedef struct emap_full_alloc_ctx_s emap_full_alloc_ctx_t;
 struct emap_full_alloc_ctx_s {
-	szind_t szind;
+	szind_t  szind;
-	bool slab;
+	bool     slab;
 	edata_t *edata;
 };
 
 bool emap_init(emap_t *emap, base_t *base, bool zeroed);
 
-void emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind,
+void emap_remap(
-    bool slab);
+    tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab);
 
-void emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
+void emap_update_edata_state(
-    extent_state_t state);
+    tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t state);
 
 /*
  * The two acquire functions below allow accessing neighbor edatas, if it's safe
@@ -60,16 +62,16 @@ edata_t *emap_try_acquire_edata_neighbor(tsdn_t *tsdn, emap_t *emap,
     bool forward);
 edata_t *emap_try_acquire_edata_neighbor_expand(tsdn_t *tsdn, emap_t *emap,
     edata_t *edata, extent_pai_t pai, extent_state_t expected_state);
-void emap_release_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
+void     emap_release_edata(
-    extent_state_t new_state);
+        tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t new_state);
 
 /*
  * Associate the given edata with its beginning and end address, setting the
  * szind and slab info appropriately.
  * Returns true on error (i.e. resource exhaustion).
  */
-bool emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
+bool emap_register_boundary(
-    szind_t szind, bool slab);
+    tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab);
 
 /*
  * Does the same thing, but with the interior of the range, for slab
@@ -90,8 +92,8 @@ bool emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
  * touched, so no allocation is necessary to fill the interior once the boundary
  * has been touched.
  */
-void emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
+void emap_register_interior(
-    szind_t szind);
+    tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind);
 
 void emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata);
 void emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata);
@@ -159,8 +161,8 @@ emap_edata_in_transition(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
 	emap_assert_mapped(tsdn, emap, edata);
 
 	EMAP_DECLARE_RTREE_CTX;
-	rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx,
+	rtree_contents_t contents = rtree_read(
-	    (uintptr_t)edata_base_get(edata));
+	    tsdn, &emap->rtree, rtree_ctx, (uintptr_t)edata_base_get(edata));
 
 	return edata_state_in_transition(contents.metadata.state);
 }
@@ -185,16 +187,16 @@ emap_edata_is_acquired(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
 	 */
 	EMAP_DECLARE_RTREE_CTX;
 	rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
-	    rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ true,
+	    rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ false,
 	    /* init_missing */ false);
 	if (elm == NULL) {
 		return true;
 	}
 	rtree_contents_t contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm,
-	    /* dependent */ true);
+	    /* dependent */ false);
-	if (contents.edata == NULL ||
+	if (contents.edata == NULL
-	    contents.metadata.state == extent_state_active ||
+	    || contents.metadata.state == extent_state_active
-	    edata_state_in_transition(contents.metadata.state)) {
+	    || edata_state_in_transition(contents.metadata.state)) {
 		return true;
 	}
 
@@ -209,8 +211,8 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) {
 	assert(edata_state_get(inner) == extent_state_active);
 	assert(edata_state_get(outer) == extent_state_merging);
 	assert(!edata_guarded_get(inner) && !edata_guarded_get(outer));
-	assert(edata_base_get(inner) == edata_past_get(outer) ||
+	assert(edata_base_get(inner) == edata_past_get(outer)
-	    edata_base_get(outer) == edata_past_get(inner));
+	    || edata_base_get(outer) == edata_past_get(inner));
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -229,16 +231,46 @@ emap_edata_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr) {
 	return rtree_read(tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr).edata;
 }
 
+JEMALLOC_ALWAYS_INLINE void
+emap_alloc_ctx_init(
+    emap_alloc_ctx_t *alloc_ctx, szind_t szind, bool slab, size_t usize) {
+	alloc_ctx->szind = szind;
+	alloc_ctx->slab = slab;
+	alloc_ctx->usize = usize;
+	assert(
+	    sz_large_size_classes_disabled() || usize == sz_index2size(szind));
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+emap_alloc_ctx_usize_get(emap_alloc_ctx_t *alloc_ctx) {
+	assert(alloc_ctx->szind < SC_NSIZES);
+	if (alloc_ctx->slab) {
+		assert(alloc_ctx->usize == sz_index2size(alloc_ctx->szind));
+		return sz_index2size(alloc_ctx->szind);
+	}
+	assert(sz_large_size_classes_disabled()
+	    || alloc_ctx->usize == sz_index2size(alloc_ctx->szind));
+	assert(alloc_ctx->usize <= SC_LARGE_MAXCLASS);
+	return alloc_ctx->usize;
+}
+
 /* Fills in alloc_ctx with the info in the map. */
 JEMALLOC_ALWAYS_INLINE void
-emap_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
+emap_alloc_ctx_lookup(
-    emap_alloc_ctx_t *alloc_ctx) {
+    tsdn_t *tsdn, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
 	EMAP_DECLARE_RTREE_CTX;
 
-	rtree_metadata_t metadata = rtree_metadata_read(tsdn, &emap->rtree,
+	rtree_contents_t contents = rtree_read(
-	    rtree_ctx, (uintptr_t)ptr);
+	    tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr);
-	alloc_ctx->szind = metadata.szind;
+	/*
-	alloc_ctx->slab = metadata.slab;
+	 * If the alloc is invalid, do not calculate usize since edata
+	 * could be corrupted.
+	 */
+	emap_alloc_ctx_init(alloc_ctx, contents.metadata.szind,
+	    contents.metadata.slab,
+	    (contents.metadata.szind == SC_NSIZES || contents.edata == NULL)
+	        ? 0
+	        : edata_usize_get(contents.edata));
 }
 
 /* The pointer must be mapped. */
@@ -247,8 +279,8 @@ emap_full_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
     emap_full_alloc_ctx_t *full_alloc_ctx) {
 	EMAP_DECLARE_RTREE_CTX;
 
-	rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx,
+	rtree_contents_t contents = rtree_read(
-	    (uintptr_t)ptr);
+	    tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr);
 	full_alloc_ctx->edata = contents.edata;
 	full_alloc_ctx->szind = contents.metadata.szind;
 	full_alloc_ctx->slab = contents.metadata.slab;
@@ -265,8 +297,8 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
 	EMAP_DECLARE_RTREE_CTX;
 
 	rtree_contents_t contents;
-	bool err = rtree_read_independent(tsdn, &emap->rtree, rtree_ctx,
+	bool             err = rtree_read_independent(
-	    (uintptr_t)ptr, &contents);
+            tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr, &contents);
 	if (err) {
 		return true;
 	}
@@ -281,19 +313,26 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
  * fast path, e.g. when the metadata key is not cached.
  */
 JEMALLOC_ALWAYS_INLINE bool
-emap_alloc_ctx_try_lookup_fast(tsd_t *tsd, emap_t *emap, const void *ptr,
+emap_alloc_ctx_try_lookup_fast(
-    emap_alloc_ctx_t *alloc_ctx) {
+    tsd_t *tsd, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
 	/* Use the unsafe getter since this may gets called during exit. */
 	rtree_ctx_t *rtree_ctx = tsd_rtree_ctxp_get_unsafe(tsd);
 
 	rtree_metadata_t metadata;
-	bool err = rtree_metadata_try_read_fast(tsd_tsdn(tsd), &emap->rtree,
+	bool             err = rtree_metadata_try_read_fast(
-	    rtree_ctx, (uintptr_t)ptr, &metadata);
+            tsd_tsdn(tsd), &emap->rtree, rtree_ctx, (uintptr_t)ptr, &metadata);
 	if (err) {
 		return true;
 	}
+	/*
+	 * Small allocs using the fastpath can always use index to get the
+	 * usize.  Therefore, do not set alloc_ctx->usize here.
+	 */
 	alloc_ctx->szind = metadata.szind;
 	alloc_ctx->slab = metadata.slab;
+	if (config_debug) {
+		alloc_ctx->usize = SC_LARGE_MAXCLASS + 1;
+	}
 	return false;
 }
 
@@ -308,11 +347,12 @@ typedef const void *(*emap_ptr_getter)(void *ctx, size_t ind);
  * This allows size-checking assertions, which we can only do while we're in the
  * process of edata lookups.
  */
-typedef void (*emap_metadata_visitor)(void *ctx, emap_full_alloc_ctx_t *alloc_ctx);
+typedef void (*emap_metadata_visitor)(
+    void *ctx, emap_full_alloc_ctx_t *alloc_ctx);
 
 typedef union emap_batch_lookup_result_u emap_batch_lookup_result_t;
 union emap_batch_lookup_result_u {
-	edata_t *edata;
+	edata_t          *edata;
 	rtree_leaf_elm_t *rtree_leaf;
 };
 
@@ -338,8 +378,8 @@ emap_edata_lookup_batch(tsd_t *tsd, emap_t *emap, size_t nptrs,
 
 	for (size_t i = 0; i < nptrs; i++) {
 		rtree_leaf_elm_t *elm = result[i].rtree_leaf;
-		rtree_contents_t contents = rtree_leaf_elm_read(tsd_tsdn(tsd),
+		rtree_contents_t  contents = rtree_leaf_elm_read(
-		    &emap->rtree, elm, /* dependent */ true);
+                    tsd_tsdn(tsd), &emap->rtree, elm, /* dependent */ true);
 		result[i].edata = contents.edata;
 		emap_full_alloc_ctx_t alloc_ctx;
 		/*

dep/jemalloc/include/jemalloc/internal/emitter.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_EMITTER_H
 #define JEMALLOC_INTERNAL_EMITTER_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/ql.h"
 
 typedef enum emitter_output_e emitter_output_t;
@@ -40,18 +44,18 @@ typedef struct emitter_col_s emitter_col_t;
 struct emitter_col_s {
 	/* Filled in by the user. */
 	emitter_justify_t justify;
-	int width;
+	int               width;
-	emitter_type_t type;
+	emitter_type_t    type;
 	union {
-		bool bool_val;
+		bool        bool_val;
-		int int_val;
+		int         int_val;
-		unsigned unsigned_val;
+		unsigned    unsigned_val;
-		uint32_t uint32_val;
+		uint32_t    uint32_val;
-		uint32_t uint32_t_val;
+		uint32_t    uint32_t_val;
-		uint64_t uint64_val;
+		uint64_t    uint64_val;
-		uint64_t uint64_t_val;
+		uint64_t    uint64_t_val;
-		size_t size_val;
+		size_t      size_val;
-		ssize_t ssize_val;
+		ssize_t     ssize_val;
 		const char *str_val;
 	};
 
@@ -69,8 +73,8 @@ struct emitter_s {
 	emitter_output_t output;
 	/* The output information. */
 	write_cb_t *write_cb;
-	void *cbopaque;
+	void       *cbopaque;
-	int nesting_depth;
+	int         nesting_depth;
 	/* True if we've already emitted a value at the given depth. */
 	bool item_at_depth;
 	/* True if we emitted a key and will emit corresponding value next. */
@@ -79,8 +83,8 @@ struct emitter_s {
 
 static inline bool
 emitter_outputs_json(emitter_t *emitter) {
-	return emitter->output == emitter_output_json ||
+	return emitter->output == emitter_output_json
-	    emitter->output == emitter_output_json_compact;
+	    || emitter->output == emitter_output_json_compact;
 }
 
 /* Internal convenience function.  Write to the emitter the given string. */
@@ -94,26 +98,57 @@ emitter_printf(emitter_t *emitter, const char *format, ...) {
 	va_end(ap);
 }
 
-static inline const char * JEMALLOC_FORMAT_ARG(3)
+static inline const char *
-emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
+JEMALLOC_FORMAT_ARG(3) emitter_gen_fmt(char *out_fmt, size_t out_size,
-    emitter_justify_t justify, int width) {
+    const char *fmt_specifier, emitter_justify_t justify, int width) {
 	size_t written;
 	fmt_specifier++;
 	if (justify == emitter_justify_none) {
-		written = malloc_snprintf(out_fmt, out_size,
+		written = malloc_snprintf(
-		    "%%%s", fmt_specifier);
+		    out_fmt, out_size, "%%%s", fmt_specifier);
 	} else if (justify == emitter_justify_left) {
-		written = malloc_snprintf(out_fmt, out_size,
+		written = malloc_snprintf(
-		    "%%-%d%s", width, fmt_specifier);
+		    out_fmt, out_size, "%%-%d%s", width, fmt_specifier);
 	} else {
-		written = malloc_snprintf(out_fmt, out_size,
+		written = malloc_snprintf(
-		    "%%%d%s", width, fmt_specifier);
+		    out_fmt, out_size, "%%%d%s", width, fmt_specifier);
 	}
 	/* Only happens in case of bad format string, which *we* choose. */
-	assert(written <  out_size);
+	assert(written < out_size);
 	return out_fmt;
 }
 
+static inline void
+emitter_emit_str(emitter_t *emitter, emitter_justify_t justify, int width,
+    char *fmt, size_t fmt_size, const char *str) {
+#define BUF_SIZE 256
+	char   buf[BUF_SIZE];
+	size_t str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", str);
+	emitter_printf(
+	    emitter, emitter_gen_fmt(fmt, fmt_size, "%s", justify, width), buf);
+	if (str_written < BUF_SIZE) {
+		return;
+	}
+	/*
+	 * There is no support for long string justification at the moment as
+	 * we output them partially with multiple malloc_snprintf calls and
+	 * justufication will work correctly only withing one call.
+	 * Fortunately this is not a big concern as we don't use justufication
+	 * with long strings right now.
+	 *
+	 * We emitted leading quotation mark and trailing '\0', hence need to
+	 * exclude extra characters from str shift.
+	 */
+	str += BUF_SIZE - 2;
+	do {
+		str_written = malloc_snprintf(buf, BUF_SIZE, "%s\"", str);
+		str += str_written >= BUF_SIZE ? BUF_SIZE - 1 : str_written;
+		emitter_printf(emitter,
+		    emitter_gen_fmt(fmt, fmt_size, "%s", justify, width), buf);
+	} while (str_written >= BUF_SIZE);
+#undef BUF_SIZE
+}
+
 /*
  * Internal.  Emit the given value type in the relevant encoding (so that the
  * bool true gets mapped to json "true", but the string "true" gets mapped to
@@ -124,8 +159,6 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
 static inline void
 emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
     emitter_type_t value_type, const void *value) {
-	size_t str_written;
-#define BUF_SIZE 256
 #define FMT_SIZE 10
 	/*
 	 * We dynamically generate a format string to emit, to let us use the
@@ -134,18 +167,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
 	 * cases.
 	 */
 	char fmt[FMT_SIZE];
-	char buf[BUF_SIZE];
 
-#define EMIT_SIMPLE(type, format)					\
+#define EMIT_SIMPLE(type, format)                                              \
-	emitter_printf(emitter,						\
+	emitter_printf(emitter,                                                \
-	    emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width),	\
+	    emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width),            \
 	    *(const type *)value);
 
 	switch (value_type) {
 	case emitter_type_bool:
 		emitter_printf(emitter,
 		    emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width),
-		    *(const bool *)value ?  "true" : "false");
+		    *(const bool *)value ? "true" : "false");
 		break;
 	case emitter_type_int:
 		EMIT_SIMPLE(int, "%d")
@@ -163,15 +195,8 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
 		EMIT_SIMPLE(size_t, "%zu")
 		break;
 	case emitter_type_string:
-		str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"",
+		emitter_emit_str(emitter, justify, width, fmt, FMT_SIZE,
 		    *(const char *const *)value);
-		/*
-		 * We control the strings we output; we shouldn't get anything
-		 * anywhere near the fmt size.
-		 */
-		assert(str_written < BUF_SIZE);
-		emitter_printf(emitter,
-		    emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf);
 		break;
 	case emitter_type_uint32:
 		EMIT_SIMPLE(uint32_t, "%" FMTu32)
@@ -185,11 +210,9 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
 	default:
 		unreachable();
 	}
-#undef BUF_SIZE
 #undef FMT_SIZE
 }
 
-
 /* Internal functions.  In json mode, tracks nesting state. */
 static inline void
 emitter_nest_inc(emitter_t *emitter) {
@@ -205,7 +228,7 @@ emitter_nest_dec(emitter_t *emitter) {
 
 static inline void
 emitter_indent(emitter_t *emitter) {
-	int amount = emitter->nesting_depth;
+	int         amount = emitter->nesting_depth;
 	const char *indent_str;
 	assert(emitter->output != emitter_output_json_compact);
 	if (emitter->output == emitter_output_json) {
@@ -267,12 +290,12 @@ emitter_json_key(emitter_t *emitter, const char *json_key) {
 }
 
 static inline void
-emitter_json_value(emitter_t *emitter, emitter_type_t value_type,
+emitter_json_value(
-    const void *value) {
+    emitter_t *emitter, emitter_type_t value_type, const void *value) {
 	if (emitter_outputs_json(emitter)) {
 		emitter_json_key_prefix(emitter);
-		emitter_print_value(emitter, emitter_justify_none, -1,
+		emitter_print_value(
-		    value_type, value);
+		    emitter, emitter_justify_none, -1, value_type, value);
 		emitter->item_at_depth = true;
 	}
 }
@@ -343,7 +366,6 @@ emitter_json_object_end(emitter_t *emitter) {
 	}
 }
 
-
 /******************************************************************************/
 /* Table public API. */
 
@@ -365,14 +387,13 @@ emitter_table_dict_end(emitter_t *emitter) {
 
 static inline void
 emitter_table_kv_note(emitter_t *emitter, const char *table_key,
-    emitter_type_t value_type, const void *value,
+    emitter_type_t value_type, const void *value, const char *table_note_key,
-    const char *table_note_key, emitter_type_t table_note_value_type,
+    emitter_type_t table_note_value_type, const void *table_note_value) {
-    const void *table_note_value) {
 	if (emitter->output == emitter_output_table) {
 		emitter_indent(emitter);
 		emitter_printf(emitter, "%s: ", table_key);
-		emitter_print_value(emitter, emitter_justify_none, -1,
+		emitter_print_value(
-		    value_type, value);
+		    emitter, emitter_justify_none, -1, value_type, value);
 		if (table_note_key != NULL) {
 			emitter_printf(emitter, " (%s: ", table_note_key);
 			emitter_print_value(emitter, emitter_justify_none, -1,
@@ -391,7 +412,6 @@ emitter_table_kv(emitter_t *emitter, const char *table_key,
 	    emitter_type_bool, NULL);
 }
 
-
 /* Write to the emitter the given string, but only in table mode. */
 JEMALLOC_FORMAT_PRINTF(2, 3)
 static inline void
@@ -399,7 +419,8 @@ emitter_table_printf(emitter_t *emitter, const char *format, ...) {
 	if (emitter->output == emitter_output_table) {
 		va_list ap;
 		va_start(ap, format);
-		malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
+		malloc_vcprintf(
+		    emitter->write_cb, emitter->cbopaque, format, ap);
 		va_end(ap);
 	}
 }
@@ -410,7 +431,7 @@ emitter_table_row(emitter_t *emitter, emitter_row_t *row) {
 		return;
 	}
 	emitter_col_t *col;
-	ql_foreach(col, &row->cols, link) {
+	ql_foreach (col, &row->cols, link) {
 		emitter_print_value(emitter, col->justify, col->width,
 		    col->type, (const void *)&col->bool_val);
 	}
@@ -428,7 +449,6 @@ emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
 	ql_tail_insert(&row->cols, col, link);
 }
 
-
 /******************************************************************************/
 /*
  * Generalized public API. Emits using either JSON or table, according to
@@ -440,9 +460,8 @@ emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
  */
 static inline void
 emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
-    emitter_type_t value_type, const void *value,
+    emitter_type_t value_type, const void *value, const char *table_note_key,
-    const char *table_note_key, emitter_type_t table_note_value_type,
+    emitter_type_t table_note_value_type, const void *table_note_value) {
-    const void *table_note_value) {
 	if (emitter_outputs_json(emitter)) {
 		emitter_json_key(emitter, json_key);
 		emitter_json_value(emitter, value_type, value);
@@ -461,8 +480,8 @@ emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
 }
 
 static inline void
-emitter_dict_begin(emitter_t *emitter, const char *json_key,
+emitter_dict_begin(
-    const char *table_header) {
+    emitter_t *emitter, const char *json_key, const char *table_header) {
 	if (emitter_outputs_json(emitter)) {
 		emitter_json_key(emitter, json_key);
 		emitter_json_object_begin(emitter);
@@ -502,8 +521,9 @@ emitter_end(emitter_t *emitter) {
 	if (emitter_outputs_json(emitter)) {
 		assert(emitter->nesting_depth == 1);
 		emitter_nest_dec(emitter);
-		emitter_printf(emitter, "%s", emitter->output ==
+		emitter_printf(emitter, "%s",
-		    emitter_output_json_compact ? "}" : "\n}\n");
+		    emitter->output == emitter_output_json_compact ? "}"
+		                                                   : "\n}\n");
 	}
 }
 

dep/jemalloc/include/jemalloc/internal/eset.h

@@ -1,9 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_ESET_H
 #define JEMALLOC_INTERNAL_ESET_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/fb.h"
 #include "jemalloc/internal/edata.h"
+#include "jemalloc/internal/fb.h"
 #include "jemalloc/internal/mutex.h"
 
 /*

dep/jemalloc/include/jemalloc/internal/exp_grow.h

@@ -1,6 +1,8 @@
 #ifndef JEMALLOC_INTERNAL_EXP_GROW_H
 #define JEMALLOC_INTERNAL_EXP_GROW_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/sz.h"
 typedef struct exp_grow_s exp_grow_t;
 struct exp_grow_s {
 	/*
@@ -25,8 +27,7 @@ exp_grow_size_prepare(exp_grow_t *exp_grow, size_t alloc_size_min,
 	*r_alloc_size = sz_pind2sz(exp_grow->next + *r_skip);
 	while (*r_alloc_size < alloc_size_min) {
 		(*r_skip)++;
-		if (exp_grow->next + *r_skip  >=
+		if (exp_grow->next + *r_skip >= sz_psz2ind(SC_LARGE_MAXCLASS)) {
-		    sz_psz2ind(SC_LARGE_MAXCLASS)) {
 			/* Outside legal range. */
 			return true;
 		}
@@ -42,7 +43,6 @@ exp_grow_size_commit(exp_grow_t *exp_grow, pszind_t skip) {
 	} else {
 		exp_grow->next = exp_grow->limit;
 	}
-
 }
 
 void exp_grow_init(exp_grow_t *exp_grow);

dep/jemalloc/include/jemalloc/internal/extent.h

@@ -1,8 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_EXTENT_H
 #define JEMALLOC_INTERNAL_EXTENT_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/ecache.h"
 #include "jemalloc/internal/ehooks.h"
+#include "jemalloc/internal/pac.h"
 #include "jemalloc/internal/ph.h"
 #include "jemalloc/internal/rtree.h"
 
@@ -19,50 +21,58 @@
 #define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6
 extern size_t opt_lg_extent_max_active_fit;
 
+#define PROCESS_MADVISE_MAX_BATCH_DEFAULT 0
+extern size_t opt_process_madvise_max_batch;
+
+#ifdef JEMALLOC_HAVE_PROCESS_MADVISE
+/* The iovec is on stack.  Limit the max batch to avoid stack overflow. */
+#	define PROCESS_MADVISE_MAX_BATCH_LIMIT                                \
+		(VARIABLE_ARRAY_SIZE_MAX / sizeof(struct iovec))
+#else
+#	define PROCESS_MADVISE_MAX_BATCH_LIMIT 0
+#endif
+
 edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
     bool zero, bool guarded);
 edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
     bool zero, bool guarded);
-void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
-    ecache_t *ecache, edata_t *edata);
+    edata_t *edata);
 edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, size_t npages_min);
 
 void extent_gdump_add(tsdn_t *tsdn, const edata_t *edata);
 void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
     edata_t *edata);
-void extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+void extent_dalloc_gap(
-    edata_t *edata);
+    tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
 edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     void *new_addr, size_t size, size_t alignment, bool zero, bool *commit,
     bool growing_retained);
-void extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+void     extent_dalloc_wrapper(
-    edata_t *edata);
+        tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
-void extent_destroy_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+void extent_dalloc_wrapper_purged(
-    edata_t *edata);
+    tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
-bool extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
+void extent_destroy_wrapper(
-    size_t offset, size_t length);
+    tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
-bool extent_decommit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
-    size_t offset, size_t length);
 bool extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
     size_t offset, size_t length);
 bool extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
     size_t offset, size_t length);
-edata_t *extent_split_wrapper(tsdn_t *tsdn, pac_t *pac,
+edata_t *extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
-    ehooks_t *ehooks, edata_t *edata, size_t size_a, size_t size_b,
+    edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks);
-    bool holding_core_locks);
+bool     extent_merge_wrapper(
-bool extent_merge_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+        tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, edata_t *b);
-    edata_t *a, edata_t *b);
+bool   extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
-bool extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
+      bool commit, bool zero, bool growing_retained);
-    bool commit, bool zero, bool growing_retained);
 size_t extent_sn_next(pac_t *pac);
-bool extent_boot(void);
+bool   extent_boot(void);
 
 JEMALLOC_ALWAYS_INLINE bool
-extent_neighbor_head_state_mergeable(bool edata_is_head,
+extent_neighbor_head_state_mergeable(
-    bool neighbor_is_head, bool forward) {
+    bool edata_is_head, bool neighbor_is_head, bool forward) {
 	/*
 	 * Head states checking: disallow merging if the higher addr extent is a
 	 * head extent.  This helps preserve first-fit, and more importantly
@@ -90,8 +100,8 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
 	}
 	/* It's not safe to access *neighbor yet; must verify states first. */
 	bool neighbor_is_head = contents.metadata.is_head;
-	if (!extent_neighbor_head_state_mergeable(edata_is_head_get(edata),
+	if (!extent_neighbor_head_state_mergeable(
-	    neighbor_is_head, forward)) {
+	        edata_is_head_get(edata), neighbor_is_head, forward)) {
 		return false;
 	}
 	extent_state_t neighbor_state = contents.metadata.state;
@@ -100,8 +110,9 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
 			return false;
 		}
 		/* From this point, it's safe to access *neighbor. */
-		if (!expanding && (edata_committed_get(edata) !=
+		if (!expanding
-		    edata_committed_get(neighbor))) {
+		    && (edata_committed_get(edata)
+		        != edata_committed_get(neighbor))) {
 			/*
 			 * Some platforms (e.g. Windows) require an explicit
 			 * commit step (and writing to uncommitted memory is not
@@ -121,11 +132,11 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
 		return false;
 	}
 	if (opt_retain) {
-		assert(edata_arena_ind_get(edata) ==
+		assert(edata_arena_ind_get(edata)
-		    edata_arena_ind_get(neighbor));
+		    == edata_arena_ind_get(neighbor));
 	} else {
-		if (edata_arena_ind_get(edata) !=
+		if (edata_arena_ind_get(edata)
-		    edata_arena_ind_get(neighbor)) {
+		    != edata_arena_ind_get(neighbor)) {
 			return false;
 		}
 	}

dep/jemalloc/include/jemalloc/internal/extent_dss.h

@@ -1,26 +1,30 @@
 #ifndef JEMALLOC_INTERNAL_EXTENT_DSS_H
 #define JEMALLOC_INTERNAL_EXTENT_DSS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/tsd_types.h"
+
 typedef enum {
-	dss_prec_disabled  = 0,
+	dss_prec_disabled = 0,
-	dss_prec_primary   = 1,
+	dss_prec_primary = 1,
 	dss_prec_secondary = 2,
 
-	dss_prec_limit     = 3
+	dss_prec_limit = 3
 } dss_prec_t;
 #define DSS_PREC_DEFAULT dss_prec_secondary
 #define DSS_DEFAULT "secondary"
 
-extern const char *dss_prec_names[];
+extern const char *const dss_prec_names[];
 
 extern const char *opt_dss;
 
 dss_prec_t extent_dss_prec_get(void);
-bool extent_dss_prec_set(dss_prec_t dss_prec);
+bool       extent_dss_prec_set(dss_prec_t dss_prec);
-void *extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
+void      *extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
-    size_t size, size_t alignment, bool *zero, bool *commit);
+         size_t size, size_t alignment, bool *zero, bool *commit);
-bool extent_in_dss(void *addr);
+bool       extent_in_dss(void *addr);
-bool extent_dss_mergeable(void *addr_a, void *addr_b);
+bool       extent_dss_mergeable(void *addr_a, void *addr_b);
-void extent_dss_boot(void);
+void       extent_dss_boot(void);
 
 #endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */

dep/jemalloc/include/jemalloc/internal/extent_mmap.h

@@ -1,10 +1,12 @@
 #ifndef JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H
 #define JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+
 extern bool opt_retain;
 
-void *extent_alloc_mmap(void *new_addr, size_t size, size_t alignment,
+void *extent_alloc_mmap(
-    bool *zero, bool *commit);
+    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit);
 bool extent_dalloc_mmap(void *addr, size_t size);
 
 #endif /* JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H */

dep/jemalloc/include/jemalloc/internal/fb.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_FB_H
 #define JEMALLOC_INTERNAL_FB_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/bit_util.h"
+
 /*
  * The flat bitmap module.  This has a larger API relative to the bitmap module
  * (supporting things like backwards searches, and searching for both set and
@@ -11,8 +15,8 @@
 
 typedef unsigned long fb_group_t;
 #define FB_GROUP_BITS (ZU(1) << (LG_SIZEOF_LONG + 3))
-#define FB_NGROUPS(nbits) ((nbits) / FB_GROUP_BITS \
+#define FB_NGROUPS(nbits)                                                      \
-    + ((nbits) % FB_GROUP_BITS == 0 ? 0 : 1))
+	((nbits) / FB_GROUP_BITS + ((nbits) % FB_GROUP_BITS == 0 ? 0 : 1))
 
 static inline void
 fb_init(fb_group_t *fb, size_t nbits) {
@@ -71,7 +75,6 @@ fb_unset(fb_group_t *fb, size_t nbits, size_t bit) {
 	fb[group_ind] &= ~((fb_group_t)1 << bit_ind);
 }
 
-
 /*
  * Some implementation details.  This visitation function lets us apply a group
  * visitor to each group in the bitmap (potentially modifying it).  The mask
@@ -90,7 +93,8 @@ fb_visit_impl(fb_group_t *fb, size_t nbits, fb_group_visitor_t visit, void *ctx,
 	 * to from bit 0.
 	 */
 	size_t first_group_cnt = (start_bit_ind + cnt > FB_GROUP_BITS
-		? FB_GROUP_BITS - start_bit_ind : cnt);
+	        ? FB_GROUP_BITS - start_bit_ind
+	        : cnt);
 	/*
 	 * We can basically split affected words into:
 	 *   - The first group, where we touch only the high bits
@@ -100,8 +104,8 @@ fb_visit_impl(fb_group_t *fb, size_t nbits, fb_group_visitor_t visit, void *ctx,
 	 * this can lead to bad codegen for those middle words.
 	 */
 	/* First group */
-	fb_group_t mask = ((~(fb_group_t)0)
+	fb_group_t mask =
-	    >> (FB_GROUP_BITS - first_group_cnt))
+	    ((~(fb_group_t)0) >> (FB_GROUP_BITS - first_group_cnt))
 	    << start_bit_ind;
 	visit(ctx, &fb[group_ind], mask);
 
@@ -172,12 +176,12 @@ fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
  * Returns the number of bits in the bitmap if no such bit exists.
  */
 JEMALLOC_ALWAYS_INLINE ssize_t
-fb_find_impl(fb_group_t *fb, size_t nbits, size_t start, bool val,
+fb_find_impl(
-    bool forward) {
+    fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) {
 	assert(start < nbits);
-	size_t ngroups = FB_NGROUPS(nbits);
+	size_t  ngroups = FB_NGROUPS(nbits);
 	ssize_t group_ind = start / FB_GROUP_BITS;
-	size_t bit_ind = start % FB_GROUP_BITS;
+	size_t  bit_ind = start % FB_GROUP_BITS;
 
 	fb_group_t maybe_invert = (val ? 0 : (fb_group_t)-1);
 
@@ -261,8 +265,8 @@ fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
 		return false;
 	}
 	/* Half open range; the set bits are [begin, end). */
-	ssize_t next_range_end = fb_find_impl(fb, nbits, next_range_begin, !val,
+	ssize_t next_range_end = fb_find_impl(
-	    forward);
+	    fb, nbits, next_range_begin, !val, forward);
 	if (forward) {
 		*r_begin = next_range_begin;
 		*r_len = next_range_end - next_range_begin;
@@ -320,8 +324,9 @@ fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
 	size_t begin = 0;
 	size_t longest_len = 0;
 	size_t len = 0;
-	while (begin < nbits && fb_iter_range_impl(fb, nbits, begin, &begin,
+	while (begin < nbits
-	    &len, val, /* forward */ true)) {
+	    && fb_iter_range_impl(
+	        fb, nbits, begin, &begin, &len, val, /* forward */ true)) {
 		if (len > longest_len) {
 			longest_len = len;
 		}

dep/jemalloc/include/jemalloc/internal/fxp.h

@@ -1,6 +1,9 @@
 #ifndef JEMALLOC_INTERNAL_FXP_H
 #define JEMALLOC_INTERNAL_FXP_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/assert.h"
+
 /*
  * A simple fixed-point math implementation, supporting only unsigned values
  * (with overflow being an error).
@@ -86,7 +89,7 @@ fxp_round_down(fxp_t a) {
 
 static inline uint32_t
 fxp_round_nearest(fxp_t a) {
-	uint32_t fractional_part = (a  & ((1U << 16) - 1));
+	uint32_t fractional_part = (a & ((1U << 16) - 1));
 	uint32_t increment = (uint32_t)(fractional_part >= (1U << 15));
 	return (a >> 16) + increment;
 }

dep/jemalloc/include/jemalloc/internal/hash.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_HASH_H
 #define JEMALLOC_INTERNAL_HASH_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/assert.h"
 
 /*
@@ -24,7 +25,7 @@ hash_rotl_64(uint64_t x, int8_t r) {
 static inline uint32_t
 hash_get_block_32(const uint32_t *p, int i) {
 	/* Handle unaligned read. */
-	if (unlikely((uintptr_t)p & (sizeof(uint32_t)-1)) != 0) {
+	if (unlikely((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0) {
 		uint32_t ret;
 
 		memcpy(&ret, (uint8_t *)(p + i), sizeof(uint32_t));
@@ -37,7 +38,7 @@ hash_get_block_32(const uint32_t *p, int i) {
 static inline uint64_t
 hash_get_block_64(const uint64_t *p, int i) {
 	/* Handle unaligned read. */
-	if (unlikely((uintptr_t)p & (sizeof(uint64_t)-1)) != 0) {
+	if (unlikely((uintptr_t)p & (sizeof(uint64_t) - 1)) != 0) {
 		uint64_t ret;
 
 		memcpy(&ret, (uint8_t *)(p + i), sizeof(uint64_t));
@@ -71,8 +72,8 @@ hash_fmix_64(uint64_t k) {
 
 static inline uint32_t
 hash_x86_32(const void *key, int len, uint32_t seed) {
-	const uint8_t *data = (const uint8_t *) key;
+	const uint8_t *data = (const uint8_t *)key;
-	const int nblocks = len / 4;
+	const int      nblocks = len / 4;
 
 	uint32_t h1 = seed;
 
@@ -81,8 +82,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
 
 	/* body */
 	{
-		const uint32_t *blocks = (const uint32_t *) (data + nblocks*4);
+		const uint32_t *blocks = (const uint32_t *)(data + nblocks * 4);
-		int i;
+		int             i;
 
 		for (i = -nblocks; i; i++) {
 			uint32_t k1 = hash_get_block_32(blocks, i);
@@ -93,21 +94,29 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
 
 			h1 ^= k1;
 			h1 = hash_rotl_32(h1, 13);
-			h1 = h1*5 + 0xe6546b64;
+			h1 = h1 * 5 + 0xe6546b64;
 		}
 	}
 
 	/* tail */
 	{
-		const uint8_t *tail = (const uint8_t *) (data + nblocks*4);
+		const uint8_t *tail = (const uint8_t *)(data + nblocks * 4);
 
 		uint32_t k1 = 0;
 
 		switch (len & 3) {
-		case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH;
+		case 3:
-		case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH;
+			k1 ^= tail[2] << 16;
-		case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15);
+			JEMALLOC_FALLTHROUGH;
-			k1 *= c2; h1 ^= k1;
+		case 2:
+			k1 ^= tail[1] << 8;
+			JEMALLOC_FALLTHROUGH;
+		case 1:
+			k1 ^= tail[0];
+			k1 *= c1;
+			k1 = hash_rotl_32(k1, 15);
+			k1 *= c2;
+			h1 ^= k1;
 		}
 	}
 
@@ -120,10 +129,9 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
 }
 
 static inline void
-hash_x86_128(const void *key, const int len, uint32_t seed,
+hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) {
-    uint64_t r_out[2]) {
+	const uint8_t *data = (const uint8_t *)key;
-	const uint8_t * data = (const uint8_t *) key;
+	const int      nblocks = len / 16;
-	const int nblocks = len / 16;
 
 	uint32_t h1 = seed;
 	uint32_t h2 = seed;
@@ -137,95 +145,161 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
 
 	/* body */
 	{
-		const uint32_t *blocks = (const uint32_t *) (data + nblocks*16);
+		const uint32_t *blocks = (const uint32_t *)(data
-		int i;
+		    + nblocks * 16);
+		int             i;
 
 		for (i = -nblocks; i; i++) {
-			uint32_t k1 = hash_get_block_32(blocks, i*4 + 0);
+			uint32_t k1 = hash_get_block_32(blocks, i * 4 + 0);
-			uint32_t k2 = hash_get_block_32(blocks, i*4 + 1);
+			uint32_t k2 = hash_get_block_32(blocks, i * 4 + 1);
-			uint32_t k3 = hash_get_block_32(blocks, i*4 + 2);
+			uint32_t k3 = hash_get_block_32(blocks, i * 4 + 2);
-			uint32_t k4 = hash_get_block_32(blocks, i*4 + 3);
+			uint32_t k4 = hash_get_block_32(blocks, i * 4 + 3);
 
-			k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+			k1 *= c1;
+			k1 = hash_rotl_32(k1, 15);
+			k1 *= c2;
+			h1 ^= k1;
 
-			h1 = hash_rotl_32(h1, 19); h1 += h2;
+			h1 = hash_rotl_32(h1, 19);
-			h1 = h1*5 + 0x561ccd1b;
+			h1 += h2;
+			h1 = h1 * 5 + 0x561ccd1b;
 
-			k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+			k2 *= c2;
+			k2 = hash_rotl_32(k2, 16);
+			k2 *= c3;
+			h2 ^= k2;
 
-			h2 = hash_rotl_32(h2, 17); h2 += h3;
+			h2 = hash_rotl_32(h2, 17);
-			h2 = h2*5 + 0x0bcaa747;
+			h2 += h3;
+			h2 = h2 * 5 + 0x0bcaa747;
 
-			k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
+			k3 *= c3;
+			k3 = hash_rotl_32(k3, 17);
+			k3 *= c4;
+			h3 ^= k3;
 
-			h3 = hash_rotl_32(h3, 15); h3 += h4;
+			h3 = hash_rotl_32(h3, 15);
-			h3 = h3*5 + 0x96cd1c35;
+			h3 += h4;
+			h3 = h3 * 5 + 0x96cd1c35;
 
-			k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+			k4 *= c4;
+			k4 = hash_rotl_32(k4, 18);
+			k4 *= c1;
+			h4 ^= k4;
 
-			h4 = hash_rotl_32(h4, 13); h4 += h1;
+			h4 = hash_rotl_32(h4, 13);
-			h4 = h4*5 + 0x32ac3b17;
+			h4 += h1;
+			h4 = h4 * 5 + 0x32ac3b17;
 		}
 	}
 
 	/* tail */
 	{
-		const uint8_t *tail = (const uint8_t *) (data + nblocks*16);
+		const uint8_t *tail = (const uint8_t *)(data + nblocks * 16);
-		uint32_t k1 = 0;
+		uint32_t       k1 = 0;
-		uint32_t k2 = 0;
+		uint32_t       k2 = 0;
-		uint32_t k3 = 0;
+		uint32_t       k3 = 0;
-		uint32_t k4 = 0;
+		uint32_t       k4 = 0;
 
 		switch (len & 15) {
-		case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH;
+		case 15:
-		case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH;
+			k4 ^= tail[14] << 16;
-		case 13: k4 ^= tail[12] << 0;
-			k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
 			JEMALLOC_FALLTHROUGH;
-		case 12: k3 ^= (uint32_t) tail[11] << 24; JEMALLOC_FALLTHROUGH;
+		case 14:
-		case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH;
+			k4 ^= tail[13] << 8;
-		case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH;
-		case  9: k3 ^= tail[ 8] << 0;
-			k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
 			JEMALLOC_FALLTHROUGH;
-		case  8: k2 ^= (uint32_t) tail[ 7] << 24; JEMALLOC_FALLTHROUGH;
+		case 13:
-		case  7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH;
+			k4 ^= tail[12] << 0;
-		case  6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH;
+			k4 *= c4;
-		case  5: k2 ^= tail[ 4] << 0;
+			k4 = hash_rotl_32(k4, 18);
-			k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+			k4 *= c1;
+			h4 ^= k4;
 			JEMALLOC_FALLTHROUGH;
-		case  4: k1 ^= (uint32_t) tail[ 3] << 24; JEMALLOC_FALLTHROUGH;
+		case 12:
-		case  3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH;
+			k3 ^= (uint32_t)tail[11] << 24;
-		case  2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH;
+			JEMALLOC_FALLTHROUGH;
-		case  1: k1 ^= tail[ 0] << 0;
+		case 11:
-			k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+			k3 ^= tail[10] << 16;
+			JEMALLOC_FALLTHROUGH;
+		case 10:
+			k3 ^= tail[9] << 8;
+			JEMALLOC_FALLTHROUGH;
+		case 9:
+			k3 ^= tail[8] << 0;
+			k3 *= c3;
+			k3 = hash_rotl_32(k3, 17);
+			k3 *= c4;
+			h3 ^= k3;
+			JEMALLOC_FALLTHROUGH;
+		case 8:
+			k2 ^= (uint32_t)tail[7] << 24;
+			JEMALLOC_FALLTHROUGH;
+		case 7:
+			k2 ^= tail[6] << 16;
+			JEMALLOC_FALLTHROUGH;
+		case 6:
+			k2 ^= tail[5] << 8;
+			JEMALLOC_FALLTHROUGH;
+		case 5:
+			k2 ^= tail[4] << 0;
+			k2 *= c2;
+			k2 = hash_rotl_32(k2, 16);
+			k2 *= c3;
+			h2 ^= k2;
+			JEMALLOC_FALLTHROUGH;
+		case 4:
+			k1 ^= (uint32_t)tail[3] << 24;
+			JEMALLOC_FALLTHROUGH;
+		case 3:
+			k1 ^= tail[2] << 16;
+			JEMALLOC_FALLTHROUGH;
+		case 2:
+			k1 ^= tail[1] << 8;
+			JEMALLOC_FALLTHROUGH;
+		case 1:
+			k1 ^= tail[0] << 0;
+			k1 *= c1;
+			k1 = hash_rotl_32(k1, 15);
+			k1 *= c2;
+			h1 ^= k1;
 			break;
 		}
 	}
 
 	/* finalization */
-	h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+	h1 ^= len;
+	h2 ^= len;
+	h3 ^= len;
+	h4 ^= len;
 
-	h1 += h2; h1 += h3; h1 += h4;
+	h1 += h2;
-	h2 += h1; h3 += h1; h4 += h1;
+	h1 += h3;
+	h1 += h4;
+	h2 += h1;
+	h3 += h1;
+	h4 += h1;
 
 	h1 = hash_fmix_32(h1);
 	h2 = hash_fmix_32(h2);
 	h3 = hash_fmix_32(h3);
 	h4 = hash_fmix_32(h4);
 
-	h1 += h2; h1 += h3; h1 += h4;
+	h1 += h2;
-	h2 += h1; h3 += h1; h4 += h1;
+	h1 += h3;
+	h1 += h4;
+	h2 += h1;
+	h3 += h1;
+	h4 += h1;
 
-	r_out[0] = (((uint64_t) h2) << 32) | h1;
+	r_out[0] = (((uint64_t)h2) << 32) | h1;
-	r_out[1] = (((uint64_t) h4) << 32) | h3;
+	r_out[1] = (((uint64_t)h4) << 32) | h3;
 }
 
 static inline void
-hash_x64_128(const void *key, const int len, const uint32_t seed,
+hash_x64_128(
-    uint64_t r_out[2]) {
+    const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) {
-	const uint8_t *data = (const uint8_t *) key;
+	const uint8_t *data = (const uint8_t *)key;
-	const int nblocks = len / 16;
+	const int      nblocks = len / 16;
 
 	uint64_t h1 = seed;
 	uint64_t h2 = seed;
@@ -235,56 +309,99 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
 
 	/* body */
 	{
-		const uint64_t *blocks = (const uint64_t *) (data);
+		const uint64_t *blocks = (const uint64_t *)(data);
-		int i;
+		int             i;
 
 		for (i = 0; i < nblocks; i++) {
-			uint64_t k1 = hash_get_block_64(blocks, i*2 + 0);
+			uint64_t k1 = hash_get_block_64(blocks, i * 2 + 0);
-			uint64_t k2 = hash_get_block_64(blocks, i*2 + 1);
+			uint64_t k2 = hash_get_block_64(blocks, i * 2 + 1);
 
-			k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+			k1 *= c1;
+			k1 = hash_rotl_64(k1, 31);
+			k1 *= c2;
+			h1 ^= k1;
 
-			h1 = hash_rotl_64(h1, 27); h1 += h2;
+			h1 = hash_rotl_64(h1, 27);
-			h1 = h1*5 + 0x52dce729;
+			h1 += h2;
+			h1 = h1 * 5 + 0x52dce729;
 
-			k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+			k2 *= c2;
+			k2 = hash_rotl_64(k2, 33);
+			k2 *= c1;
+			h2 ^= k2;
 
-			h2 = hash_rotl_64(h2, 31); h2 += h1;
+			h2 = hash_rotl_64(h2, 31);
-			h2 = h2*5 + 0x38495ab5;
+			h2 += h1;
+			h2 = h2 * 5 + 0x38495ab5;
 		}
 	}
 
 	/* tail */
 	{
-		const uint8_t *tail = (const uint8_t*)(data + nblocks*16);
+		const uint8_t *tail = (const uint8_t *)(data + nblocks * 16);
-		uint64_t k1 = 0;
+		uint64_t       k1 = 0;
-		uint64_t k2 = 0;
+		uint64_t       k2 = 0;
 
 		switch (len & 15) {
-		case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH;
+		case 15:
-		case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH;
+			k2 ^= ((uint64_t)(tail[14])) << 48;
-		case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH;
-		case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH;
-		case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH;
-		case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8;  JEMALLOC_FALLTHROUGH;
-		case  9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
-			k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
 			JEMALLOC_FALLTHROUGH;
-		case  8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH;
+		case 14:
-		case  7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH;
+			k2 ^= ((uint64_t)(tail[13])) << 40;
-		case  6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH;
+			JEMALLOC_FALLTHROUGH;
-		case  5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH;
+		case 13:
-		case  4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH;
+			k2 ^= ((uint64_t)(tail[12])) << 32;
-		case  3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH;
+			JEMALLOC_FALLTHROUGH;
-		case  2: k1 ^= ((uint64_t)(tail[ 1])) << 8;  JEMALLOC_FALLTHROUGH;
+		case 12:
-		case  1: k1 ^= ((uint64_t)(tail[ 0])) << 0;
+			k2 ^= ((uint64_t)(tail[11])) << 24;
-			k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+			JEMALLOC_FALLTHROUGH;
+		case 11:
+			k2 ^= ((uint64_t)(tail[10])) << 16;
+			JEMALLOC_FALLTHROUGH;
+		case 10:
+			k2 ^= ((uint64_t)(tail[9])) << 8;
+			JEMALLOC_FALLTHROUGH;
+		case 9:
+			k2 ^= ((uint64_t)(tail[8])) << 0;
+			k2 *= c2;
+			k2 = hash_rotl_64(k2, 33);
+			k2 *= c1;
+			h2 ^= k2;
+			JEMALLOC_FALLTHROUGH;
+		case 8:
+			k1 ^= ((uint64_t)(tail[7])) << 56;
+			JEMALLOC_FALLTHROUGH;
+		case 7:
+			k1 ^= ((uint64_t)(tail[6])) << 48;
+			JEMALLOC_FALLTHROUGH;
+		case 6:
+			k1 ^= ((uint64_t)(tail[5])) << 40;
+			JEMALLOC_FALLTHROUGH;
+		case 5:
+			k1 ^= ((uint64_t)(tail[4])) << 32;
+			JEMALLOC_FALLTHROUGH;
+		case 4:
+			k1 ^= ((uint64_t)(tail[3])) << 24;
+			JEMALLOC_FALLTHROUGH;
+		case 3:
+			k1 ^= ((uint64_t)(tail[2])) << 16;
+			JEMALLOC_FALLTHROUGH;
+		case 2:
+			k1 ^= ((uint64_t)(tail[1])) << 8;
+			JEMALLOC_FALLTHROUGH;
+		case 1:
+			k1 ^= ((uint64_t)(tail[0])) << 0;
+			k1 *= c1;
+			k1 = hash_rotl_64(k1, 31);
+			k1 *= c2;
+			h1 ^= k1;
 			break;
 		}
 	}
 
 	/* finalization */
-	h1 ^= len; h2 ^= len;
+	h1 ^= len;
+	h2 ^= len;
 
 	h1 += h2;
 	h2 += h1;

dep/jemalloc/include/jemalloc/internal/hook.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_HOOK_H
 #define JEMALLOC_INTERNAL_HOOK_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/tsd.h"
 
 /*
@@ -55,6 +56,7 @@ enum hook_alloc_e {
 	hook_alloc_calloc,
 	hook_alloc_memalign,
 	hook_alloc_valloc,
+	hook_alloc_pvalloc,
 	hook_alloc_mallocx,
 
 	/* The reallocating functions have both alloc and dalloc variants */
@@ -81,7 +83,6 @@ enum hook_dalloc_e {
 };
 typedef enum hook_dalloc_e hook_dalloc_t;
 
-
 enum hook_expand_e {
 	hook_expand_realloc,
 	hook_expand_rallocx,
@@ -89,23 +90,22 @@ enum hook_expand_e {
 };
 typedef enum hook_expand_e hook_expand_t;
 
-typedef void (*hook_alloc)(
+typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result,
-    void *extra, hook_alloc_t type, void *result, uintptr_t result_raw,
+    uintptr_t result_raw, uintptr_t args_raw[3]);
-    uintptr_t args_raw[3]);
 
 typedef void (*hook_dalloc)(
     void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
 
-typedef void (*hook_expand)(
+typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address,
-    void *extra, hook_expand_t type, void *address, size_t old_usize,
+    size_t old_usize, size_t new_usize, uintptr_t result_raw,
-    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
+    uintptr_t args_raw[4]);
 
 typedef struct hooks_s hooks_t;
 struct hooks_s {
-	hook_alloc alloc_hook;
+	hook_alloc  alloc_hook;
 	hook_dalloc dalloc_hook;
 	hook_expand expand_hook;
-	void *extra;
+	void       *extra;
 };
 
 /*
@@ -143,9 +143,9 @@ struct hook_ralloc_args_s {
  * Returns an opaque handle to be used when removing the hook.  NULL means that
  * we couldn't install the hook.
  */
-bool hook_boot();
+bool hook_boot(void);
 
-void *hook_install(tsdn_t *tsdn, hooks_t *hooks);
+void *hook_install(tsdn_t *tsdn, hooks_t *to_install);
 /* Uninstalls the hook with the handle previously returned from hook_install. */
 void hook_remove(tsdn_t *tsdn, void *opaque);
 
@@ -154,8 +154,8 @@ void hook_remove(tsdn_t *tsdn, void *opaque);
 void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
     uintptr_t args_raw[3]);
 
-void hook_invoke_dalloc(hook_dalloc_t type, void *address,
+void hook_invoke_dalloc(
-    uintptr_t args_raw[3]);
+    hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
 
 void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
     size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);

dep/jemalloc/include/jemalloc/internal/hpa.h

@@ -1,42 +1,18 @@
 #ifndef JEMALLOC_INTERNAL_HPA_H
 #define JEMALLOC_INTERNAL_HPA_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
+#include "jemalloc/internal/edata_cache.h"
+#include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/exp_grow.h"
+#include "jemalloc/internal/hpa_central.h"
 #include "jemalloc/internal/hpa_hooks.h"
 #include "jemalloc/internal/hpa_opts.h"
+#include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/psset.h"
-
+#include "jemalloc/internal/sec.h"
-typedef struct hpa_central_s hpa_central_t;
-struct hpa_central_s {
-	/*
-	 * The mutex guarding most of the operations on the central data
-	 * structure.
-	 */
-	malloc_mutex_t mtx;
-	/*
-	 * Guards expansion of eden.  We separate this from the regular mutex so
-	 * that cheaper operations can still continue while we're doing the OS
-	 * call.
-	 */
-	malloc_mutex_t grow_mtx;
-	/*
-	 * Either NULL (if empty), or some integer multiple of a
-	 * hugepage-aligned number of hugepages.  We carve them off one at a
-	 * time to satisfy new pageslab requests.
-	 *
-	 * Guarded by grow_mtx.
-	 */
-	void *eden;
-	size_t eden_len;
-	/* Source for metadata. */
-	base_t *base;
-	/* Number of grow operations done on this hpa_central_t. */
-	uint64_t age_counter;
-
-	/* The HPA hooks. */
-	hpa_hooks_t hooks;
-};
 
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
@@ -61,6 +37,14 @@ struct hpa_shard_nonderived_stats_s {
 	 * Guarded by mtx.
 	 */
 	uint64_t nhugifies;
+
+	/*
+	 * The number of times we've tried to hugify a pageslab, but failed.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t nhugify_failures;
+
 	/*
 	 * The number of times we've dehugified a pageslab.
 	 *
@@ -72,8 +56,9 @@ struct hpa_shard_nonderived_stats_s {
 /* Completely derived; only used by CTL. */
 typedef struct hpa_shard_stats_s hpa_shard_stats_t;
 struct hpa_shard_stats_s {
-	psset_stats_t psset_stats;
+	psset_stats_t                psset_stats;
 	hpa_shard_nonderived_stats_t nonderived_stats;
+	sec_stats_t                  secstats;
 };
 
 typedef struct hpa_shard_s hpa_shard_t;
@@ -86,14 +71,17 @@ struct hpa_shard_s {
 
 	/* The central allocator we get our hugepages from. */
 	hpa_central_t *central;
+
 	/* Protects most of this shard's state. */
 	malloc_mutex_t mtx;
+
 	/*
 	 * Guards the shard's access to the central allocator (preventing
 	 * multiple threads operating on this shard from accessing the central
 	 * allocator).
 	 */
 	malloc_mutex_t grow_mtx;
+
 	/* The base metadata allocator. */
 	base_t *base;
 
@@ -104,6 +92,9 @@ struct hpa_shard_s {
 	 */
 	edata_cache_fast_t ecf;
 
+	/* Small extent cache (not guarded by mtx) */
+	JEMALLOC_ALIGNED(CACHELINE) sec_t sec;
+
 	psset_t psset;
 
 	/*
@@ -141,22 +132,31 @@ struct hpa_shard_s {
 	 * Last time we performed purge on this shard.
 	 */
 	nstime_t last_purge;
+
+	/*
+	 * Last time when we attempted work (purging or hugifying). If deferral
+	 * of the work is allowed (we have background thread), this is the time
+	 * when background thread checked if purging or hugifying needs to be
+	 * done. If deferral is not allowed, this is the time of (hpa_alloc or
+	 * hpa_dalloc) activity in the shard.
+	 */
+	nstime_t last_time_work_attempted;
 };
 
+bool hpa_hugepage_size_exceeds_limit(void);
 /*
- * Whether or not the HPA can be used given the current configuration.  This is
+ * Whether or not the HPA can be used given the current configuration.  This
  * is not necessarily a guarantee that it backs its allocations by hugepages,
  * just that it can function properly given the system it's running on.
  */
-bool hpa_supported();
+bool hpa_supported(void);
-bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
+bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
-bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
+    emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
-    base_t *base, edata_cache_t *edata_cache, unsigned ind,
+    const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
-    const hpa_shard_opts_t *opts);
 
 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
-void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
+void hpa_shard_stats_merge(
-    hpa_shard_stats_t *dst);
+    tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst);
 
 /*
  * Notify the shard that we won't use it for allocations much longer.  Due to
@@ -165,15 +165,18 @@ void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
  */
 void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard);
+/* Flush caches that shard may be using */
+void hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard);
 
-void hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard,
+void hpa_shard_set_deferral_allowed(
-    bool deferral_allowed);
+    tsdn_t *tsdn, hpa_shard_t *shard, bool deferral_allowed);
 void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
 
 /*
  * We share the fork ordering with the PA and arena prefork handling; that's why
- * these are 3 and 4 rather than 0 and 1.
+ * these are 2, 3 and 4 rather than 0 and 1.
  */
+void hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard);

dep/jemalloc/include/jemalloc/internal/hpa_central.h

@@ -0,0 +1,41 @@
+#ifndef JEMALLOC_INTERNAL_HPA_CENTRAL_H
+#define JEMALLOC_INTERNAL_HPA_CENTRAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
+#include "jemalloc/internal/hpa_hooks.h"
+#include "jemalloc/internal/hpdata.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/tsd_types.h"
+
+typedef struct hpa_central_s hpa_central_t;
+struct hpa_central_s {
+	/*
+	 * Guards expansion of eden.  We separate this from the regular mutex so
+	 * that cheaper operations can still continue while we're doing the OS
+	 * call.
+	 */
+	malloc_mutex_t grow_mtx;
+	/*
+	 * Either NULL (if empty), or some integer multiple of a
+	 * hugepage-aligned number of hugepages.  We carve them off one at a
+	 * time to satisfy new pageslab requests.
+	 *
+	 * Guarded by grow_mtx.
+	 */
+	void  *eden;
+	size_t eden_len;
+	/* Source for metadata. */
+	base_t *base;
+
+	/* The HPA hooks. */
+	hpa_hooks_t hooks;
+};
+
+bool hpa_central_init(
+    hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
+
+hpdata_t *hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
+    uint64_t age, bool hugify_eager, bool *oom);
+
+#endif /* JEMALLOC_INTERNAL_HPA_CENTRAL_H */

dep/jemalloc/include/jemalloc/internal/hpa_hooks.h

@@ -1,17 +1,21 @@
 #ifndef JEMALLOC_INTERNAL_HPA_HOOKS_H
 #define JEMALLOC_INTERNAL_HPA_HOOKS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/nstime.h"
+
 typedef struct hpa_hooks_s hpa_hooks_t;
 struct hpa_hooks_s {
 	void *(*map)(size_t size);
 	void (*unmap)(void *ptr, size_t size);
 	void (*purge)(void *ptr, size_t size);
-	void (*hugify)(void *ptr, size_t size);
+	bool (*hugify)(void *ptr, size_t size, bool sync);
 	void (*dehugify)(void *ptr, size_t size);
 	void (*curtime)(nstime_t *r_time, bool first_reading);
 	uint64_t (*ms_since)(nstime_t *r_time);
+	bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes);
 };
 
-extern hpa_hooks_t hpa_hooks_default;
+extern const hpa_hooks_t hpa_hooks_default;
 
 #endif /* JEMALLOC_INTERNAL_HPA_HOOKS_H */

dep/jemalloc/include/jemalloc/internal/hpa_opts.h

@@ -1,13 +1,66 @@
 #ifndef JEMALLOC_INTERNAL_HPA_OPTS_H
 #define JEMALLOC_INTERNAL_HPA_OPTS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/fxp.h"
 
 /*
  * This file is morally part of hpa.h, but is split out for header-ordering
  * reasons.
+ *
+ * All of these hpa_shard_opts below are experimental. We are exploring more
+ * efficient packing, hugifying, and purging approaches to make efficient
+ * trade-offs between CPU, memory, latency, and usability. This means all of
+ * them are at the risk of being deprecated and corresponding configurations
+ * should be updated once the final version settles.
  */
 
+/*
+ * This enum controls how jemalloc hugifies/dehugifies pages.  Each style may be
+ * more suitable depending on deployment environments.
+ *
+ * hpa_hugify_style_none
+ * Using this means that jemalloc will not be hugifying or dehugifying pages,
+ * but will let the kernel make those decisions.  This style only makes sense
+ * when deploying on systems where THP are enabled in 'always' mode.  With this
+ * style, you most likely want to have no purging at all (dirty_mult=-1) or
+ * purge_threshold=HUGEPAGE bytes (2097152 for 2Mb page), although other
+ * thresholds may work well depending on kernel settings of your deployment
+ * targets.
+ *
+ * hpa_hugify_style_eager
+ * This style results in jemalloc giving hugepage advice, if needed, to
+ * anonymous memory immediately after it is mapped, so huge pages can be backing
+ * that memory at page-fault time.  This is usually more efficient than doing
+ * it later, and it allows us to benefit from the hugepages from the start.
+ * Same options for purging as for the style 'none' are good starting choices:
+ * no purging, or purge_threshold=HUGEPAGE, some min_purge_delay_ms that allows
+ * for page not to be purged quickly, etc.  This is a good choice if you can
+ * afford extra memory and your application gets performance increase from
+ * transparent hughepages.
+ *
+ * hpa_hugify_style_lazy
+ * This style is suitable when you purge more aggressively (you sacrifice CPU
+ * performance for less memory).  When this style is chosen, jemalloc will
+ * hugify once hugification_threshold is reached, and dehugify before purging.
+ * If the kernel is configured to use direct compaction you may experience some
+ * allocation latency when using this style.  The best is to measure what works
+ * better for your application needs, and in the target deployment environment.
+ * This is a good choice for apps that cannot afford a lot of memory regression,
+ * but would still like to benefit from backing certain memory regions with
+ * hugepages.
+ */
+enum hpa_hugify_style_e {
+	hpa_hugify_style_auto = 0,
+	hpa_hugify_style_none = 1,
+	hpa_hugify_style_eager = 2,
+	hpa_hugify_style_lazy = 3,
+	hpa_hugify_style_limit = hpa_hugify_style_lazy + 1
+};
+typedef enum hpa_hugify_style_e hpa_hugify_style_t;
+
+extern const char *const hpa_hugify_style_names[];
+
 typedef struct hpa_shard_opts_s hpa_shard_opts_t;
 struct hpa_shard_opts_s {
 	/*
@@ -44,12 +97,64 @@ struct hpa_shard_opts_s {
 	 */
 	uint64_t hugify_delay_ms;
 
+	/*
+	 * Hugify pages synchronously (hugify will happen even if hugify_style
+	 * is not hpa_hugify_style_lazy).
+	 */
+	bool hugify_sync;
+
 	/*
 	 * Minimum amount of time between purges.
 	 */
 	uint64_t min_purge_interval_ms;
+
+	/*
+	 * Maximum number of hugepages to purge on each purging attempt.
+	 */
+	ssize_t experimental_max_purge_nhp;
+
+	/*
+	 * Minimum number of inactive bytes needed for a non-empty page to be
+	 * considered purgable.
+	 *
+	 * When the number of touched inactive bytes on non-empty hugepage is
+	 * >= purge_threshold, the page is purgable.  Empty pages are always
+	 * purgable.  Setting this to HUGEPAGE bytes would only purge empty
+	 * pages if using hugify_style_eager and the purges would be exactly
+	 * HUGEPAGE bytes.  Depending on your kernel settings, this may result
+	 * in better performance.
+	 *
+	 * Please note, when threshold is reached, we will purge all the dirty
+	 * bytes, and not just up to the threshold.  If this is PAGE bytes, then
+	 * all the pages that have any dirty bytes are purgable.  We treat
+	 * purgability constraint for purge_threshold as stronger than
+	 * dirty_mult, IOW, if no page meets purge_threshold, we will not purge
+	 * even if we are above dirty_mult.
+	 */
+	size_t purge_threshold;
+
+	/*
+	 * Minimum number of ms that needs to elapse between HP page becoming
+	 * eligible for purging and actually getting purged.
+	 *
+	 * Setting this to a larger number would give better chance of reusing
+	 * that memory.  Setting it to 0 means that page is eligible for purging
+	 * as soon as it meets the purge_threshold.  The clock resets when
+	 * purgability of the page changes (page goes from being non-purgable to
+	 * purgable).  When using eager style you probably want to allow for
+	 * some delay, to avoid purging the page too quickly and give it time to
+	 * be used.
+	 */
+	uint64_t min_purge_delay_ms;
+
+	/*
+	 * Style of hugification/dehugification (see comment at
+	 * hpa_hugify_style_t for options).
+	 */
+	hpa_hugify_style_t hugify_style;
 };
 
+/* clang-format off */
 #define HPA_SHARD_OPTS_DEFAULT {					\
 	/* slab_max_alloc */						\
 	64 * 1024,							\
@@ -67,8 +172,19 @@ struct hpa_shard_opts_s {
 	false,								\
 	/* hugify_delay_ms */						\
 	10 * 1000,							\
+	/* hugify_sync */						\
+	false,								\
 	/* min_purge_interval_ms */					\
-	5 * 1000							\
+	5 * 1000,							\
+	/* experimental_max_purge_nhp */				\
+	-1,      							\
+	/* size_t purge_threshold */					\
+	PAGE,								\
+	/* min_purge_delay_ms */             				\
+	0,  								\
+	/* hugify_style */                				\
+	hpa_hugify_style_lazy						\
 }
+/* clang-format on */
 
 #endif /* JEMALLOC_INTERNAL_HPA_OPTS_H */

dep/jemalloc/include/jemalloc/internal/hpa_utils.h

@@ -0,0 +1,161 @@
+#ifndef JEMALLOC_INTERNAL_HPA_UTILS_H
+#define JEMALLOC_INTERNAL_HPA_UTILS_H
+
+#include "jemalloc/internal/hpa.h"
+#include "jemalloc/internal/extent.h"
+
+#define HPA_MIN_VAR_VEC_SIZE 8
+/*
+ * This is used for jemalloc internal tuning and may change in the future based
+ * on production traffic.
+ *
+ * This value protects two things:
+ *    1. Stack size
+ *    2. Number of huge pages that are being purged in a batch as we do not
+ *       allow allocations while making madvise syscall.
+ */
+#define HPA_PURGE_BATCH_MAX 16
+
+#ifdef JEMALLOC_HAVE_PROCESS_MADVISE
+typedef struct iovec hpa_io_vector_t;
+#else
+typedef struct {
+	void  *iov_base;
+	size_t iov_len;
+} hpa_io_vector_t;
+#endif
+
+static inline size_t
+hpa_process_madvise_max_iovec_len(void) {
+	assert(
+	    opt_process_madvise_max_batch <= PROCESS_MADVISE_MAX_BATCH_LIMIT);
+	return opt_process_madvise_max_batch == 0
+	    ? HPA_MIN_VAR_VEC_SIZE
+	    : opt_process_madvise_max_batch;
+}
+
+/* Actually invoke hooks. If we fail vectorized, use single purges */
+static void
+hpa_try_vectorized_purge(
+    hpa_hooks_t *hooks, hpa_io_vector_t *vec, size_t vlen, size_t nbytes) {
+	bool success = opt_process_madvise_max_batch > 0
+	    && !hooks->vectorized_purge(vec, vlen, nbytes);
+	if (!success) {
+		/* On failure, it is safe to purge again (potential perf
+		 * penalty) If kernel can tell exactly which regions
+		 * failed, we could avoid that penalty.
+		 */
+		for (size_t i = 0; i < vlen; ++i) {
+			hooks->purge(vec[i].iov_base, vec[i].iov_len);
+		}
+	}
+}
+
+/*
+ * This structure accumulates the regions for process_madvise. It invokes the
+ * hook when batch limit is reached.
+ */
+typedef struct {
+	hpa_io_vector_t *vp;
+	size_t           cur;
+	size_t           total_bytes;
+	size_t           capacity;
+} hpa_range_accum_t;
+
+static inline void
+hpa_range_accum_init(hpa_range_accum_t *ra, hpa_io_vector_t *v, size_t sz) {
+	ra->vp = v;
+	ra->capacity = sz;
+	ra->total_bytes = 0;
+	ra->cur = 0;
+}
+
+static inline void
+hpa_range_accum_flush(hpa_range_accum_t *ra, hpa_hooks_t *hooks) {
+	assert(ra->total_bytes > 0 && ra->cur > 0);
+	hpa_try_vectorized_purge(hooks, ra->vp, ra->cur, ra->total_bytes);
+	ra->cur = 0;
+	ra->total_bytes = 0;
+}
+
+static inline void
+hpa_range_accum_add(
+    hpa_range_accum_t *ra, void *addr, size_t sz, hpa_hooks_t *hooks) {
+	assert(ra->cur < ra->capacity);
+
+	ra->vp[ra->cur].iov_base = addr;
+	ra->vp[ra->cur].iov_len = sz;
+	ra->total_bytes += sz;
+	ra->cur++;
+
+	if (ra->cur == ra->capacity) {
+		hpa_range_accum_flush(ra, hooks);
+	}
+}
+
+static inline void
+hpa_range_accum_finish(hpa_range_accum_t *ra, hpa_hooks_t *hooks) {
+	if (ra->cur > 0) {
+		hpa_range_accum_flush(ra, hooks);
+	}
+}
+
+/*
+ * For purging more than one page we use batch of these items
+ */
+typedef struct {
+	hpdata_purge_state_t state;
+	hpdata_t            *hp;
+	bool                 dehugify;
+} hpa_purge_item_t;
+
+typedef struct hpa_purge_batch_s hpa_purge_batch_t;
+struct hpa_purge_batch_s {
+	hpa_purge_item_t *items;
+	size_t            items_capacity;
+	/* Number of huge pages to purge in current batch */
+	size_t item_cnt;
+	/* Number of ranges to purge in current batch */
+	size_t nranges;
+	/* Total number of dirty pages in current batch*/
+	size_t ndirty_in_batch;
+
+	/* Max number of huge pages to purge */
+	size_t max_hp;
+	/*
+	 * Once we are above this watermark we should not add more pages
+	 * to the same batch. This is because while we want to minimize
+	 * number of madvise calls we also do not want to be preventing
+	 * allocations from too many huge pages (which we have to do
+	 * while they are being purged)
+	 */
+	size_t range_watermark;
+
+	size_t npurged_hp_total;
+};
+
+static inline bool
+hpa_batch_full(hpa_purge_batch_t *b) {
+	/* It's okay for ranges to go above */
+	return b->npurged_hp_total == b->max_hp
+	    || b->item_cnt == b->items_capacity
+	    || b->nranges >= b->range_watermark;
+}
+
+static inline void
+hpa_batch_pass_start(hpa_purge_batch_t *b) {
+	b->item_cnt = 0;
+	b->nranges = 0;
+	b->ndirty_in_batch = 0;
+}
+
+static inline bool
+hpa_batch_empty(hpa_purge_batch_t *b) {
+	return b->item_cnt == 0;
+}
+
+/* Purge pages in a batch using given hooks */
+void hpa_purge_batch(
+    hpa_hooks_t *hooks, hpa_purge_item_t *batch, size_t batch_sz);
+
+#endif /* JEMALLOC_INTERNAL_HPA_UTILS_H */

dep/jemalloc/include/jemalloc/internal/hpdata.h

@@ -1,7 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_HPDATA_H
 #define JEMALLOC_INTERNAL_HPDATA_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/fb.h"
+#include "jemalloc/internal/nstime.h"
+#include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/ph.h"
 #include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/typed_list.h"
@@ -17,8 +20,14 @@
  * an observable property of any given region of address space).  It's just
  * hugepage-sized and hugepage-aligned; it's *potentially* huge.
  */
+
+/*
+ * The max enumeration num should not exceed 2^16 - 1, see comments in edata.h
+ * for ESET_ENUMERATE_MAX_NUM for more details.
+ */
+#define PSSET_ENUMERATE_MAX_NUM 32
 typedef struct hpdata_s hpdata_t;
-ph_structs(hpdata_age_heap, hpdata_t);
+ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM);
 struct hpdata_s {
 	/*
 	 * We likewise follow the edata convention of mangling names and forcing
@@ -64,7 +73,7 @@ struct hpdata_s {
 	bool h_hugify_allowed;
 	/* When we became a hugification candidate. */
 	nstime_t h_time_hugify_allowed;
-	bool h_in_psset_hugify_container;
+	bool     h_in_psset_hugify_container;
 
 	/* Whether or not a purge or hugify is currently happening. */
 	bool h_mid_purge;
@@ -115,6 +124,12 @@ struct hpdata_s {
 
 	/* The touched pages (using the same definition as above). */
 	fb_group_t touched_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
+
+	/* Time when this extent (hpdata) becomes eligible for purging */
+	nstime_t h_time_purge_allowed;
+
+	/* True if the extent was huge and empty last time when it was purged */
+	bool h_purged_when_empty_and_huge;
 };
 
 TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
@@ -177,8 +192,8 @@ hpdata_purge_allowed_get(const hpdata_t *hpdata) {
 
 static inline void
 hpdata_purge_allowed_set(hpdata_t *hpdata, bool purge_allowed) {
-       assert(purge_allowed == false || !hpdata->h_mid_purge);
+	assert(purge_allowed == false || !hpdata->h_mid_purge);
-       hpdata->h_purge_allowed = purge_allowed;
+	hpdata->h_purge_allowed = purge_allowed;
 }
 
 static inline bool
@@ -241,7 +256,6 @@ hpdata_changing_state_get(const hpdata_t *hpdata) {
 	return hpdata->h_mid_purge || hpdata->h_mid_hugify;
 }
 
-
 static inline bool
 hpdata_updating_get(const hpdata_t *hpdata) {
 	return hpdata->h_updating;
@@ -276,17 +290,17 @@ hpdata_longest_free_range_set(hpdata_t *hpdata, size_t longest_free_range) {
 }
 
 static inline size_t
-hpdata_nactive_get(hpdata_t *hpdata) {
+hpdata_nactive_get(const hpdata_t *hpdata) {
 	return hpdata->h_nactive;
 }
 
 static inline size_t
-hpdata_ntouched_get(hpdata_t *hpdata) {
+hpdata_ntouched_get(const hpdata_t *hpdata) {
 	return hpdata->h_ntouched;
 }
 
 static inline size_t
-hpdata_ndirty_get(hpdata_t *hpdata) {
+hpdata_ndirty_get(const hpdata_t *hpdata) {
 	return hpdata->h_ntouched - hpdata->h_nactive;
 }
 
@@ -295,6 +309,26 @@ hpdata_nretained_get(hpdata_t *hpdata) {
 	return HUGEPAGE_PAGES - hpdata->h_ntouched;
 }
 
+static inline void
+hpdata_time_purge_allowed_set(hpdata_t *hpdata, const nstime_t *v) {
+	nstime_copy(&hpdata->h_time_purge_allowed, v);
+}
+
+static inline const nstime_t *
+hpdata_time_purge_allowed_get(const hpdata_t *hpdata) {
+	return &hpdata->h_time_purge_allowed;
+}
+
+static inline bool
+hpdata_purged_when_empty_and_huge_get(const hpdata_t *hpdata) {
+	return hpdata->h_purged_when_empty_and_huge;
+}
+
+static inline void
+hpdata_purged_when_empty_and_huge_set(hpdata_t *hpdata, bool v) {
+	hpdata->h_purged_when_empty_and_huge = v;
+}
+
 static inline void
 hpdata_assert_empty(hpdata_t *hpdata) {
 	assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
@@ -308,58 +342,95 @@ hpdata_assert_empty(hpdata_t *hpdata) {
  */
 static inline bool
 hpdata_consistent(hpdata_t *hpdata) {
-	if(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES)
+	bool res = true;
-	    != hpdata_longest_free_range_get(hpdata)) {
+
-		return false;
+	const size_t active_urange_longest = fb_urange_longest(
+	    hpdata->active_pages, HUGEPAGE_PAGES);
+	const size_t longest_free_range = hpdata_longest_free_range_get(hpdata);
+	if (active_urange_longest != longest_free_range) {
+		malloc_printf(
+		    "<jemalloc>: active_fb_urange_longest=%zu != hpdata_longest_free_range=%zu\n",
+		    active_urange_longest, longest_free_range);
+		res = false;
 	}
-	if (fb_scount(hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)
+
-	    != hpdata->h_nactive) {
+	const size_t active_scount = fb_scount(
-		return false;
+	    hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
+	if (active_scount != hpdata->h_nactive) {
+		malloc_printf(
+		    "<jemalloc>: active_fb_scount=%zu != hpdata_nactive=%zu\n",
+		    active_scount, hpdata->h_nactive);
+		res = false;
 	}
-	if (fb_scount(hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)
+
-	    != hpdata->h_ntouched) {
+	const size_t touched_scount = fb_scount(
-		return false;
+	    hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
+	if (touched_scount != hpdata->h_ntouched) {
+		malloc_printf(
+		    "<jemalloc>: touched_fb_scount=%zu != hpdata_ntouched=%zu\n",
+		    touched_scount, hpdata->h_ntouched);
+		res = false;
 	}
+
 	if (hpdata->h_ntouched < hpdata->h_nactive) {
-		return false;
+		malloc_printf(
+		    "<jemalloc>: hpdata_ntouched=%zu < hpdata_nactive=%zu\n",
+		    hpdata->h_ntouched, hpdata->h_nactive);
+		res = false;
 	}
-	if (hpdata->h_huge && hpdata->h_ntouched != HUGEPAGE_PAGES) {
+
-		return false;
+	if (hpdata->h_huge && (hpdata->h_ntouched != HUGEPAGE_PAGES)) {
+		malloc_printf(
+		    "<jemalloc>: hpdata_huge=%d && (hpdata_ntouched=%zu != hugepage_pages=%zu)\n",
+		    hpdata->h_huge, hpdata->h_ntouched, HUGEPAGE_PAGES);
+		res = false;
 	}
-	if (hpdata_changing_state_get(hpdata)
+
-	    && ((hpdata->h_purge_allowed) || hpdata->h_hugify_allowed)) {
+	const bool changing_state = hpdata_changing_state_get(hpdata);
-		return false;
+	if (changing_state
+	    && (hpdata->h_purge_allowed || hpdata->h_hugify_allowed)) {
+		malloc_printf(
+		    "<jemalloc>: hpdata_changing_state=%d && (hpdata_purge_allowed=%d || hpdata_hugify_allowed=%d)\n",
+		    changing_state, hpdata->h_purge_allowed,
+		    hpdata->h_hugify_allowed);
+		res = false;
 	}
+
 	if (hpdata_hugify_allowed_get(hpdata)
 	    != hpdata_in_psset_hugify_container_get(hpdata)) {
-		return false;
+		malloc_printf(
+		    "<jemalloc>: hpdata_hugify_allowed=%d != hpdata_in_psset_hugify_container=%d\n",
+		    hpdata_hugify_allowed_get(hpdata),
+		    hpdata_in_psset_hugify_container_get(hpdata));
+		res = false;
 	}
-	return true;
+
+	return res;
 }
 
-static inline void
+#define hpdata_assert_consistent(hpdata)                                       \
-hpdata_assert_consistent(hpdata_t *hpdata) {
+	do {                                                                   \
-	assert(hpdata_consistent(hpdata));
+		assert(hpdata_consistent(hpdata));                             \
-}
+	} while (0)
 
 static inline bool
-hpdata_empty(hpdata_t *hpdata) {
+hpdata_empty(const hpdata_t *hpdata) {
 	return hpdata->h_nactive == 0;
 }
 
 static inline bool
-hpdata_full(hpdata_t *hpdata) {
+hpdata_full(const hpdata_t *hpdata) {
 	return hpdata->h_nactive == HUGEPAGE_PAGES;
 }
 
-void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
+void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 
 /*
  * Given an hpdata which can serve an allocation request, pick and reserve an
  * offset within that allocation.
  */
 void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
-void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz);
+void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
 /*
  * The hpdata_purge_prepare_t allows grabbing the metadata required to purge
@@ -368,10 +439,10 @@ void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz);
  */
 typedef struct hpdata_purge_state_s hpdata_purge_state_t;
 struct hpdata_purge_state_s {
-	size_t npurged;
+	size_t     npurged;
-	size_t ndirty_to_purge;
+	size_t     ndirty_to_purge;
 	fb_group_t to_purge[FB_NGROUPS(HUGEPAGE_PAGES)];
-	size_t next_purge_search_begin;
+	size_t     next_purge_search_begin;
 };
 
 /*
@@ -386,9 +457,11 @@ struct hpdata_purge_state_s {
  * until you're done, and then end.  Allocating out of an hpdata undergoing
  * purging is not allowed.
  *
- * Returns the number of dirty pages that will be purged.
+ * Returns the number of dirty pages that will be purged and sets nranges
+ * to number of ranges with dirty pages that will be purged.
  */
-size_t hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state);
+size_t hpdata_purge_begin(
+    hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges);
 
 /*
  * If there are more extents to purge, sets *r_purge_addr and *r_purge_size to

dep/jemalloc/include/jemalloc/internal/inspect.h

@@ -1,6 +1,9 @@
 #ifndef JEMALLOC_INTERNAL_INSPECT_H
 #define JEMALLOC_INTERNAL_INSPECT_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd_types.h"
+
 /*
  * This module contains the heap introspection capabilities.  For now they are
  * exposed purely through mallctl APIs in the experimental namespace, but this
@@ -23,7 +26,7 @@ typedef struct inspect_extent_util_stats_verbose_s
     inspect_extent_util_stats_verbose_t;
 
 struct inspect_extent_util_stats_verbose_s {
-	void *slabcur_addr;
+	void  *slabcur_addr;
 	size_t nfree;
 	size_t nregs;
 	size_t size;
@@ -31,10 +34,10 @@ struct inspect_extent_util_stats_verbose_s {
 	size_t bin_nregs;
 };
 
-void inspect_extent_util_stats_get(tsdn_t *tsdn, const void *ptr,
+void inspect_extent_util_stats_get(
-    size_t *nfree, size_t *nregs, size_t *size);
+    tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size);
 void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
-    size_t *nfree, size_t *nregs, size_t *size,
+    size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
-    size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr);
+    size_t *bin_nregs, void **slabcur_addr);
 
 #endif /* JEMALLOC_INTERNAL_INSPECT_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h

@@ -3,64 +3,65 @@
 
 #include <math.h>
 #ifdef _WIN32
-#  include <windows.h>
+#	include <windows.h>
-#  include "msvc_compat/windows_extra.h"
+#	include "msvc_compat/windows_extra.h"
-#  include "msvc_compat/strings.h"
+#	include "msvc_compat/strings.h"
-#  ifdef _WIN64
+#	ifdef _WIN64
-#    if LG_VADDR <= 32
+#		if LG_VADDR <= 32
-#      error Generate the headers using x64 vcargs
+#			error Generate the headers using x64 vcargs
-#    endif
+#		endif
-#  else
+#	else
-#    if LG_VADDR > 32
+#		if LG_VADDR > 32
-#      undef LG_VADDR
+#			undef LG_VADDR
-#      define LG_VADDR 32
+#			define LG_VADDR 32
-#    endif
+#		endif
-#  endif
+#	endif
 #else
-#  include <sys/param.h>
+#	include <sys/param.h>
-#  include <sys/mman.h>
+#	include <sys/mman.h>
-#  if !defined(__pnacl__) && !defined(__native_client__)
+#	if !defined(__pnacl__) && !defined(__native_client__)
-#    include <sys/syscall.h>
+#		include <sys/syscall.h>
-#    if !defined(SYS_write) && defined(__NR_write)
+#		if !defined(SYS_write) && defined(__NR_write)
-#      define SYS_write __NR_write
+#			define SYS_write __NR_write
-#    endif
+#		endif
-#    if defined(SYS_open) && defined(__aarch64__)
+#		if defined(SYS_open) && defined(__aarch64__)
-       /* Android headers may define SYS_open to __NR_open even though
+/* Android headers may define SYS_open to __NR_open even though
         * __NR_open may not exist on AArch64 (superseded by __NR_openat). */
-#      undef SYS_open
+#			undef SYS_open
-#    endif
+#		endif
-#    include <sys/uio.h>
+#		include <sys/uio.h>
-#  endif
+#	endif
-#  include <pthread.h>
+#	include <pthread.h>
-#  if defined(__FreeBSD__) || defined(__DragonFly__)
+#	if defined(__FreeBSD__) || defined(__DragonFly__)                     \
-#  include <pthread_np.h>
+	    || defined(__OpenBSD__)
-#  include <sched.h>
+#		include <pthread_np.h>
-#  if defined(__FreeBSD__)
+#		include <sched.h>
-#    define cpu_set_t cpuset_t
+#		if defined(__FreeBSD__)
-#  endif
+#			define cpu_set_t cpuset_t
-#  endif
+#		endif
-#  include <signal.h>
+#	endif
-#  ifdef JEMALLOC_OS_UNFAIR_LOCK
+#	include <signal.h>
-#    include <os/lock.h>
+#	ifdef JEMALLOC_OS_UNFAIR_LOCK
-#  endif
+#		include <os/lock.h>
-#  ifdef JEMALLOC_GLIBC_MALLOC_HOOK
+#	endif
-#    include <sched.h>
+#	ifdef JEMALLOC_GLIBC_MALLOC_HOOK
-#  endif
+#		include <sched.h>
-#  include <errno.h>
+#	endif
-#  include <sys/time.h>
+#	include <errno.h>
-#  include <time.h>
+#	include <sys/time.h>
-#  ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
+#	include <time.h>
-#    include <mach/mach_time.h>
+#	ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
-#  endif
+#		include <mach/mach_time.h>
+#	endif
 #endif
 #include <sys/types.h>
 
 #include <limits.h>
 #ifndef SIZE_T_MAX
-#  define SIZE_T_MAX	SIZE_MAX
+#	define SIZE_T_MAX SIZE_MAX
 #endif
 #ifndef SSIZE_MAX
-#  define SSIZE_MAX	((ssize_t)(SIZE_T_MAX >> 1))
+#	define SSIZE_MAX ((ssize_t)(SIZE_T_MAX >> 1))
 #endif
 #include <stdarg.h>
 #include <stdbool.h>
@@ -69,30 +70,30 @@
 #include <stdint.h>
 #include <stddef.h>
 #ifndef offsetof
-#  define offsetof(type, member)	((size_t)&(((type *)NULL)->member))
+#	define offsetof(type, member) ((size_t) & (((type *)NULL)->member))
 #endif
 #include <string.h>
 #include <strings.h>
 #include <ctype.h>
 #ifdef _MSC_VER
-#  include <io.h>
+#	include <io.h>
 typedef intptr_t ssize_t;
-#  define PATH_MAX 1024
+#	define PATH_MAX 1024
-#  define STDERR_FILENO 2
+#	define STDERR_FILENO 2
-#  define __func__ __FUNCTION__
+#	define __func__ __FUNCTION__
-#  ifdef JEMALLOC_HAS_RESTRICT
+#	ifdef JEMALLOC_HAS_RESTRICT
-#    define restrict __restrict
+#		define restrict __restrict
-#  endif
+#	endif
 /* Disable warnings about deprecated system functions. */
-#  pragma warning(disable: 4996)
+#	pragma warning(disable : 4996)
-#if _MSC_VER < 1800
+#	if _MSC_VER < 1800
 static int
 isblank(int c) {
 	return (c == '\t' || c == ' ');
 }
-#endif
+#	endif
 #else
-#  include <unistd.h>
+#	include <unistd.h>
 #endif
 #include <fcntl.h>
 
@@ -102,7 +103,24 @@ isblank(int c) {
  * classes.
  */
 #ifdef small
-#  undef small
+#	undef small
 #endif
 
+/*
+ * Oftentimes we'd like to perform some kind of arithmetic to obtain
+ * a pointer from another pointer but with some offset or mask applied.
+ * Naively you would accomplish this by casting the source pointer to
+ * `uintptr_t`, performing all of the relevant arithmetic, and then casting
+ * the result to the desired pointer type. However, this has the unfortunate
+ * side-effect of concealing pointer provenance, hiding useful information for
+ * optimization from the compiler (see here for details:
+ * https://clang.llvm.org/extra/clang-tidy/checks/performance/no-int-to-ptr.html
+ * )
+ * Instead what one should do is cast the source pointer to `char *` and perform
+ * the equivalent arithmetic (since `char` of course represents one byte). But
+ * because `char *` has the semantic meaning of "string", we define this typedef
+ * simply to make it clearer where we are performing such pointer arithmetic.
+ */
+typedef char byte_t;
+
 #endif /* JEMALLOC_INTERNAL_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -14,10 +14,13 @@
  */
 #cmakedefine JEMALLOC_OVERRIDE___LIBC_CALLOC
 #cmakedefine JEMALLOC_OVERRIDE___LIBC_FREE
+#cmakedefine JEMALLOC_OVERRIDE___LIBC_FREE_SIZED
+#cmakedefine JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED
 #cmakedefine JEMALLOC_OVERRIDE___LIBC_MALLOC
 #cmakedefine JEMALLOC_OVERRIDE___LIBC_MEMALIGN
 #cmakedefine JEMALLOC_OVERRIDE___LIBC_REALLOC
 #cmakedefine JEMALLOC_OVERRIDE___LIBC_VALLOC
+#cmakedefine JEMALLOC_OVERRIDE___LIBC_PVALLOC
 #cmakedefine JEMALLOC_OVERRIDE___POSIX_MEMALIGN
 
 /*
@@ -88,6 +91,9 @@
 /* Defined if pthread_getname_np(3) is available. */
 #cmakedefine JEMALLOC_HAVE_PTHREAD_GETNAME_NP
 
+/* Defined if pthread_set_name_np(3) is available. */
+#cmakedefine JEMALLOC_HAVE_PTHREAD_SET_NAME_NP
+
 /* Defined if pthread_get_name_np(3) is available. */
 #cmakedefine JEMALLOC_HAVE_PTHREAD_GET_NAME_NP
 
@@ -111,6 +117,11 @@
  */
 #cmakedefine JEMALLOC_HAVE_CLOCK_REALTIME
 
+/*
+ * Defined if clock_gettime_nsec_np(CLOCK_UPTIME_RAW) is available.
+ */
+#cmakedefine JEMALLOC_HAVE_CLOCK_GETTIME_NSEC_NP
+
 /*
  * Defined if _malloc_thread_cleanup() exists.  At least in the case of
  * FreeBSD, pthread_key_create() allocates, which if used during malloc
@@ -161,6 +172,15 @@
 /* Use gcc intrinsics for profile backtracing if defined. */
 #cmakedefine JEMALLOC_PROF_GCC
 
+/* Use frame pointer for profile backtracing if defined. Linux only. */
+#cmakedefine JEMALLOC_PROF_FRAME_POINTER
+
+/* JEMALLOC_PAGEID enabled page id */
+#cmakedefine JEMALLOC_PAGEID
+
+/* JEMALLOC_HAVE_PRCTL checks prctl */
+#cmakedefine JEMALLOC_HAVE_PRCTL
+
 /*
  * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage
  * segment (DSS).
@@ -259,6 +279,12 @@
  */
 #cmakedefine JEMALLOC_READLINKAT
 
+/*
+ * If defined, use getenv() (instead of secure_getenv() or
+ * alternatives) to access MALLOC_CONF.
+ */
+#cmakedefine JEMALLOC_FORCE_GETENV
+
 /*
  * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
  */
@@ -282,6 +308,13 @@
  */
 #cmakedefine JEMALLOC_HAVE_MADVISE_HUGE
 
+/*
+ * Defined if best-effort synchronous collapse of the native
+ * pages mapped by the memory range into transparent huge pages is supported
+ * via MADV_COLLAPSE arguments to madvise(2).
+ */
+#cmakedefine JEMALLOC_HAVE_MADVISE_COLLAPSE
+
 /*
  * Methods for purging unused pages differ between operating systems.
  *
@@ -312,9 +345,23 @@
  */
 #cmakedefine JEMALLOC_MADVISE_NOCORE
 
+/* Defined if process_madvise(2) is available. */
+#cmakedefine JEMALLOC_HAVE_PROCESS_MADVISE
+
+#cmakedefine EXPERIMENTAL_SYS_PROCESS_MADVISE_NR @EXPERIMENTAL_SYS_PROCESS_MADVISE_NR@
+
 /* Defined if mprotect(2) is available. */
 #cmakedefine JEMALLOC_HAVE_MPROTECT
 
+/* Defined if sys/sdt.h is available and sdt tracing enabled */
+#cmakedefine JEMALLOC_EXPERIMENTAL_USDT_STAP
+
+/*
+ * Defined if sys/sdt.h is unavailable, sdt tracing enabled, and
+ * platform is supported
+ */
+#cmakedefine JEMALLOC_EXPERIMENTAL_USDT_CUSTOM
+
 /*
  * Defined if transparent huge pages (THPs) are supported via the
  * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled.
@@ -378,12 +425,18 @@
 /* Adaptive mutex support in pthreads. */
 #cmakedefine JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP
 
+/* gettid() support */
+#cmakedefine JEMALLOC_HAVE_GETTID
+
 /* GNU specific sched_getcpu support */
 #cmakedefine JEMALLOC_HAVE_SCHED_GETCPU
 
 /* GNU specific sched_setaffinity support */
 #cmakedefine JEMALLOC_HAVE_SCHED_SETAFFINITY
 
+/* pthread_setaffinity_np support */
+#cmakedefine JEMALLOC_HAVE_PTHREAD_SETAFFINITY_NP
+
 /*
  * If defined, all the features necessary for background threads are present.
  */
@@ -412,6 +465,9 @@
 /* Is C++ support being built? */
 #cmakedefine JEMALLOC_ENABLE_CXX
 
+/* Are C++ exceptions enabled? */
+#cmakedefine JEMALLOC_HAVE_CXX_EXCEPTIONS
+
 /* Performs additional size checks when defined. */
 #cmakedefine JEMALLOC_OPT_SIZE_CHECKS
 
@@ -424,4 +480,18 @@
 /* If defined, realloc(ptr, 0) defaults to "free" instead of "alloc". */
 #cmakedefine JEMALLOC_ZERO_REALLOC_DEFAULT_FREE
 
+/* If defined, use volatile asm during benchmarks. */
+#cmakedefine JEMALLOC_HAVE_ASM_VOLATILE
+
+/*
+ * If defined, support the use of rdtscp to get the time stamp counter
+ * and the processor ID.
+ */
+#cmakedefine JEMALLOC_HAVE_RDTSCP
+
+/* If defined, use __int128 for optimization. */
+#cmakedefine JEMALLOC_HAVE_INT128
+
+#include "jemalloc/internal/jemalloc_internal_overrides.h"
+
 #endif /* JEMALLOC_INTERNAL_DEFS_H_ */

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h

@@ -1,39 +1,52 @@
 #ifndef JEMALLOC_INTERNAL_EXTERNS_H
 #define JEMALLOC_INTERNAL_EXTERNS_H
 
+#include "jemalloc/internal/arena_types.h"
 #include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/fxp.h"
 #include "jemalloc/internal/hpa_opts.h"
+#include "jemalloc/internal/nstime.h"
 #include "jemalloc/internal/sec_opts.h"
 #include "jemalloc/internal/tsd_types.h"
-#include "jemalloc/internal/nstime.h"
 
 /* TSD checks this to set thread local slow state accordingly. */
 extern bool malloc_slow;
 
 /* Run-time options. */
-extern bool opt_abort;
+extern bool             opt_abort;
-extern bool opt_abort_conf;
+extern bool             opt_abort_conf;
-extern bool opt_trust_madvise;
+extern bool             opt_trust_madvise;
-extern bool opt_confirm_conf;
+extern bool             opt_experimental_hpa_start_huge_if_thp_always;
-extern bool opt_hpa;
+extern bool             opt_experimental_hpa_enforce_hugify;
+extern bool             opt_confirm_conf;
+extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;
-extern sec_opts_t opt_hpa_sec_opts;
+extern sec_opts_t       opt_hpa_sec_opts;
 
 extern const char *opt_junk;
-extern bool opt_junk_alloc;
+extern bool        opt_junk_alloc;
-extern bool opt_junk_free;
+extern bool        opt_junk_free;
-extern void (*junk_free_callback)(void *ptr, size_t size);
+extern void (*JET_MUTABLE junk_free_callback)(void *ptr, size_t size);
-extern void (*junk_alloc_callback)(void *ptr, size_t size);
+extern void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size);
-extern bool opt_utrace;
+extern void (*JET_MUTABLE invalid_conf_abort)(void);
-extern bool opt_xmalloc;
+extern bool                  opt_utrace;
-extern bool opt_experimental_infallible_new;
+extern bool                  opt_xmalloc;
-extern bool opt_zero;
+extern bool                  opt_experimental_infallible_new;
-extern unsigned opt_narenas;
+extern bool                  opt_experimental_tcache_gc;
+extern bool                  opt_zero;
+extern unsigned              opt_narenas;
+extern fxp_t                 opt_narenas_ratio;
 extern zero_realloc_action_t opt_zero_realloc_action;
-extern malloc_init_t malloc_init_state;
+extern malloc_init_t         malloc_init_state;
-extern const char *zero_realloc_mode_names[];
+extern const char *const     zero_realloc_mode_names[];
-extern atomic_zu_t zero_realloc_count;
+extern atomic_zu_t           zero_realloc_count;
-extern bool opt_cache_oblivious;
+extern bool                  opt_cache_oblivious;
+extern unsigned              opt_debug_double_free_max_scan;
+extern size_t                opt_calloc_madvise_threshold;
+extern bool                  opt_disable_large_size_classes;
+
+extern const char *opt_malloc_conf_symlink;
+extern const char *opt_malloc_conf_env_var;
 
 /* Escape free-fastpath when ptr & mask == 0 (for sanitization purpose). */
 extern uintptr_t san_cache_bin_nonfast_mask;
@@ -53,23 +66,26 @@ extern unsigned manual_arena_base;
  */
 extern atomic_p_t arenas[];
 
-void *a0malloc(size_t size);
+extern unsigned huge_arena_ind;
-void a0dalloc(void *ptr);
+
-void *bootstrap_malloc(size_t size);
+void    *a0malloc(size_t size);
-void *bootstrap_calloc(size_t num, size_t size);
+void     a0dalloc(void *ptr);
-void bootstrap_free(void *ptr);
+void    *bootstrap_malloc(size_t size);
-void arena_set(unsigned ind, arena_t *arena);
+void    *bootstrap_calloc(size_t num, size_t size);
+void     bootstrap_free(void *ptr);
+void     arena_set(unsigned ind, arena_t *arena);
 unsigned narenas_total_get(void);
 arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
 arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
-void arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
+void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
-void iarena_cleanup(tsd_t *tsd);
+void     iarena_cleanup(tsd_t *tsd);
-void arena_cleanup(tsd_t *tsd);
+void     arena_cleanup(tsd_t *tsd);
-size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags);
+size_t   batch_alloc(void **ptrs, size_t num, size_t size, int flags);
-void jemalloc_prefork(void);
+void     jemalloc_prefork(void);
-void jemalloc_postfork_parent(void);
+void     jemalloc_postfork_parent(void);
-void jemalloc_postfork_child(void);
+void     jemalloc_postfork_child(void);
-void je_sdallocx_noflags(void *ptr, size_t size);
+void     sdallocx_default(void *ptr, size_t size, int flags);
-void *malloc_default(size_t size);
+void     free_default(void *ptr);
+void    *malloc_default(size_t size);
 
 #endif /* JEMALLOC_INTERNAL_EXTERNS_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -1,10 +1,14 @@
 #ifndef JEMALLOC_INTERNAL_INLINES_A_H
 #define JEMALLOC_INTERNAL_INLINES_A_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena_types.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/tcache_externs.h"
 #include "jemalloc/internal/ticker.h"
 
 JEMALLOC_ALWAYS_INLINE malloc_cpuid_t
@@ -14,6 +18,15 @@ malloc_getcpu(void) {
 	return GetCurrentProcessorNumber();
 #elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
 	return (malloc_cpuid_t)sched_getcpu();
+#elif defined(JEMALLOC_HAVE_RDTSCP)
+	unsigned int ecx;
+	asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
+	return (malloc_cpuid_t)(ecx & 0xfff);
+#elif defined(__aarch64__) && defined(__APPLE__)
+	/* Other oses most likely use tpidr_el0 instead */
+	uintptr_t c;
+	asm volatile("mrs %x0, tpidrro_el0" : "=r"(c)::"memory");
+	return (malloc_cpuid_t)(c & (1 << 3) - 1);
 #else
 	not_reached();
 	return -1;
@@ -29,8 +42,8 @@ percpu_arena_choose(void) {
 	assert(cpuid >= 0);
 
 	unsigned arena_ind;
-	if ((opt_percpu_arena == percpu_arena) || ((unsigned)cpuid < ncpus /
+	if ((opt_percpu_arena == percpu_arena)
-	    2)) {
+	    || ((unsigned)cpuid < ncpus / 2)) {
 		arena_ind = cpuid;
 	} else {
 		assert(opt_percpu_arena == per_phycpu_arena);

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -1,7 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_INLINES_B_H
 #define JEMALLOC_INTERNAL_INLINES_B_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_inlines_a.h"
 #include "jemalloc/internal/extent.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 
 static inline void
 percpu_arena_update(tsd_t *tsd, unsigned cpu) {
@@ -20,13 +23,13 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 		tcache_t *tcache = tcache_get(tsd);
 		if (tcache != NULL) {
 			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow,
+			assert(tcache_slow->arena != NULL);
-			    tcache, newarena);
+			tcache_arena_reassociate(
+			    tsd_tsdn(tsd), tcache_slow, tcache, newarena);
 		}
 	}
 }
 
-
 /* Choose an arena based on a per-thread value. */
 static inline arena_t *
 arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
@@ -47,18 +50,18 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
 		assert(ret);
 		if (tcache_available(tsd)) {
 			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			tcache_t *tcache = tsd_tcachep_get(tsd);
+			tcache_t      *tcache = tsd_tcachep_get(tsd);
 			if (tcache_slow->arena != NULL) {
 				/* See comments in tsd_tcache_data_init().*/
-				assert(tcache_slow->arena ==
+				assert(tcache_slow->arena
-				    arena_get(tsd_tsdn(tsd), 0, false));
+				    == arena_get(tsd_tsdn(tsd), 0, false));
 				if (tcache_slow->arena != ret) {
 					tcache_arena_reassociate(tsd_tsdn(tsd),
 					    tcache_slow, tcache, ret);
 				}
 			} else {
-				tcache_arena_associate(tsd_tsdn(tsd),
+				tcache_arena_associate(
-				    tcache_slow, tcache, ret);
+				    tsd_tsdn(tsd), tcache_slow, tcache, ret);
 			}
 		}
 	}
@@ -68,10 +71,10 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
 	 * auto percpu arena range, (i.e. thread is assigned to a manually
 	 * managed arena), then percpu arena is skipped.
 	 */
-	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena) &&
+	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
-	    !internal && (arena_ind_get(ret) <
+	    && !internal
-	    percpu_arena_ind_limit(opt_percpu_arena)) && (ret->last_thd !=
+	    && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
-	    tsd_tsdn(tsd))) {
+	    && (ret->last_thd != tsd_tsdn(tsd))) {
 		unsigned ind = percpu_arena_choose();
 		if (arena_ind_get(ret) != ind) {
 			percpu_arena_update(tsd, ind);

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_INLINES_C_H
 #define JEMALLOC_INTERNAL_INLINES_C_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/hook.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/log.h"
@@ -8,6 +12,15 @@
 #include "jemalloc/internal/thread_event.h"
 #include "jemalloc/internal/witness.h"
 
+/*
+ * These correspond to the macros in jemalloc/jemalloc_macros.h.  Broadly, we
+ * should have one constant here per magic value there.  Note however that the
+ * representations need not be related.
+ */
+#define TCACHE_IND_NONE ((unsigned)-1)
+#define TCACHE_IND_AUTOMATIC ((unsigned)-2)
+#define ARENA_IND_AUTOMATIC ((unsigned)-1)
+
 /*
  * Translating the names of the 'i' functions:
  *   Abbreviations used in the first part of the function name (before
@@ -41,24 +54,35 @@ isalloc(tsdn_t *tsdn, const void *ptr) {
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
+iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
-    bool is_internal, arena_t *arena, bool slow_path) {
+    bool slab, tcache_t *tcache, bool is_internal, arena_t *arena,
+    bool slow_path) {
 	void *ret;
 
+	assert(!slab || sz_can_use_slab(size)); /* slab && large is illegal */
 	assert(!is_internal || tcache == NULL);
 	assert(!is_internal || arena == NULL || arena_is_auto(arena));
 	if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) {
-		witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+		witness_assert_depth_to_rank(
-		    WITNESS_RANK_CORE, 0);
+		    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	}
 
-	ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path);
+	ret = arena_malloc(
+	    tsdn, arena, size, ind, zero, slab, tcache, slow_path);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
 	}
 	return ret;
 }
 
+JEMALLOC_ALWAYS_INLINE void *
+iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
+    bool is_internal, arena_t *arena, bool slow_path) {
+	bool slab = sz_can_use_slab(size);
+	return iallocztm_explicit_slab(
+	    tsdn, size, ind, zero, slab, tcache, is_internal, arena, slow_path);
+}
+
 JEMALLOC_ALWAYS_INLINE void *
 ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) {
 	return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false,
@@ -66,18 +90,19 @@ ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) {
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
+ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
-    tcache_t *tcache, bool is_internal, arena_t *arena) {
+    bool zero, bool slab, tcache_t *tcache, bool is_internal, arena_t *arena) {
 	void *ret;
 
+	assert(!slab || sz_can_use_slab(usize)); /* slab && large is illegal */
 	assert(usize != 0);
 	assert(usize == sz_sa2u(usize, alignment));
 	assert(!is_internal || tcache == NULL);
 	assert(!is_internal || arena == NULL || arena_is_auto(arena));
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+	witness_assert_depth_to_rank(
-	    WITNESS_RANK_CORE, 0);
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 
-	ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache);
+	ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -85,12 +110,26 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
 	return ret;
 }
 
+JEMALLOC_ALWAYS_INLINE void *
+ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
+    tcache_t *tcache, bool is_internal, arena_t *arena) {
+	return ipallocztm_explicit_slab(tsdn, usize, alignment, zero,
+	    sz_can_use_slab(usize), tcache, is_internal, arena);
+}
+
 JEMALLOC_ALWAYS_INLINE void *
 ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, arena_t *arena) {
 	return ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena);
 }
 
+JEMALLOC_ALWAYS_INLINE void *
+ipalloct_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
+    bool slab, tcache_t *tcache, arena_t *arena) {
+	return ipallocztm_explicit_slab(
+	    tsdn, usize, alignment, zero, slab, tcache, false, arena);
+}
+
 JEMALLOC_ALWAYS_INLINE void *
 ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) {
 	return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero,
@@ -108,13 +147,13 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
 	assert(ptr != NULL);
 	assert(!is_internal || tcache == NULL);
 	assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr)));
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+	witness_assert_depth_to_rank(
-	    WITNESS_RANK_CORE, 0);
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	if (config_stats && is_internal) {
 		arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr));
 	}
-	if (!is_internal && !tsdn_null(tsdn) &&
+	if (!is_internal && !tsdn_null(tsdn)
-	    tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
+	    && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
 		assert(tcache == NULL);
 	}
 	arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
@@ -128,25 +167,26 @@ idalloc(tsd_t *tsd, void *ptr) {
 JEMALLOC_ALWAYS_INLINE void
 isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+	witness_assert_depth_to_rank(
-	    WITNESS_RANK_CORE, 0);
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
     hook_ralloc_args_t *hook_args) {
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+	witness_assert_depth_to_rank(
-	    WITNESS_RANK_CORE, 0);
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-	void *p;
+	void  *p;
 	size_t usize, copysize;
 
 	usize = sz_sa2u(size, alignment);
 	if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
 		return NULL;
 	}
-	p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
+	p = ipalloct_explicit_slab(
+	    tsdn, usize, alignment, zero, slab, tcache, arena);
 	if (p == NULL) {
 		return NULL;
 	}
@@ -156,11 +196,12 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(p, ptr, copysize);
-	hook_invoke_alloc(hook_args->is_realloc
+	hook_invoke_alloc(
-	    ? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p,
+	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p,
-	    hook_args->args);
+	    (uintptr_t)p, hook_args->args);
-	hook_invoke_dalloc(hook_args->is_realloc
+	hook_invoke_dalloc(
-	    ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
+	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
+	    ptr, hook_args->args);
 	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
 	return p;
 }
@@ -173,33 +214,42 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
  * passed-around anywhere.
  */
 JEMALLOC_ALWAYS_INLINE void *
-iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
+iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    bool zero, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args)
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
-{
+    hook_ralloc_args_t *hook_args) {
 	assert(ptr != NULL);
 	assert(size != 0);
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+	witness_assert_depth_to_rank(
-	    WITNESS_RANK_CORE, 0);
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 
-	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
+	if (alignment != 0
-	    != 0) {
+	    && ((uintptr_t)ptr & ((uintptr_t)alignment - 1)) != 0) {
 		/*
 		 * Existing object alignment is inadequate; allocate new space
 		 * and copy.
 		 */
 		return iralloct_realign(tsdn, ptr, oldsize, size, alignment,
-		    zero, tcache, arena, hook_args);
+		    zero, slab, tcache, arena, hook_args);
 	}
 
 	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
-	    tcache, hook_args);
+	    slab, tcache, hook_args);
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
+    size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
+    hook_ralloc_args_t *hook_args) {
+	bool slab = sz_can_use_slab(usize);
+	return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero,
+	    slab, tcache, arena, hook_args);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    bool zero, hook_ralloc_args_t *hook_args) {
+    size_t usize, bool zero, hook_ralloc_args_t *hook_args) {
-	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero,
+	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, usize,
-	    tcache_get(tsd), NULL, hook_args);
+	    zero, tcache_get(tsd), NULL, hook_args);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -207,29 +257,27 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
     size_t alignment, bool zero, size_t *newsize) {
 	assert(ptr != NULL);
 	assert(size != 0);
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+	witness_assert_depth_to_rank(
-	    WITNESS_RANK_CORE, 0);
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 
-	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
+	if (alignment != 0
-	    != 0) {
+	    && ((uintptr_t)ptr & ((uintptr_t)alignment - 1)) != 0) {
 		/* Existing object alignment is inadequate. */
 		*newsize = oldsize;
 		return true;
 	}
 
-	return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero,
+	return arena_ralloc_no_move(
-	    newsize);
+	    tsdn, ptr, oldsize, size, extra, zero, newsize);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-fastpath_success_finish(tsd_t *tsd, uint64_t allocated_after,
+fastpath_success_finish(
-    cache_bin_t *bin, void *ret) {
+    tsd_t *tsd, uint64_t allocated_after, cache_bin_t *bin, void *ret) {
 	thread_allocated_set(tsd, allocated_after);
 	if (config_stats) {
 		bin->tstats.nrequests++;
 	}
-
-	LOG("core.malloc.exit", "result: %p", ret);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -256,7 +304,6 @@ malloc_initialized(void) {
  */
 JEMALLOC_ALWAYS_INLINE void *
 imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
-	LOG("core.malloc.entry", "size: %zu", size);
 	if (tsd_get_allocates() && unlikely(!malloc_initialized())) {
 		return fallback_alloc(size);
 	}
@@ -284,8 +331,8 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
 	sz_size2index_usize_fastpath(size, &ind, &usize);
 	/* Fast path relies on size being a bin. */
 	assert(ind < SC_NBINS);
-	assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS) &&
+	assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS)
-	    (size <= SC_SMALL_MAXCLASS));
+	    && (size <= SC_SMALL_MAXCLASS));
 
 	uint64_t allocated, threshold;
 	te_malloc_fastpath_ctx(tsd, &allocated, &threshold);
@@ -314,7 +361,9 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
 	tcache_t *tcache = tsd_tcachep_get(tsd);
 	assert(tcache == tcache_get(tsd));
 	cache_bin_t *bin = &tcache->bins[ind];
-	bool tcache_success;
+	/* Suppress spurious warning from static analysis */
+	assert(bin != NULL);
+	bool  tcache_success;
 	void *ret;
 
 	/*
@@ -337,4 +386,215 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
 	return fallback_alloc(size);
 }
 
+JEMALLOC_ALWAYS_INLINE tcache_t *
+tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) {
+	tcache_t *tcache;
+	if (tcache_ind == TCACHE_IND_AUTOMATIC) {
+		if (likely(!slow)) {
+			/* Getting tcache ptr unconditionally. */
+			tcache = tsd_tcachep_get(tsd);
+			assert(tcache == tcache_get(tsd));
+		} else if (is_alloc
+		    || likely(tsd_reentrancy_level_get(tsd) == 0)) {
+			tcache = tcache_get(tsd);
+		} else {
+			tcache = NULL;
+		}
+	} else {
+		/*
+                 * Should not specify tcache on deallocation path when being
+                 * reentrant.
+                 */
+		assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0
+		    || tsd_state_nocleanup(tsd));
+		if (tcache_ind == TCACHE_IND_NONE) {
+			tcache = NULL;
+		} else {
+			tcache = tcaches_get(tsd, tcache_ind);
+		}
+	}
+	return tcache;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) {
+	if (config_opt_size_checks) {
+		emap_alloc_ctx_t dbg_ctx;
+		emap_alloc_ctx_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr, &dbg_ctx);
+		if (alloc_ctx->szind != dbg_ctx.szind) {
+			safety_check_fail_sized_dealloc(
+			    /* current_dealloc */ true, ptr,
+			    /* true_size */ emap_alloc_ctx_usize_get(&dbg_ctx),
+			    /* input_size */
+			    emap_alloc_ctx_usize_get(alloc_ctx));
+			return true;
+		}
+		if (alloc_ctx->slab != dbg_ctx.slab) {
+			safety_check_fail(
+			    "Internal heap corruption detected: "
+			    "mismatch in slab bit");
+			return true;
+		}
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+prof_sample_aligned(const void *ptr) {
+	return ((uintptr_t)ptr & PROF_SAMPLE_ALIGNMENT_MASK) == 0;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+free_fastpath_nonfast_aligned(void *ptr, bool check_prof) {
+	/*
+         * free_fastpath do not handle two uncommon cases: 1) sampled profiled
+         * objects and 2) sampled junk & stash for use-after-free detection.
+         * Both have special alignments which are used to escape the fastpath.
+         *
+         * prof_sample is page-aligned, which covers the UAF check when both
+         * are enabled (the assertion below).  Avoiding redundant checks since
+         * this is on the fastpath -- at most one runtime branch from this.
+         */
+	if (config_debug && cache_bin_nonfast_aligned(ptr)) {
+		assert(prof_sample_aligned(ptr));
+	}
+
+	if (config_prof && check_prof) {
+		/* When prof is enabled, the prof_sample alignment is enough. */
+		if (prof_sample_aligned(ptr)) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	if (config_uaf_detection) {
+		if (cache_bin_nonfast_aligned(ptr)) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	return false;
+}
+
+/* Returns whether or not the free attempt was successful. */
+JEMALLOC_ALWAYS_INLINE
+bool
+free_fastpath(void *ptr, size_t size, bool size_hint) {
+	tsd_t *tsd = tsd_get(false);
+	/* The branch gets optimized away unless tsd_get_allocates(). */
+	if (unlikely(tsd == NULL)) {
+		return false;
+	}
+	/*
+         *  The tsd_fast() / initialized checks are folded into the branch
+         *  testing (deallocated_after >= threshold) later in this function.
+         *  The threshold will be set to 0 when !tsd_fast.
+         */
+	assert(tsd_fast(tsd)
+	    || *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0);
+
+	emap_alloc_ctx_t alloc_ctx JEMALLOC_CC_SILENCE_INIT({0, 0, false});
+	size_t                     usize;
+	if (!size_hint) {
+		bool err = emap_alloc_ctx_try_lookup_fast(
+		    tsd, &arena_emap_global, ptr, &alloc_ctx);
+
+		/* Note: profiled objects will have alloc_ctx.slab set */
+		if (unlikely(err || !alloc_ctx.slab
+		        || free_fastpath_nonfast_aligned(ptr,
+		            /* check_prof */ false))) {
+			return false;
+		}
+		assert(alloc_ctx.szind != SC_NSIZES);
+		usize = sz_index2size(alloc_ctx.szind);
+	} else {
+		/*
+                 * Check for both sizes that are too large, and for sampled /
+                 * special aligned objects.  The alignment check will also check
+                 * for null ptr.
+                 */
+		if (unlikely(size > SC_LOOKUP_MAXCLASS
+		        || free_fastpath_nonfast_aligned(ptr,
+		            /* check_prof */ true))) {
+			return false;
+		}
+		sz_size2index_usize_fastpath(size, &alloc_ctx.szind, &usize);
+		/* Max lookup class must be small. */
+		assert(alloc_ctx.szind < SC_NBINS);
+		/* This is a dead store, except when opt size checking is on. */
+		alloc_ctx.slab = true;
+	}
+	/*
+         * Currently the fastpath only handles small sizes.  The branch on
+         * SC_LOOKUP_MAXCLASS makes sure of it.  This lets us avoid checking
+         * tcache szind upper limit (i.e. tcache_max) as well.
+         */
+	assert(alloc_ctx.slab);
+
+	uint64_t deallocated, threshold;
+	te_free_fastpath_ctx(tsd, &deallocated, &threshold);
+
+	uint64_t deallocated_after = deallocated + usize;
+	/*
+         * Check for events and tsd non-nominal (fast_threshold will be set to
+         * 0) in a single branch.  Note that this handles the uninitialized case
+         * as well (TSD init will be triggered on the non-fastpath).  Therefore
+         * anything depends on a functional TSD (e.g. the alloc_ctx sanity check
+         * below) needs to be after this branch.
+         */
+	if (unlikely(deallocated_after >= threshold)) {
+		return false;
+	}
+	assert(tsd_fast(tsd));
+	bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx);
+	if (fail) {
+		/* See the comment in isfree. */
+		return true;
+	}
+
+	tcache_t    *tcache = tcache_get_from_ind(tsd, TCACHE_IND_AUTOMATIC,
+	       /* slow */ false, /* is_alloc */ false);
+	cache_bin_t *bin = &tcache->bins[alloc_ctx.szind];
+
+	/*
+         * If junking were enabled, this is where we would do it.  It's not
+         * though, since we ensured above that we're on the fast path.  Assert
+         * that to double-check.
+         */
+	assert(!opt_junk_free);
+
+	if (!cache_bin_dalloc_easy(bin, ptr)) {
+		return false;
+	}
+
+	*tsd_thread_deallocatedp_get(tsd) = deallocated_after;
+
+	return true;
+}
+
+JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW
+je_sdallocx_noflags(void *ptr, size_t size) {
+	if (!free_fastpath(ptr, size, true)) {
+		sdallocx_default(ptr, size, 0);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW
+je_sdallocx_impl(void *ptr, size_t size, int flags) {
+	if (flags != 0 || !free_fastpath(ptr, size, true)) {
+		sdallocx_default(ptr, size, flags);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW
+je_free_impl(void *ptr) {
+	if (!free_fastpath(ptr, 0, false)) {
+		free_default(ptr);
+	}
+}
+
 #endif /* JEMALLOC_INTERNAL_INLINES_C_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h

@@ -2,43 +2,46 @@
 #define JEMALLOC_INTERNAL_MACROS_H
 
 #ifdef JEMALLOC_DEBUG
-#  define JEMALLOC_ALWAYS_INLINE static inline
+#	define JEMALLOC_ALWAYS_INLINE static inline
 #else
-#  ifdef _MSC_VER
+#	ifdef _MSC_VER
-#    define JEMALLOC_ALWAYS_INLINE static __forceinline
+#		define JEMALLOC_ALWAYS_INLINE static __forceinline
-#  else
+#	else
-#    define JEMALLOC_ALWAYS_INLINE JEMALLOC_ATTR(always_inline) static inline
+#		define JEMALLOC_ALWAYS_INLINE                                 \
-#  endif
+			JEMALLOC_ATTR(always_inline) static inline
+#	endif
 #endif
 #ifdef _MSC_VER
-#  define inline _inline
+#	define inline _inline
 #endif
 
 #define UNUSED JEMALLOC_ATTR(unused)
 
-#define ZU(z)	((size_t)z)
+#define ZU(z) ((size_t)z)
-#define ZD(z)	((ssize_t)z)
+#define ZD(z) ((ssize_t)z)
-#define QU(q)	((uint64_t)q)
+#define QU(q) ((uint64_t)q)
-#define QD(q)	((int64_t)q)
+#define QD(q) ((int64_t)q)
 
-#define KZU(z)	ZU(z##ULL)
+#define KZU(z) ZU(z##ULL)
-#define KZD(z)	ZD(z##LL)
+#define KZD(z) ZD(z##LL)
-#define KQU(q)	QU(q##ULL)
+#define KQU(q) QU(q##ULL)
-#define KQD(q)	QI(q##LL)
+#define KQD(q) QI(q##LL)
 
 #ifndef __DECONST
-#  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
+#	define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
 #endif
 
 #if !defined(JEMALLOC_HAS_RESTRICT) || defined(__cplusplus)
-#  define restrict
+#	define restrict
 #endif
 
 /* Various function pointers are static and immutable except during testing. */
 #ifdef JEMALLOC_JET
-#  define JET_MUTABLE
+#	define JET_MUTABLE
+#	define JET_EXTERN extern
 #else
-#  define JET_MUTABLE const
+#	define JET_MUTABLE const
+#	define JET_EXTERN static
 #endif
 
 #define JEMALLOC_VA_ARGS_HEAD(head, ...) head
@@ -46,62 +49,94 @@
 
 /* Diagnostic suppression macros */
 #if defined(_MSC_VER) && !defined(__clang__)
-#  define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push))
+#	define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push))
-#  define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop))
+#	define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop))
-#  define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W))
+#	define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable : W))
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
-#  define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED
+#	define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 /* #pragma GCC diagnostic first appeared in gcc 4.6. */
-#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \
+#elif (defined(__GNUC__)                                                       \
-  (__GNUC_MINOR__ > 5)))) || defined(__clang__)
+    && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5))))          \
+    || defined(__clang__)
 /*
  * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang
  * diagnostic suppression macros and should not be used anywhere else.
  */
-#  define JEMALLOC_PRAGMA__(X) _Pragma(#X)
+#	define JEMALLOC_PRAGMA__(X) _Pragma(#X)
-#  define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push)
+#	define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push)
-#  define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop)
+#	define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop)
-#  define JEMALLOC_DIAGNOSTIC_IGNORE(W) \
+#	define JEMALLOC_DIAGNOSTIC_IGNORE(W)                                  \
-     JEMALLOC_PRAGMA__(GCC diagnostic ignored W)
+		JEMALLOC_PRAGMA__(GCC diagnostic ignored W)
 
 /*
  * The -Wmissing-field-initializers warning is buggy in GCC versions < 5.1 and
  * all clang versions up to version 7 (currently trunk, unreleased).  This macro
  * suppresses the warning for the affected compiler versions only.
  */
-#  if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \
+#	if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5))     \
-     defined(__clang__)
+	    || defined(__clang__)
-#    define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS  \
+#		define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \
-          JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers")
+			JEMALLOC_DIAGNOSTIC_IGNORE(                                  \
-#  else
+			    "-Wmissing-field-initializers")
-#    define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
+#	else
-#  endif
+#		define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
+#	endif
 
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS  \
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS                       \
-     JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits")
+		JEMALLOC_DIAGNOSTIC_IGNORE("-Wframe-address")
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS                         \
-     JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter")
+		JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits")
-#  if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7)
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER                    \
-#    define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \
+		JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter")
-       JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=")
+#	if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7)
-#  else
+#		define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN      \
-#    define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
+			JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=")
-#  endif
+#	else
-#  define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \
+#		define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
-  JEMALLOC_DIAGNOSTIC_PUSH \
+#	endif
-  JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER
+#	ifdef JEMALLOC_HAVE_ATTR_DEPRECATED
+#		define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED                  \
+			JEMALLOC_DIAGNOSTIC_IGNORE("-Wdeprecated-declarations")
+#	else
+#		define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED
+#	endif
+#	define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS                           \
+		JEMALLOC_DIAGNOSTIC_PUSH                                       \
+		JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER
 #else
-#  define JEMALLOC_DIAGNOSTIC_PUSH
+#	define JEMALLOC_DIAGNOSTIC_PUSH
-#  define JEMALLOC_DIAGNOSTIC_POP
+#	define JEMALLOC_DIAGNOSTIC_POP
-#  define JEMALLOC_DIAGNOSTIC_IGNORE(W)
+#	define JEMALLOC_DIAGNOSTIC_IGNORE(W)
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS
-#  define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
-#  define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
+#	define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED
+#	define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 #endif
 
+#ifdef __clang_analyzer__
+#	define JEMALLOC_CLANG_ANALYZER
+#endif
+
+#ifdef JEMALLOC_CLANG_ANALYZER
+#	define JEMALLOC_CLANG_ANALYZER_SUPPRESS __attribute__((suppress))
+#	define JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(v) = v
+#else
+#	define JEMALLOC_CLANG_ANALYZER_SUPPRESS
+#	define JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(v)
+#endif
+
+#define JEMALLOC_SUPPRESS_WARN_ON_USAGE(...)                                   \
+	JEMALLOC_DIAGNOSTIC_PUSH                                               \
+	JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED                                  \
+	__VA_ARGS__                                                            \
+	JEMALLOC_DIAGNOSTIC_POP
+
 /*
  * Disables spurious diagnostics for all headers.  Since these headers are not
  * included by users directly, it does not affect their diagnostic settings.

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_overrides.h

@@ -0,0 +1,22 @@
+#ifndef JEMALLOC_INTERNAL_OVERRIDES_H
+#define JEMALLOC_INTERNAL_OVERRIDES_H
+
+/*
+ * Under normal circumstances this header serves no purpose, as these settings
+ * can be customized via the corresponding autoconf options at configure-time.
+ * Overriding in this fashion is useful when the header files generated by
+ * autoconf are used as input for another build system.
+ */
+
+#ifdef JEMALLOC_OVERRIDE_LG_PAGE
+#	undef LG_PAGE
+#	define LG_PAGE JEMALLOC_OVERRIDE_LG_PAGE
+#endif
+
+#ifdef JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF
+#	undef JEMALLOC_CONFIG_MALLOC_CONF
+#	define JEMALLOC_CONFIG_MALLOC_CONF                                    \
+		JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF
+#endif
+
+#endif /* JEMALLOC_INTERNAL_OVERRIDES_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h

@@ -18,13 +18,13 @@ enum zero_realloc_action_e {
 typedef enum zero_realloc_action_e zero_realloc_action_t;
 
 /* Signature of write callback. */
-typedef void (write_cb_t)(void *, const char *);
+typedef void(write_cb_t)(void *, const char *);
 
 enum malloc_init_e {
-	malloc_init_uninitialized	= 3,
+	malloc_init_uninitialized = 3,
-	malloc_init_a0_initialized	= 2,
+	malloc_init_a0_initialized = 2,
-	malloc_init_recursible		= 1,
+	malloc_init_recursible = 1,
-	malloc_init_initialized		= 0 /* Common case --> jnz. */
+	malloc_init_initialized = 0 /* Common case --> jnz. */
 };
 typedef enum malloc_init_e malloc_init_t;
 
@@ -39,48 +39,46 @@ typedef enum malloc_init_e malloc_init_t;
  *
  * aaaaaaaa aaaatttt tttttttt 0znnnnnn
  */
-#define MALLOCX_ARENA_BITS	12
+#define MALLOCX_ARENA_BITS 12
-#define MALLOCX_TCACHE_BITS	12
+#define MALLOCX_TCACHE_BITS 12
-#define MALLOCX_LG_ALIGN_BITS	6
+#define MALLOCX_LG_ALIGN_BITS 6
-#define MALLOCX_ARENA_SHIFT	20
+#define MALLOCX_ARENA_SHIFT 20
-#define MALLOCX_TCACHE_SHIFT	8
+#define MALLOCX_TCACHE_SHIFT 8
-#define MALLOCX_ARENA_MASK \
+#define MALLOCX_ARENA_MASK                                                     \
-    (((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)
+	((unsigned)(((1U << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT))
 /* NB: Arena index bias decreases the maximum number of arenas by 1. */
-#define MALLOCX_ARENA_LIMIT	((1 << MALLOCX_ARENA_BITS) - 1)
+#define MALLOCX_ARENA_LIMIT ((unsigned)((1U << MALLOCX_ARENA_BITS) - 1))
-#define MALLOCX_TCACHE_MASK \
+#define MALLOCX_TCACHE_MASK                                                    \
-    (((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT)
+	((unsigned)(((1U << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT))
-#define MALLOCX_TCACHE_MAX	((1 << MALLOCX_TCACHE_BITS) - 3)
+#define MALLOCX_TCACHE_MAX ((unsigned)((1U << MALLOCX_TCACHE_BITS) - 3))
-#define MALLOCX_LG_ALIGN_MASK	((1 << MALLOCX_LG_ALIGN_BITS) - 1)
+#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1)
 /* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
-#define MALLOCX_ALIGN_GET_SPECIFIED(flags)				\
+#define MALLOCX_ALIGN_GET_SPECIFIED(flags)                                     \
-    (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
+	(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
-#define MALLOCX_ALIGN_GET(flags)					\
+#define MALLOCX_ALIGN_GET(flags)                                               \
-    (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))
+	(MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX - 1))
-#define MALLOCX_ZERO_GET(flags)						\
+#define MALLOCX_ZERO_GET(flags) ((bool)(flags & MALLOCX_ZERO))
-    ((bool)(flags & MALLOCX_ZERO))
 
-#define MALLOCX_TCACHE_GET(flags)					\
+#define MALLOCX_TCACHE_GET(flags)                                              \
-    (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2)
+	(((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT))   \
-#define MALLOCX_ARENA_GET(flags)					\
+	    - 2)
-    (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1)
+#define MALLOCX_ARENA_GET(flags)                                               \
+	(((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1)
 
 /* Smallest size class to support. */
-#define TINY_MIN		(1U << LG_TINY_MIN)
+#define TINY_MIN (1U << LG_TINY_MIN)
 
-#define LONG			((size_t)(1U << LG_SIZEOF_LONG))
+#define LONG ((size_t)(1U << LG_SIZEOF_LONG))
-#define LONG_MASK		(LONG - 1)
+#define LONG_MASK (LONG - 1)
 
 /* Return the smallest long multiple that is >= a. */
-#define LONG_CEILING(a)							\
+#define LONG_CEILING(a) (((a) + LONG_MASK) & ~LONG_MASK)
-	(((a) + LONG_MASK) & ~LONG_MASK)
 
-#define SIZEOF_PTR		(1U << LG_SIZEOF_PTR)
+#define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
-#define PTR_MASK		(SIZEOF_PTR - 1)
+#define PTR_MASK (SIZEOF_PTR - 1)
 
 /* Return the smallest (void *) multiple that is >= a. */
-#define PTR_CEILING(a)							\
+#define PTR_CEILING(a) (((a) + PTR_MASK) & ~PTR_MASK)
-	(((a) + PTR_MASK) & ~PTR_MASK)
 
 /*
  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing.
@@ -89,42 +87,62 @@ typedef enum malloc_init_e malloc_init_t;
  * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
  * only handle raw constants.
  */
-#define LG_CACHELINE		6
+#define LG_CACHELINE 6
-#define CACHELINE		64
+#define CACHELINE 64
-#define CACHELINE_MASK		(CACHELINE - 1)
+#define CACHELINE_MASK (CACHELINE - 1)
 
 /* Return the smallest cacheline multiple that is >= s. */
-#define CACHELINE_CEILING(s)						\
+#define CACHELINE_CEILING(s) (((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
-	(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
 
 /* Return the nearest aligned address at or below a. */
-#define ALIGNMENT_ADDR2BASE(a, alignment)				\
+#define ALIGNMENT_ADDR2BASE(a, alignment)                                      \
-	((void *)((uintptr_t)(a) & ((~(alignment)) + 1)))
+	((void *)(((byte_t *)(a))                                              \
+	    - (((uintptr_t)(a)) - ((uintptr_t)(a) & ((~(alignment)) + 1)))))
 
 /* Return the offset between a and the nearest aligned address at or below a. */
-#define ALIGNMENT_ADDR2OFFSET(a, alignment)				\
+#define ALIGNMENT_ADDR2OFFSET(a, alignment)                                    \
 	((size_t)((uintptr_t)(a) & (alignment - 1)))
 
 /* Return the smallest alignment multiple that is >= s. */
-#define ALIGNMENT_CEILING(s, alignment)					\
+#define ALIGNMENT_CEILING(s, alignment)                                        \
 	(((s) + (alignment - 1)) & ((~(alignment)) + 1))
 
+/*
+ * Return the nearest aligned address at or above a.
+ *
+ * While at first glance this would appear to be merely a more complicated
+ * way to perform the same computation as `ALIGNMENT_CEILING`,
+ * this has the important additional property of not concealing pointer
+ * provenance from the compiler. See the block-comment on the
+ * definition of `byte_t` for more details.
+ */
+#define ALIGNMENT_ADDR2CEILING(a, alignment)                                   \
+	((void *)(((byte_t *)(a))                                              \
+	    + (((((uintptr_t)(a)) + (alignment - 1)) & ((~(alignment)) + 1))   \
+	        - ((uintptr_t)(a)))))
+
 /* Declare a variable-length array. */
-#if __STDC_VERSION__ < 199901L
+#if __STDC_VERSION__ < 199901L || defined(__STDC_NO_VLA__)
-#  ifdef _MSC_VER
+#	ifdef _MSC_VER
-#    include <malloc.h>
+#		include <malloc.h>
-#    define alloca _alloca
+#		define alloca _alloca
-#  else
+#	else
-#    ifdef JEMALLOC_HAS_ALLOCA_H
+#		ifdef JEMALLOC_HAS_ALLOCA_H
-#      include <alloca.h>
+#			include <alloca.h>
-#    else
+#		else
-#      include <stdlib.h>
+#			include <stdlib.h>
-#    endif
+#		endif
-#  endif
+#	endif
-#  define VARIABLE_ARRAY(type, name, count) \
+#	define VARIABLE_ARRAY_UNSAFE(type, name, count)                       \
-	type *name = alloca(sizeof(type) * (count))
+		type *name = alloca(sizeof(type) * (count))
 #else
-#  define VARIABLE_ARRAY(type, name, count) type name[(count)]
+#	define VARIABLE_ARRAY_UNSAFE(type, name, count) type name[(count)]
 #endif
+#define VARIABLE_ARRAY_SIZE_MAX 2048
+#define VARIABLE_ARRAY(type, name, count)                                      \
+	assert(sizeof(type) * (count) <= VARIABLE_ARRAY_SIZE_MAX);             \
+	VARIABLE_ARRAY_UNSAFE(type, name, count)
+
+#define CALLOC_MADVISE_THRESHOLD_DEFAULT (((size_t)1) << 23) /* 8 MB */
 
 #endif /* JEMALLOC_INTERNAL_TYPES_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h

@@ -57,6 +57,15 @@
 #  define JEMALLOC_MADV_FREE 8
 #endif
 
+/*
+ * Can be defined at compile time, in cases, when it is known
+ * madvise(..., MADV_COLLAPSE) feature is supported, but MADV_COLLAPSE
+ * constant is not defined.
+ */
+#ifdef JEMALLOC_DEFINE_MADVISE_COLLAPSE
+#  define JEMALLOC_MADV_COLLAPSE 25
+#endif
+
 static const bool config_debug =
 #ifdef JEMALLOC_DEBUG
     true
@@ -78,6 +87,13 @@ static const bool have_madvise_huge =
     false
 #endif
     ;
+static const bool have_process_madvise =
+#ifdef JEMALLOC_HAVE_PROCESS_MADVISE
+    true
+#else
+    false
+#endif
+    ;
 static const bool config_fill =
 #ifdef JEMALLOC_FILL
     true
@@ -114,6 +130,13 @@ static const bool config_prof_libunwind =
     false
 #endif
     ;
+static const bool config_prof_frameptr =
+#ifdef JEMALLOC_PROF_FRAME_POINTER
+    true
+#else
+    false
+#endif
+    ;
 static const bool maps_coalesce =
 #ifdef JEMALLOC_MAPS_COALESCE
     true
@@ -215,7 +238,7 @@ static const bool config_enable_cxx =
 #endif
 ;
 
-#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
+#if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA
 #endif

dep/jemalloc/include/jemalloc/internal/jemalloc_probe.h

@@ -0,0 +1,49 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_PROBE_H
+#define JEMALLOC_INTERNAL_JEMALLOC_PROBE_H
+
+#include <jemalloc/internal/jemalloc_preamble.h>
+
+#ifdef JEMALLOC_EXPERIMENTAL_USDT_STAP
+#include <jemalloc/internal/jemalloc_probe_stap.h>
+#elif defined(JEMALLOC_EXPERIMENTAL_USDT_CUSTOM)
+#include <jemalloc/internal/jemalloc_probe_custom.h>
+#elif defined(_MSC_VER)
+#define JE_USDT(name, N, ...) /* Nothing */
+#else /*  no USDT, just check the args */
+
+#define JE_USDT(name, N, ...) _JE_USDT_CHECK_ARG##N(__VA_ARGS__)
+
+#define _JE_USDT_CHECK_ARG1(a)						\
+	do {								\
+		(void)(a);						\
+	} while (0)
+#define _JE_USDT_CHECK_ARG2(a, b)					\
+	do {								\
+		(void)(a);						\
+		(void)(b);						\
+	} while (0)
+#define _JE_USDT_CHECK_ARG3(a, b, c)					\
+	do {								\
+		(void)(a);						\
+		(void)(b);						\
+		(void)(c);						\
+	} while (0)
+#define _JE_USDT_CHECK_ARG4(a, b, c, d)					\
+	do {								\
+		(void)(a);						\
+		(void)(b);						\
+		(void)(c);						\
+		(void)(d);						\
+	} while (0)
+#define _JE_USDT_CHECK_ARG5(a, b, c, d, e)				\
+	do {								\
+		(void)(a);						\
+		(void)(b);						\
+		(void)(c);						\
+		(void)(d);						\
+		(void)(e);						\
+	} while (0)
+
+#endif /* JEMALLOC_EXPERIMENTAL_USDT_* */
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_PROBE_H */

dep/jemalloc/include/jemalloc/internal/jemalloc_probe_custom.h

@@ -0,0 +1,148 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_PROBE_CUSTOM_H
+#define JEMALLOC_INTERNAL_JEMALLOC_PROBE_CUSTOM_H
+
+/* clang-format off */
+
+/*
+ * This section is based on sys/sdt.h and
+ * https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation
+ */
+
+/* Emit NOP for the probe. */
+#if (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || \
+     defined(__arm__)) && defined(__linux__)
+#define JE_SDT_NOP nop
+#else
+#error "Architecture not supported"
+#endif
+
+/* Assembly macros */
+#define JE_SDT_S(x) #x
+
+#define JE_SDT_ASM_1(x) JE_SDT_S(x) "\n"
+
+#define JE_SDT_ASM_2(x, y)			\
+	JE_SDT_S(x) "," JE_SDT_S(y) "\n"
+
+#define JE_SDT_ASM_3(x, y, z)					\
+	JE_SDT_S(x) "," JE_SDT_S(y) ","  JE_SDT_S(z) "\n"
+
+#define JE_SDT_ASM_3(x, y, z)					\
+	JE_SDT_S(x) "," JE_SDT_S(y) ","  JE_SDT_S(z) "\n"
+
+#define JE_SDT_ASM_4(x, y, z, p)					\
+	JE_SDT_S(x) "," JE_SDT_S(y) "," JE_SDT_S(z) "," JE_SDT_S(p) "\n"
+
+#define JE_SDT_ASM_5(x, y, z, p, q)					\
+	JE_SDT_S(x) "," JE_SDT_S(y) ","	JE_SDT_S(z) "," JE_SDT_S(p) ","	\
+		JE_SDT_S(q) "\n"
+
+/* Arg size */
+#ifdef __LP64__
+#define JE_SDT_ASM_ADDR            .8byte
+#else
+#define JE_SDT_ASM_ADDR            .4byte
+#endif
+
+#define JE_SDT_NOTE_NAME  "stapsdt"
+#define JE_SDT_NOTE_TYPE  3
+
+#define JE_SDT_SEMAPHORE_NONE(provider, name)			\
+	JE_SDT_ASM_1(JE_SDT_ASM_ADDR 0) /* No Semaphore support */
+#define JE_SDT_SEMAPHORE_OPERAND(provider, name)	\
+	[__sdt_semaphore] "ip" (0) /* No Semaphore */
+
+#define JE_SDT_ASM_STRING(x)     JE_SDT_ASM_1(.asciz JE_SDT_S(x))
+
+#define JE_SDT_NOTE(provider, name, arg_template)			\
+	JE_SDT_ASM_1(990: JE_SDT_NOP)					\
+	JE_SDT_ASM_3(     .pushsection .note.stapsdt,"?","note")	\
+	JE_SDT_ASM_1(     .balign 4)					\
+	JE_SDT_ASM_3(     .4byte 992f-991f, 994f-993f, JE_SDT_NOTE_TYPE) \
+	JE_SDT_ASM_1(991: .asciz JE_SDT_NOTE_NAME)			\
+	JE_SDT_ASM_1(992: .balign 4)					\
+	JE_SDT_ASM_1(993: JE_SDT_ASM_ADDR 990b)				\
+	JE_SDT_ASM_1(     JE_SDT_ASM_ADDR _.stapsdt.base)		\
+	JE_SDT_SEMAPHORE_NONE(provider, name)				\
+	JE_SDT_ASM_STRING(provider)					\
+	JE_SDT_ASM_STRING(name)						\
+	JE_SDT_ASM_STRING(arg_template)					\
+	JE_SDT_ASM_1(994: .balign 4)					\
+	JE_SDT_ASM_1(     .popsection)
+
+#define JE_SDT_BASE							\
+	JE_SDT_ASM_1(     .ifndef _.stapsdt.base)			\
+	JE_SDT_ASM_5(     .pushsection .stapsdt.base, "aG", "progbits",	\
+		    .stapsdt.base,comdat)				\
+	JE_SDT_ASM_1(     .weak _.stapsdt.base)				\
+	JE_SDT_ASM_1(     .hidden _.stapsdt.base)			\
+	JE_SDT_ASM_1(     _.stapsdt.base: .space 1)			\
+	JE_SDT_ASM_2(     .size _.stapsdt.base, 1)			\
+	JE_SDT_ASM_1(     .popsection)					\
+	JE_SDT_ASM_1(     .endif)
+
+
+/*
+ * Default constraint for probes arguments.
+ * See https://gcc.gnu.org/onlinedocs/gcc/Constraints.html
+ */
+#ifndef JE_SDT_ARG_CONSTRAINT
+#define JE_SDT_ARG_CONSTRAINT      "nor"
+#endif
+
+#define JE_SDT_ARGARRAY(x)  ((__builtin_classify_type(x) == 14) ||  \
+			     (__builtin_classify_type(x) == 5))
+#define JE_SDT_ARGSIZE(x)   (JE_SDT_ARGARRAY(x) ? sizeof(void*) : sizeof(x))
+
+/*
+ * Format of each probe argument as operand.  Size tagged with JE_SDT_Sn,
+ * with "n" constraint.  Value is tagged with JE_SDT_An with configured
+ * constraint.
+ */
+#define JE_SDT_ARG(n, x)						\
+	[JE_SDT_S##n] "n"                ((size_t)JE_SDT_ARGSIZE(x)),	\
+		[JE_SDT_A##n] JE_SDT_ARG_CONSTRAINT(x)
+
+/* Templates to append arguments as operands. */
+#define JE_SDT_OPERANDS_0()     [__sdt_dummy] "g" (0)
+#define JE_SDT_OPERANDS_1(_1)      JE_SDT_ARG(1, _1)
+#define JE_SDT_OPERANDS_2(_1, _2)  JE_SDT_OPERANDS_1(_1), JE_SDT_ARG(2, _2)
+#define JE_SDT_OPERANDS_3(_1, _2, _3) JE_SDT_OPERANDS_2(_1, _2), JE_SDT_ARG(3, _3)
+#define JE_SDT_OPERANDS_4(_1, _2, _3, _4)			\
+	JE_SDT_OPERANDS_3(_1, _2, _3), JE_SDT_ARG(4, _4)
+#define JE_SDT_OPERANDS_5(_1, _2, _3, _4, _5)			\
+	JE_SDT_OPERANDS_4(_1, _2, _3, _4), JE_SDT_ARG(5, _5)
+#define JE_SDT_OPERANDS_6(_1, _2, _3, _4, _5, _6)			\
+	JE_SDT_OPERANDS_5(_1, _2, _3, _4, _5), JE_SDT_ARG(6, _6)
+#define JE_SDT_OPERANDS_7(_1, _2, _3, _4, _5, _6, _7)		\
+	JE_SDT_OPERANDS_6(_1, _2, _3, _4, _5, _6), JE_SDT_ARG(7, _7)
+
+/* Templates to reference the arguments from operands. */
+#define JE_SDT_ARGFMT(num)        %n[JE_SDT_S##num]@%[JE_SDT_A##num]
+#define JE_SDT_ARG_TEMPLATE_0    /* No args */
+#define JE_SDT_ARG_TEMPLATE_1    JE_SDT_ARGFMT(1)
+#define JE_SDT_ARG_TEMPLATE_2    JE_SDT_ARG_TEMPLATE_1 JE_SDT_ARGFMT(2)
+#define JE_SDT_ARG_TEMPLATE_3    JE_SDT_ARG_TEMPLATE_2 JE_SDT_ARGFMT(3)
+#define JE_SDT_ARG_TEMPLATE_4    JE_SDT_ARG_TEMPLATE_3 JE_SDT_ARGFMT(4)
+#define JE_SDT_ARG_TEMPLATE_5    JE_SDT_ARG_TEMPLATE_4 JE_SDT_ARGFMT(5)
+#define JE_SDT_ARG_TEMPLATE_6    JE_SDT_ARG_TEMPLATE_5 JE_SDT_ARGFMT(6)
+#define JE_SDT_ARG_TEMPLATE_7    JE_SDT_ARG_TEMPLATE_6 JE_SDT_ARGFMT(7)
+
+#define JE_SDT_PROBE(							\
+	provider, name, n, arglist)					\
+	do {								\
+		__asm__ __volatile__(					\
+			JE_SDT_NOTE(provider, name,			\
+				    JE_SDT_ARG_TEMPLATE_##n)		\
+			:: JE_SDT_SEMAPHORE_OPERAND(provider, name),	\
+			JE_SDT_OPERANDS_##n arglist);			\
+		__asm__ __volatile__(JE_SDT_BASE);			\
+	} while (0)
+
+#define JE_USDT(name, N, ...)						\
+  JE_SDT_PROBE(jemalloc, name, N, (__VA_ARGS__))
+
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_PROBE_CUSTOM_H */
+
+/* clang-format on */

dep/jemalloc/include/jemalloc/internal/jemalloc_probe_stap.h

@@ -0,0 +1,11 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_PROBE_STAP_H
+#define JEMALLOC_INTERNAL_JEMALLOC_PROBE_STAP_H
+
+#include <sys/sdt.h>
+
+#define JE_USDT(name, N, ...) JE_USDT_PROBE_N(name, N, ##__VA_ARGS__)
+
+#define JE_USDT_PROBE_N(name, N, ...)                                          \
+	STAP_PROBE##N(jemalloc, name, ##__VA_ARGS__)
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_PROBE_STAP_H */

dep/jemalloc/include/jemalloc/internal/large_externs.h

@@ -1,23 +1,24 @@
 #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H
 #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/hook.h"
 
 void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
-void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
+void *large_palloc(
-    bool zero);
+    tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
-bool large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
+bool  large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
-    size_t usize_max, bool zero);
+     size_t usize_max, bool zero);
 void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
     size_t alignment, bool zero, tcache_t *tcache,
     hook_ralloc_args_t *hook_args);
 
-void large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
+void   large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
-void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);
+void   large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);
-void large_dalloc(tsdn_t *tsdn, edata_t *edata);
+void   large_dalloc(tsdn_t *tsdn, edata_t *edata);
-size_t large_salloc(tsdn_t *tsdn, const edata_t *edata);
+void   large_prof_info_get(
-void large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info,
+      tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, bool reset_recent);
-    bool reset_recent);
 void large_prof_tctx_reset(edata_t *edata);
 void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size);
 

dep/jemalloc/include/jemalloc/internal/lockedint.h

@@ -1,6 +1,11 @@
 #ifndef JEMALLOC_INTERNAL_LOCKEDINT_H
 #define JEMALLOC_INTERNAL_LOCKEDINT_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/tsd_types.h"
+
 /*
  * In those architectures that support 64-bit atomics, we use atomic updates for
  * our 64-bit values.  Otherwise, we use a plain uint64_t and synchronize
@@ -25,33 +30,34 @@ struct locked_zu_s {
 };
 
 #ifndef JEMALLOC_ATOMIC_U64
-#  define LOCKEDINT_MTX_DECLARE(name) malloc_mutex_t name;
+#	define LOCKEDINT_MTX_DECLARE(name) malloc_mutex_t name;
-#  define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode)			\
+#	define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode)                  \
-    malloc_mutex_init(&(mu), name, rank, rank_mode)
+		malloc_mutex_init(&(mu), name, rank, rank_mode)
-#  define LOCKEDINT_MTX(mtx) (&(mtx))
+#	define LOCKEDINT_MTX(mtx) (&(mtx))
-#  define LOCKEDINT_MTX_LOCK(tsdn, mu) malloc_mutex_lock(tsdn, &(mu))
+#	define LOCKEDINT_MTX_LOCK(tsdn, mu) malloc_mutex_lock(tsdn, &(mu))
-#  define LOCKEDINT_MTX_UNLOCK(tsdn, mu) malloc_mutex_unlock(tsdn, &(mu))
+#	define LOCKEDINT_MTX_UNLOCK(tsdn, mu) malloc_mutex_unlock(tsdn, &(mu))
-#  define LOCKEDINT_MTX_PREFORK(tsdn, mu) malloc_mutex_prefork(tsdn, &(mu))
+#	define LOCKEDINT_MTX_PREFORK(tsdn, mu)                                \
-#  define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu)			\
+		malloc_mutex_prefork(tsdn, &(mu))
-    malloc_mutex_postfork_parent(tsdn, &(mu))
+#	define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu)                        \
-#  define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu)			\
+		malloc_mutex_postfork_parent(tsdn, &(mu))
-    malloc_mutex_postfork_child(tsdn, &(mu))
+#	define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu)                         \
+		malloc_mutex_postfork_child(tsdn, &(mu))
 #else
-#  define LOCKEDINT_MTX_DECLARE(name)
+#	define LOCKEDINT_MTX_DECLARE(name)
-#  define LOCKEDINT_MTX(mtx) NULL
+#	define LOCKEDINT_MTX(mtx) NULL
-#  define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode) false
+#	define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode) false
-#  define LOCKEDINT_MTX_LOCK(tsdn, mu)
+#	define LOCKEDINT_MTX_LOCK(tsdn, mu)
-#  define LOCKEDINT_MTX_UNLOCK(tsdn, mu)
+#	define LOCKEDINT_MTX_UNLOCK(tsdn, mu)
-#  define LOCKEDINT_MTX_PREFORK(tsdn, mu)
+#	define LOCKEDINT_MTX_PREFORK(tsdn, mu)
-#  define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu)
+#	define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu)
-#  define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu)
+#	define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu)
 #endif
 
 #ifdef JEMALLOC_ATOMIC_U64
-#  define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx) assert((mtx) == NULL)
+#	define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx) assert((mtx) == NULL)
 #else
-#  define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx)			\
+#	define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx)                       \
-    malloc_mutex_assert_owner(tsdn, (mtx))
+		malloc_mutex_assert_owner(tsdn, (mtx))
 #endif
 
 static inline uint64_t
@@ -65,8 +71,7 @@ locked_read_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p) {
 }
 
 static inline void
-locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
+locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, uint64_t x) {
-    uint64_t x) {
 	LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
 #ifdef JEMALLOC_ATOMIC_U64
 	atomic_fetch_add_u64(&p->val, x, ATOMIC_RELAXED);
@@ -76,8 +81,7 @@ locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
 }
 
 static inline void
-locked_dec_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
+locked_dec_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, uint64_t x) {
-    uint64_t x) {
 	LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
 #ifdef JEMALLOC_ATOMIC_U64
 	uint64_t r = atomic_fetch_sub_u64(&p->val, x, ATOMIC_RELAXED);
@@ -94,7 +98,7 @@ locked_inc_mod_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
     const uint64_t x, const uint64_t modulus) {
 	LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
 	uint64_t before, after;
-	bool overflow;
+	bool     overflow;
 #ifdef JEMALLOC_ATOMIC_U64
 	before = atomic_load_u64(&p->val, ATOMIC_RELAXED);
 	do {
@@ -104,8 +108,8 @@ locked_inc_mod_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
 		if (overflow) {
 			after %= modulus;
 		}
-	} while (!atomic_compare_exchange_weak_u64(&p->val, &before, after,
+	} while (!atomic_compare_exchange_weak_u64(
-	    ATOMIC_RELAXED, ATOMIC_RELAXED));
+	    &p->val, &before, after, ATOMIC_RELAXED, ATOMIC_RELAXED));
 #else
 	before = p->val;
 	after = before + x;
@@ -162,8 +166,7 @@ locked_read_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p) {
 }
 
 static inline void
-locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p,
+locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, size_t x) {
-    size_t x) {
 	LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
 #ifdef JEMALLOC_ATOMIC_U64
 	atomic_fetch_add_zu(&p->val, x, ATOMIC_RELAXED);
@@ -174,8 +177,7 @@ locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p,
 }
 
 static inline void
-locked_dec_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p,
+locked_dec_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, size_t x) {
-    size_t x) {
 	LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
 #ifdef JEMALLOC_ATOMIC_U64
 	size_t r = atomic_fetch_sub_zu(&p->val, x, ATOMIC_RELAXED);

dep/jemalloc/include/jemalloc/internal/log.h

@@ -1,14 +1,15 @@
 #ifndef JEMALLOC_INTERNAL_LOG_H
 #define JEMALLOC_INTERNAL_LOG_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
 
 #ifdef JEMALLOC_LOG
-#  define JEMALLOC_LOG_VAR_BUFSIZE 1000
+#	define JEMALLOC_LOG_VAR_BUFSIZE 1000
 #else
-#  define JEMALLOC_LOG_VAR_BUFSIZE 1
+#	define JEMALLOC_LOG_VAR_BUFSIZE 1
 #endif
 
 #define JEMALLOC_LOG_BUFSIZE 4096
@@ -26,16 +27,16 @@
  * log("extent.a", "log msg for extent.a"); // 5
  * log("extent.b", "log msg for extent.b"); // 6
  *
- * And your malloc_conf option is "log=arena.a|extent", then lines 2, 4, 5, and
+ * And your malloc_conf option is "log:arena.a|extent", then lines 2, 4, 5, and
  * 6 will print at runtime.  You can enable logging from all log vars by
- * writing "log=.".
+ * writing "log:.".
  *
  * None of this should be regarded as a stable API for right now.  It's intended
  * as a debugging interface, to let us keep around some of our printf-debugging
  * statements.
  */
 
-extern char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE];
+extern char       log_var_names[JEMALLOC_LOG_VAR_BUFSIZE];
 extern atomic_b_t log_init_done;
 
 typedef struct log_var_s log_var_t;
@@ -44,7 +45,7 @@ struct log_var_s {
 	 * Lowest bit is "inited", second lowest is "enabled".  Putting them in
 	 * a single word lets us avoid any fences on weak architectures.
 	 */
-	atomic_u_t state;
+	atomic_u_t  state;
 	const char *name;
 };
 
@@ -52,7 +53,8 @@ struct log_var_s {
 #define LOG_INITIALIZED_NOT_ENABLED 1U
 #define LOG_ENABLED 2U
 
-#define LOG_VAR_INIT(name_str) {ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str}
+#define LOG_VAR_INIT(name_str)                                                 \
+	{ ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str }
 
 /*
  * Returns the value we should assume for state (which is not necessarily
@@ -62,21 +64,21 @@ struct log_var_s {
 unsigned log_var_update_state(log_var_t *log_var);
 
 /* We factor out the metadata management to allow us to test more easily. */
-#define log_do_begin(log_var)						\
+#define log_do_begin(log_var)                                                  \
-if (config_log) {							\
+	if (config_log) {                                                      \
-	unsigned log_state = atomic_load_u(&(log_var).state,		\
+		unsigned log_state = atomic_load_u(                            \
-	    ATOMIC_RELAXED);						\
+		    &(log_var).state, ATOMIC_RELAXED);                         \
-	if (unlikely(log_state == LOG_NOT_INITIALIZED)) {		\
+		if (unlikely(log_state == LOG_NOT_INITIALIZED)) {              \
-		log_state = log_var_update_state(&(log_var));		\
+			log_state = log_var_update_state(&(log_var));          \
-		assert(log_state != LOG_NOT_INITIALIZED);		\
+			assert(log_state != LOG_NOT_INITIALIZED);              \
-	}								\
+		}                                                              \
-	if (log_state == LOG_ENABLED) {					\
+		if (log_state == LOG_ENABLED) {                                \
-		{
+			{
-			/* User code executes here. */
+/* User code executes here. */
-#define log_do_end(log_var)						\
+#define log_do_end(log_var)                                                    \
-		}							\
+	}                                                                      \
-	}								\
+	}                                                                      \
-}
+	}
 
 /*
  * MSVC has some preprocessor bugs in its expansion of __VA_ARGS__ during
@@ -87,29 +89,29 @@ if (config_log) {							\
  */
 static inline void
 log_impl_varargs(const char *name, ...) {
-	char buf[JEMALLOC_LOG_BUFSIZE];
+	char    buf[JEMALLOC_LOG_BUFSIZE];
 	va_list ap;
 
 	va_start(ap, name);
 	const char *format = va_arg(ap, const char *);
-	size_t dst_offset = 0;
+	size_t      dst_offset = 0;
 	dst_offset += malloc_snprintf(buf, JEMALLOC_LOG_BUFSIZE, "%s: ", name);
-	dst_offset += malloc_vsnprintf(buf + dst_offset,
+	dst_offset += malloc_vsnprintf(
-	    JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap);
+	    buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap);
-	dst_offset += malloc_snprintf(buf + dst_offset,
+	malloc_snprintf(
-	    JEMALLOC_LOG_BUFSIZE - dst_offset, "\n");
+	    buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, "\n");
 	va_end(ap);
 
 	malloc_write(buf);
 }
 
 /* Call as log("log.var.str", "format_string %d", arg_for_format_string); */
-#define LOG(log_var_str, ...)						\
+#define LOG(log_var_str, ...)                                                  \
-do {									\
+	do {                                                                   \
-	static log_var_t log_var = LOG_VAR_INIT(log_var_str);		\
+		static log_var_t log_var = LOG_VAR_INIT(log_var_str);          \
-	log_do_begin(log_var)						\
+		log_do_begin(log_var)                                          \
-		log_impl_varargs((log_var).name, __VA_ARGS__);		\
+		    log_impl_varargs((log_var).name, __VA_ARGS__);             \
-	log_do_end(log_var)						\
+		log_do_end(log_var)                                            \
-} while (0)
+	} while (0)
 
 #endif /* JEMALLOC_INTERNAL_LOG_H */

dep/jemalloc/include/jemalloc/internal/malloc_io.h

@@ -1,105 +1,101 @@
 #ifndef JEMALLOC_INTERNAL_MALLOC_IO_H
 #define JEMALLOC_INTERNAL_MALLOC_IO_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 
 #ifdef _WIN32
-#  ifdef _WIN64
+#	ifdef _WIN64
-#    define FMT64_PREFIX "ll"
+#		define FMT64_PREFIX "ll"
-#    define FMTPTR_PREFIX "ll"
+#		define FMTPTR_PREFIX "ll"
-#  else
+#	else
-#    define FMT64_PREFIX "ll"
+#		define FMT64_PREFIX "ll"
-#    define FMTPTR_PREFIX ""
+#		define FMTPTR_PREFIX ""
-#  endif
+#	endif
-#  define FMTd32 "d"
+#	define FMTd32 "d"
-#  define FMTu32 "u"
+#	define FMTu32 "u"
-#  define FMTx32 "x"
+#	define FMTx32 "x"
-#  define FMTd64 FMT64_PREFIX "d"
+#	define FMTd64 FMT64_PREFIX "d"
-#  define FMTu64 FMT64_PREFIX "u"
+#	define FMTu64 FMT64_PREFIX "u"
-#  define FMTx64 FMT64_PREFIX "x"
+#	define FMTx64 FMT64_PREFIX "x"
-#  define FMTdPTR FMTPTR_PREFIX "d"
+#	define FMTdPTR FMTPTR_PREFIX "d"
-#  define FMTuPTR FMTPTR_PREFIX "u"
+#	define FMTuPTR FMTPTR_PREFIX "u"
-#  define FMTxPTR FMTPTR_PREFIX "x"
+#	define FMTxPTR FMTPTR_PREFIX "x"
 #else
-#  include <inttypes.h>
+#	include <inttypes.h>
-#  define FMTd32 PRId32
+#	define FMTd32 PRId32
-#  define FMTu32 PRIu32
+#	define FMTu32 PRIu32
-#  define FMTx32 PRIx32
+#	define FMTx32 PRIx32
-#  define FMTd64 PRId64
+#	define FMTd64 PRId64
-#  define FMTu64 PRIu64
+#	define FMTu64 PRIu64
-#  define FMTx64 PRIx64
+#	define FMTx64 PRIx64
-#  define FMTdPTR PRIdPTR
+#	define FMTdPTR PRIdPTR
-#  define FMTuPTR PRIuPTR
+#	define FMTuPTR PRIuPTR
-#  define FMTxPTR PRIxPTR
+#	define FMTxPTR PRIxPTR
 #endif
 
 /* Size of stack-allocated buffer passed to buferror(). */
-#define BUFERROR_BUF		64
+#define BUFERROR_BUF 64
 
 /*
  * Size of stack-allocated buffer used by malloc_{,v,vc}printf().  This must be
  * large enough for all possible uses within jemalloc.
  */
-#define MALLOC_PRINTF_BUFSIZE	4096
+#define MALLOC_PRINTF_BUFSIZE 4096
 
 write_cb_t wrtmessage;
-int buferror(int err, char *buf, size_t buflen);
+int        buferror(int err, char *buf, size_t buflen);
-uintmax_t malloc_strtoumax(const char *restrict nptr, char **restrict endptr,
+uintmax_t  malloc_strtoumax(
-    int base);
+     const char *restrict nptr, char **restrict endptr, int base);
 void malloc_write(const char *s);
 
 /*
  * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
  * point math.
  */
-size_t malloc_vsnprintf(char *str, size_t size, const char *format,
+size_t malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap);
-    va_list ap);
 size_t malloc_snprintf(char *str, size_t size, const char *format, ...)
     JEMALLOC_FORMAT_PRINTF(3, 4);
 /*
  * The caller can set write_cb to null to choose to print with the
  * je_malloc_message hook.
  */
-void malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
+void malloc_vcprintf(
-    va_list ap);
+    write_cb_t *write_cb, void *cbopaque, const char *format, va_list ap);
 void malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
     ...) JEMALLOC_FORMAT_PRINTF(3, 4);
 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
 
-static inline ssize_t
+ssize_t malloc_write_fd(int fd, const void *buf, size_t count);
-malloc_write_fd(int fd, const void *buf, size_t count) {
+ssize_t malloc_read_fd(int fd, void *buf, size_t count);
-#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write)
+
-	/*
+static inline int
-	 * Use syscall(2) rather than write(2) when possible in order to avoid
+malloc_open(const char *path, int flags) {
-	 * the possibility of memory allocation within libc.  This is necessary
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
-	 * on FreeBSD; most operating systems do not have this problem though.
+	return (int)syscall(SYS_open, path, flags);
-	 *
+#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
-	 * syscall() returns long or int, depending on platform, so capture the
+	return (int)syscall(SYS_openat, AT_FDCWD, path, flags);
-	 * result in the widest plausible type to avoid compiler warnings.
-	 */
-	long result = syscall(SYS_write, fd, buf, count);
 #else
-	ssize_t result = (ssize_t)write(fd, buf,
+	return open(path, flags);
-#ifdef _WIN32
-	    (unsigned int)
 #endif
-	    count);
-#endif
-	return (ssize_t)result;
 }
 
-static inline ssize_t
+static inline int
-malloc_read_fd(int fd, void *buf, size_t count) {
+malloc_close(int fd) {
-#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
-	long result = syscall(SYS_read, fd, buf, count);
+	return (int)syscall(SYS_close, fd);
 #else
-	ssize_t result = read(fd, buf,
+	return close(fd);
-#ifdef _WIN32
-	    (unsigned int)
 #endif
-	    count);
+}
+
+static inline off_t
+malloc_lseek(int fd, off_t offset, int whence) {
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek)
+	return (off_t)syscall(SYS_lseek, fd, offset, whence);
+#else
+	return lseek(fd, offset, whence);
 #endif
-	return (ssize_t)result;
 }
 
 #endif /* JEMALLOC_INTERNAL_MALLOC_IO_H */

dep/jemalloc/include/jemalloc/internal/mpsc_queue.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_MPSC_QUEUE_H
 #define JEMALLOC_INTERNAL_MPSC_QUEUE_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 
 /*
@@ -25,6 +26,7 @@
  * two-stack tricks reverses orders in the lock-free first stack).
  */
 
+/* clang-format off */
 #define mpsc_queue(a_type)						\
 struct {								\
 	atomic_p_t tail;						\
@@ -130,5 +132,6 @@ a_prefix##pop_batch(a_queue_type *queue, a_list_type *dst) {		\
 	}								\
 	ql_concat(dst, &reversed, a_link);				\
 }
+/* clang-format on */
 
 #endif /* JEMALLOC_INTERNAL_MPSC_QUEUE_H */

dep/jemalloc/include/jemalloc/internal/mutex.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_MUTEX_H
 #define JEMALLOC_INTERNAL_MUTEX_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/mutex_prof.h"
 #include "jemalloc/internal/tsd.h"
@@ -30,26 +31,29 @@ struct malloc_mutex_s {
 			 * avoid prefetching a modified cacheline (for the
 			 * unlocking thread).
 			 */
-			mutex_prof_data_t	prof_data;
+			mutex_prof_data_t prof_data;
-#ifdef _WIN32
-#  if _WIN32_WINNT >= 0x0600
-			SRWLOCK         	lock;
-#  else
-			CRITICAL_SECTION	lock;
-#  endif
-#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
-			os_unfair_lock		lock;
-#elif (defined(JEMALLOC_MUTEX_INIT_CB))
-			pthread_mutex_t		lock;
-			malloc_mutex_t		*postponed_next;
-#else
-			pthread_mutex_t		lock;
-#endif
 			/*
 			 * Hint flag to avoid exclusive cache line contention
-			 * during spin waiting
+			 * during spin waiting.  Placed along with prof_data
+			 * since it's always modified even with no contention.
+			 * Modified by the lock owner only (after acquired, and
+			 * before release), and may be read by other threads.
 			 */
-			atomic_b_t		locked;
+			atomic_b_t locked;
+#ifdef _WIN32
+#	if _WIN32_WINNT >= 0x0600
+			SRWLOCK lock;
+#	else
+			CRITICAL_SECTION lock;
+#	endif
+#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
+			os_unfair_lock lock;
+#elif (defined(JEMALLOC_MUTEX_INIT_CB))
+			pthread_mutex_t lock;
+			malloc_mutex_t *postponed_next;
+#else
+			pthread_mutex_t lock;
+#endif
 		};
 		/*
 		 * We only touch witness when configured w/ debug.  However we
@@ -58,82 +62,118 @@ struct malloc_mutex_s {
 		 * memory cost.
 		 */
 #if !defined(JEMALLOC_DEBUG)
-		witness_t			witness;
+		witness_t                 witness;
-		malloc_mutex_lock_order_t	lock_order;
+		malloc_mutex_lock_order_t lock_order;
 #endif
 	};
 
 #if defined(JEMALLOC_DEBUG)
-	witness_t			witness;
+	witness_t                 witness;
-	malloc_mutex_lock_order_t	lock_order;
+	malloc_mutex_lock_order_t lock_order;
 #endif
 };
 
 #ifdef _WIN32
-#  if _WIN32_WINNT >= 0x0600
+#	if _WIN32_WINNT >= 0x0600
-#    define MALLOC_MUTEX_LOCK(m)    AcquireSRWLockExclusive(&(m)->lock)
+#		define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
-#    define MALLOC_MUTEX_UNLOCK(m)  ReleaseSRWLockExclusive(&(m)->lock)
+#		define MALLOC_MUTEX_UNLOCK(m)                                 \
-#    define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
+			ReleaseSRWLockExclusive(&(m)->lock)
-#  else
+#		define MALLOC_MUTEX_TRYLOCK(m)                                \
-#    define MALLOC_MUTEX_LOCK(m)    EnterCriticalSection(&(m)->lock)
+			(!TryAcquireSRWLockExclusive(&(m)->lock))
-#    define MALLOC_MUTEX_UNLOCK(m)  LeaveCriticalSection(&(m)->lock)
+#	else
-#    define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
+#		define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
-#  endif
+#		define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
+#		define MALLOC_MUTEX_TRYLOCK(m)                                \
+			(!TryEnterCriticalSection(&(m)->lock))
+#	endif
 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
-#    define MALLOC_MUTEX_LOCK(m)    os_unfair_lock_lock(&(m)->lock)
+#	define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
-#    define MALLOC_MUTEX_UNLOCK(m)  os_unfair_lock_unlock(&(m)->lock)
+#	define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
-#    define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
+#	define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
 #else
-#    define MALLOC_MUTEX_LOCK(m)    pthread_mutex_lock(&(m)->lock)
+#	define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
-#    define MALLOC_MUTEX_UNLOCK(m)  pthread_mutex_unlock(&(m)->lock)
+#	define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
-#    define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
+#	define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
 #endif
 
-#define LOCK_PROF_DATA_INITIALIZER					\
+#define LOCK_PROF_DATA_INITIALIZER                                             \
-    {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0,		\
+	{                                                                      \
-	    ATOMIC_INIT(0), 0, NULL, 0}
+		NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0,     \
+		    ATOMIC_INIT(0), 0, NULL, 0                                 \
+	}
 
 #ifdef _WIN32
-#  define MALLOC_MUTEX_INITIALIZER
+#	define MALLOC_MUTEX_INITIALIZER
 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
-#  if defined(JEMALLOC_DEBUG)
+#	if defined(JEMALLOC_DEBUG)
-#    define MALLOC_MUTEX_INITIALIZER					\
+#		define MALLOC_MUTEX_INITIALIZER                               \
-  {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
+			{                                                      \
-         WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
+				{{LOCK_PROF_DATA_INITIALIZER,                  \
-#  else
+				    ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
-#    define MALLOC_MUTEX_INITIALIZER                      \
+				    WITNESS_INITIALIZER(                       \
-  {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}},  \
+				        "mutex", WITNESS_RANK_OMIT),           \
-      WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
+				    0                                          \
-#  endif
+			}
+#	else
+#		define MALLOC_MUTEX_INITIALIZER                               \
+			{                                                      \
+				{{LOCK_PROF_DATA_INITIALIZER,                  \
+				    ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
+				    WITNESS_INITIALIZER(                       \
+				        "mutex", WITNESS_RANK_OMIT)            \
+			}
+#	endif
 #elif (defined(JEMALLOC_MUTEX_INIT_CB))
-#  if (defined(JEMALLOC_DEBUG))
+#	if (defined(JEMALLOC_DEBUG))
-#     define MALLOC_MUTEX_INITIALIZER					\
+#		define MALLOC_MUTEX_INITIALIZER                               \
-      {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}},	\
+			{                                                      \
-           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
+				{{LOCK_PROF_DATA_INITIALIZER,                  \
-#  else
+				    ATOMIC_INIT(false),                        \
-#     define MALLOC_MUTEX_INITIALIZER					\
+				    PTHREAD_MUTEX_INITIALIZER, NULL}},         \
-      {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}},	\
+				    WITNESS_INITIALIZER(                       \
-           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
+				        "mutex", WITNESS_RANK_OMIT),           \
-#  endif
+				    0                                          \
+			}
+#	else
+#		define MALLOC_MUTEX_INITIALIZER                               \
+			{                                                      \
+				{{LOCK_PROF_DATA_INITIALIZER,                  \
+				    ATOMIC_INIT(false),                        \
+				    PTHREAD_MUTEX_INITIALIZER, NULL}},         \
+				    WITNESS_INITIALIZER(                       \
+				        "mutex", WITNESS_RANK_OMIT)            \
+			}
+#	endif
 
 #else
-#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
+#	define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
-#  if defined(JEMALLOC_DEBUG)
+#	if defined(JEMALLOC_DEBUG)
-#    define MALLOC_MUTEX_INITIALIZER					\
+#		define MALLOC_MUTEX_INITIALIZER                               \
-     {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
+			{                                                      \
-           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
+				{{LOCK_PROF_DATA_INITIALIZER,                  \
-#  else
+				    ATOMIC_INIT(false),                        \
-#    define MALLOC_MUTEX_INITIALIZER                          \
+				    PTHREAD_MUTEX_INITIALIZER}},               \
-     {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}},	\
+				    WITNESS_INITIALIZER(                       \
-      WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
+				        "mutex", WITNESS_RANK_OMIT),           \
-#  endif
+				    0                                          \
+			}
+#	else
+#		define MALLOC_MUTEX_INITIALIZER                               \
+			{                                                      \
+				{{LOCK_PROF_DATA_INITIALIZER,                  \
+				    ATOMIC_INIT(false),                        \
+				    PTHREAD_MUTEX_INITIALIZER}},               \
+				    WITNESS_INITIALIZER(                       \
+				        "mutex", WITNESS_RANK_OMIT)            \
+			}
+#	endif
 #endif
 
 #ifdef JEMALLOC_LAZY_LOCK
 extern bool isthreaded;
 #else
-#  undef isthreaded /* Undo private_namespace.h definition. */
+#	undef isthreaded /* Undo private_namespace.h definition. */
-#  define isthreaded true
+#	define isthreaded true
 #endif
 
 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
@@ -154,7 +194,12 @@ malloc_mutex_lock_final(malloc_mutex_t *mutex) {
 
 static inline bool
 malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
-	return MALLOC_MUTEX_TRYLOCK(mutex);
+	bool failed = MALLOC_MUTEX_TRYLOCK(mutex);
+	if (!failed) {
+		atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
+	}
+
+	return failed;
 }
 
 static inline void
@@ -169,15 +214,21 @@ mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 	}
 }
 
+static inline bool
+malloc_mutex_is_locked(malloc_mutex_t *mutex) {
+	/* Used for sanity checking only. */
+	return atomic_load_b(&mutex->locked, ATOMIC_RELAXED);
+}
+
 /* Trylock: return false if the lock is successfully acquired. */
 static inline bool
 malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
 	if (isthreaded) {
 		if (malloc_mutex_trylock_final(mutex)) {
-			atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
 			return true;
 		}
+		assert(malloc_mutex_is_locked(mutex));
 		mutex_owner_stats_update(tsdn, mutex);
 	}
 	witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
@@ -199,12 +250,12 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
 	if (sum->max_n_thds < data->max_n_thds) {
 		sum->max_n_thds = data->max_n_thds;
 	}
-	uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
+	uint32_t cur_n_waiting_thds = atomic_load_u32(
-	    ATOMIC_RELAXED);
+	    &sum->n_waiting_thds, ATOMIC_RELAXED);
-	uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
+	uint32_t new_n_waiting_thds = cur_n_waiting_thds
-	    &data->n_waiting_thds, ATOMIC_RELAXED);
+	    + atomic_load_u32(&data->n_waiting_thds, ATOMIC_RELAXED);
-	atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
+	atomic_store_u32(
-	    ATOMIC_RELAXED);
+	    &sum->n_waiting_thds, new_n_waiting_thds, ATOMIC_RELAXED);
 	sum->n_owner_switches += data->n_owner_switches;
 	sum->n_lock_ops += data->n_lock_ops;
 }
@@ -215,8 +266,8 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 	if (isthreaded) {
 		if (malloc_mutex_trylock_final(mutex)) {
 			malloc_mutex_lock_slow(mutex);
-			atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
 		}
+		assert(malloc_mutex_is_locked(mutex));
 		mutex_owner_stats_update(tsdn, mutex);
 	}
 	witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
@@ -224,9 +275,10 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 
 static inline void
 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
-	atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
 	witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
 	if (isthreaded) {
+		assert(malloc_mutex_is_locked(mutex));
+		atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
 		MALLOC_MUTEX_UNLOCK(mutex);
 	}
 }
@@ -234,6 +286,9 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 static inline void
 malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 	witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
+	if (isthreaded) {
+		assert(malloc_mutex_is_locked(mutex));
+	}
 }
 
 static inline void
@@ -255,16 +310,16 @@ malloc_mutex_prof_copy(mutex_prof_data_t *dst, mutex_prof_data_t *source) {
 
 /* Copy the prof data from mutex for processing. */
 static inline void
-malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
+malloc_mutex_prof_read(
-    malloc_mutex_t *mutex) {
+    tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) {
 	/* Can only read holding the mutex. */
 	malloc_mutex_assert_owner(tsdn, mutex);
 	malloc_mutex_prof_copy(data, &mutex->prof_data);
 }
 
 static inline void
-malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
+malloc_mutex_prof_accum(
-    malloc_mutex_t *mutex) {
+    tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) {
 	mutex_prof_data_t *source = &mutex->prof_data;
 	/* Can only read holding the mutex. */
 	malloc_mutex_assert_owner(tsdn, mutex);
@@ -286,8 +341,8 @@ malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
 
 /* Compare the prof data and update to the maximum. */
 static inline void
-malloc_mutex_prof_max_update(tsdn_t *tsdn, mutex_prof_data_t *data,
+malloc_mutex_prof_max_update(
-    malloc_mutex_t *mutex) {
+    tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) {
 	mutex_prof_data_t *source = &mutex->prof_data;
 	/* Can only read holding the mutex. */
 	malloc_mutex_assert_owner(tsdn, mutex);

dep/jemalloc/include/jemalloc/internal/mutex_prof.h

@@ -1,80 +1,81 @@
 #ifndef JEMALLOC_INTERNAL_MUTEX_PROF_H
 #define JEMALLOC_INTERNAL_MUTEX_PROF_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/nstime.h"
 #include "jemalloc/internal/tsd_types.h"
 
-#define MUTEX_PROF_GLOBAL_MUTEXES					\
+#define MUTEX_PROF_GLOBAL_MUTEXES                                              \
-    OP(background_thread)						\
+	OP(background_thread)                                                  \
-    OP(max_per_bg_thd)							\
+	OP(max_per_bg_thd)                                                     \
-    OP(ctl)								\
+	OP(ctl)                                                                \
-    OP(prof)								\
+	OP(prof)                                                               \
-    OP(prof_thds_data)							\
+	OP(prof_thds_data)                                                     \
-    OP(prof_dump)							\
+	OP(prof_dump)                                                          \
-    OP(prof_recent_alloc)						\
+	OP(prof_recent_alloc)                                                  \
-    OP(prof_recent_dump)						\
+	OP(prof_recent_dump)                                                   \
-    OP(prof_stats)
+	OP(prof_stats)
 
 typedef enum {
 #define OP(mtx) global_prof_mutex_##mtx,
 	MUTEX_PROF_GLOBAL_MUTEXES
 #undef OP
-	mutex_prof_num_global_mutexes
+	    mutex_prof_num_global_mutexes
 } mutex_prof_global_ind_t;
 
-#define MUTEX_PROF_ARENA_MUTEXES					\
+#define MUTEX_PROF_ARENA_MUTEXES                                               \
-    OP(large)								\
+	OP(large)                                                              \
-    OP(extent_avail)							\
+	OP(extent_avail)                                                       \
-    OP(extents_dirty)							\
+	OP(extents_dirty)                                                      \
-    OP(extents_muzzy)							\
+	OP(extents_muzzy)                                                      \
-    OP(extents_retained)						\
+	OP(extents_retained)                                                   \
-    OP(decay_dirty)							\
+	OP(decay_dirty)                                                        \
-    OP(decay_muzzy)							\
+	OP(decay_muzzy)                                                        \
-    OP(base)								\
+	OP(base)                                                               \
-    OP(tcache_list)							\
+	OP(tcache_list)                                                        \
-    OP(hpa_shard)							\
+	OP(hpa_shard)                                                          \
-    OP(hpa_shard_grow)							\
+	OP(hpa_shard_grow)                                                     \
-    OP(hpa_sec)
+	OP(hpa_sec)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,
 	MUTEX_PROF_ARENA_MUTEXES
 #undef OP
-	mutex_prof_num_arena_mutexes
+	    mutex_prof_num_arena_mutexes
 } mutex_prof_arena_ind_t;
 
 /*
  * The forth parameter is a boolean value that is true for derived rate counters
  * and false for real ones.
  */
-#define MUTEX_PROF_UINT64_COUNTERS					\
+#define MUTEX_PROF_UINT64_COUNTERS                                             \
-    OP(num_ops, uint64_t, "n_lock_ops", false, num_ops)					\
+	OP(num_ops, uint64_t, "n_lock_ops", false, num_ops)                    \
-    OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops)				\
+	OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops)                     \
-    OP(num_wait, uint64_t, "n_waiting", false, num_wait)				\
+	OP(num_wait, uint64_t, "n_waiting", false, num_wait)                   \
-    OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait)				\
+	OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait)                   \
-    OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq)			\
+	OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq)          \
-    OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq)			\
+	OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq)           \
-    OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch)		\
+	OP(num_owner_switch, uint64_t, "n_owner_switch", false,                \
-    OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch)	\
+	    num_owner_switch)                                                  \
-    OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time)		\
+	OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch)   \
-    OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time)		\
+	OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \
-    OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time)
+	OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time)     \
+	OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time)
 
-#define MUTEX_PROF_UINT32_COUNTERS					\
+#define MUTEX_PROF_UINT32_COUNTERS                                             \
-    OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds)
+	OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds)
 
-#define MUTEX_PROF_COUNTERS						\
+#define MUTEX_PROF_COUNTERS                                                    \
-		MUTEX_PROF_UINT64_COUNTERS				\
+	MUTEX_PROF_UINT64_COUNTERS                                             \
-		MUTEX_PROF_UINT32_COUNTERS
+	MUTEX_PROF_UINT32_COUNTERS
 
 #define OP(counter, type, human, derived, base_counter) mutex_counter_##counter,
 
-#define COUNTER_ENUM(counter_list, t)					\
+#define COUNTER_ENUM(counter_list, t)                                          \
-		typedef enum {						\
+	typedef enum {                                                         \
-			counter_list					\
+		counter_list mutex_prof_num_##t##_counters                     \
-			mutex_prof_num_##t##_counters			\
+	} mutex_prof_##t##_counter_ind_t;
-		} mutex_prof_##t##_counter_ind_t;
 
 COUNTER_ENUM(MUTEX_PROF_UINT64_COUNTERS, uint64_t)
 COUNTER_ENUM(MUTEX_PROF_UINT32_COUNTERS, uint32_t)
@@ -88,17 +89,17 @@ typedef struct {
 	 * contention.  We update them once we have the lock.
 	 */
 	/* Total time (in nano seconds) spent waiting on this mutex. */
-	nstime_t		tot_wait_time;
+	nstime_t tot_wait_time;
 	/* Max time (in nano seconds) spent on a single lock operation. */
-	nstime_t		max_wait_time;
+	nstime_t max_wait_time;
 	/* # of times have to wait for this mutex (after spinning). */
-	uint64_t		n_wait_times;
+	uint64_t n_wait_times;
 	/* # of times acquired the mutex through local spinning. */
-	uint64_t		n_spin_acquired;
+	uint64_t n_spin_acquired;
 	/* Max # of threads waiting for the mutex at the same time. */
-	uint32_t		max_n_thds;
+	uint32_t max_n_thds;
 	/* Current # of threads waiting on the lock.  Atomic synced. */
-	atomic_u32_t		n_waiting_thds;
+	atomic_u32_t n_waiting_thds;
 
 	/*
 	 * Data touched on the fast path.  These are modified right after we
@@ -107,11 +108,11 @@ typedef struct {
 	 * cacheline.
 	 */
 	/* # of times the mutex holder is different than the previous one. */
-	uint64_t		n_owner_switches;
+	uint64_t n_owner_switches;
 	/* Previous mutex holder, to facilitate n_owner_switches. */
-	tsdn_t			*prev_owner;
+	tsdn_t *prev_owner;
 	/* # of lock() operations in total. */
-	uint64_t		n_lock_ops;
+	uint64_t n_lock_ops;
 } mutex_prof_data_t;
 
 #endif /* JEMALLOC_INTERNAL_MUTEX_PROF_H */

dep/jemalloc/include/jemalloc/internal/nstime.h

@@ -1,14 +1,19 @@
 #ifndef JEMALLOC_INTERNAL_NSTIME_H
 #define JEMALLOC_INTERNAL_NSTIME_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/assert.h"
+
 /* Maximum supported number of seconds (~584 years). */
 #define NSTIME_SEC_MAX KQU(18446744072)
 
 #define NSTIME_MAGIC ((uint32_t)0xb8a9ce37)
 #ifdef JEMALLOC_DEBUG
-#  define NSTIME_ZERO_INITIALIZER {0, NSTIME_MAGIC}
+#	define NSTIME_ZERO_INITIALIZER                                        \
+		{ 0, NSTIME_MAGIC }
 #else
-#  define NSTIME_ZERO_INITIALIZER {0}
+#	define NSTIME_ZERO_INITIALIZER                                        \
+		{ 0 }
 #endif
 
 typedef struct {
@@ -20,43 +25,43 @@ typedef struct {
 
 static const nstime_t nstime_zero = NSTIME_ZERO_INITIALIZER;
 
-void nstime_init(nstime_t *time, uint64_t ns);
+void     nstime_init(nstime_t *time, uint64_t ns);
-void nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
+void     nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t nstime_ns(const nstime_t *time);
+uint64_t nstime_ms(const nstime_t *time);
 uint64_t nstime_sec(const nstime_t *time);
-uint64_t nstime_msec(const nstime_t *time);
 uint64_t nstime_nsec(const nstime_t *time);
-void nstime_copy(nstime_t *time, const nstime_t *source);
+void     nstime_copy(nstime_t *time, const nstime_t *source);
-int nstime_compare(const nstime_t *a, const nstime_t *b);
+int      nstime_compare(const nstime_t *a, const nstime_t *b);
-void nstime_add(nstime_t *time, const nstime_t *addend);
+void     nstime_add(nstime_t *time, const nstime_t *addend);
-void nstime_iadd(nstime_t *time, uint64_t addend);
+void     nstime_iadd(nstime_t *time, uint64_t addend);
-void nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
+void     nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
-void nstime_isubtract(nstime_t *time, uint64_t subtrahend);
+void     nstime_isubtract(nstime_t *time, uint64_t subtrahend);
-void nstime_imultiply(nstime_t *time, uint64_t multiplier);
+void     nstime_imultiply(nstime_t *time, uint64_t multiplier);
-void nstime_idivide(nstime_t *time, uint64_t divisor);
+void     nstime_idivide(nstime_t *time, uint64_t divisor);
 uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
+uint64_t nstime_ns_between(const nstime_t *earlier, const nstime_t *later);
+uint64_t nstime_ms_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ns_since(const nstime_t *past);
+uint64_t nstime_ms_since(const nstime_t *past);
 
-typedef bool (nstime_monotonic_t)(void);
+typedef bool(nstime_monotonic_t)(void);
 extern nstime_monotonic_t *JET_MUTABLE nstime_monotonic;
 
-typedef void (nstime_update_t)(nstime_t *);
+typedef void(nstime_update_t)(nstime_t *);
 extern nstime_update_t *JET_MUTABLE nstime_update;
 
-typedef void (nstime_prof_update_t)(nstime_t *);
+typedef void(nstime_prof_update_t)(nstime_t *);
 extern nstime_prof_update_t *JET_MUTABLE nstime_prof_update;
 
 void nstime_init_update(nstime_t *time);
 void nstime_prof_init_update(nstime_t *time);
 
-enum prof_time_res_e {
+enum prof_time_res_e { prof_time_res_default = 0, prof_time_res_high = 1 };
-	prof_time_res_default = 0,
-	prof_time_res_high = 1
-};
 typedef enum prof_time_res_e prof_time_res_t;
 
-extern prof_time_res_t opt_prof_time_res;
+extern prof_time_res_t   opt_prof_time_res;
-extern const char *prof_time_res_mode_names[];
+extern const char *const prof_time_res_mode_names[];
 
 JEMALLOC_ALWAYS_INLINE void
 nstime_init_zero(nstime_t *time) {
@@ -64,7 +69,7 @@ nstime_init_zero(nstime_t *time) {
 }
 
 JEMALLOC_ALWAYS_INLINE bool
-nstime_equals_zero(nstime_t *time) {
+nstime_equals_zero(const nstime_t *time) {
 	int diff = nstime_compare(time, &nstime_zero);
 	assert(diff >= 0);
 	return diff == 0;

dep/jemalloc/include/jemalloc/internal/pa.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_PA_H
 #define JEMALLOC_INTERNAL_PA_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/decay.h"
 #include "jemalloc/internal/ecache.h"
@@ -95,12 +96,6 @@ struct pa_shard_s {
 	/* Allocates from a PAC. */
 	pac_t pac;
 
-	/*
-	 * We place a small extent cache in front of the HPA, since we intend
-	 * these configurations to use many fewer arenas, and therefore have a
-	 * higher risk of hot locks.
-	 */
-	sec_t hpa_sec;
 	hpa_shard_t hpa_shard;
 
 	/* The source of edata_t objects. */
@@ -108,7 +103,7 @@ struct pa_shard_s {
 
 	unsigned ind;
 
-	malloc_mutex_t *stats_mtx;
+	malloc_mutex_t   *stats_mtx;
 	pa_shard_stats_t *stats;
 
 	/* The emap this shard is tied to. */
@@ -120,8 +115,8 @@ struct pa_shard_s {
 
 static inline bool
 pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
-	return ecache_npages_get(&shard->pac.ecache_muzzy) == 0 &&
+	return ecache_npages_get(&shard->pac.ecache_muzzy) == 0
-	    pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
+	    && pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
 }
 
 static inline ehooks_t *
@@ -131,7 +126,7 @@ pa_shard_ehooks_get(pa_shard_t *shard) {
 
 /* Returns true on error. */
 bool pa_central_init(pa_central_t *central, base_t *base, bool hpa,
-    hpa_hooks_t *hpa_hooks);
+    const hpa_hooks_t *hpa_hooks);
 
 /* Returns true on error. */
 bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
@@ -165,6 +160,9 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
  */
 void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
 
+/* Flush any caches used by shard */
+void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard);
+
 /* Gets an edata for the given allocation. */
 edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
     size_t alignment, bool slab, szind_t szind, bool zero, bool guarded,
@@ -185,10 +183,10 @@ bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
  * (We could make generated_dirty the return value of course, but this is more
  * consistent with the shrink pathway and our error codes here).
  */
-void pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
+void    pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
-    bool *deferred_work_generated);
+       bool *deferred_work_generated);
-bool pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
+bool    pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
-    ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
 ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
 
 /*
@@ -198,10 +196,10 @@ ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
  * though, the arena, background thread, and PAC modules are tightly interwoven
  * in a way that's tricky to extricate, so we only do the HPA-specific parts.
  */
-void pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard,
+void pa_shard_set_deferral_allowed(
-    bool deferral_allowed);
+    tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed);
-void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+void     pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
-void pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+void     pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 
 /******************************************************************************/
@@ -223,13 +221,16 @@ void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
-void pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive,
+size_t pa_shard_nactive(pa_shard_t *shard);
-    size_t *ndirty, size_t *nmuzzy);
+size_t pa_shard_ndirty(pa_shard_t *shard);
+size_t pa_shard_nmuzzy(pa_shard_t *shard);
+
+void pa_shard_basic_stats_merge(
+    pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
 
 void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
-    hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
+    hpa_shard_stats_t *hpa_stats_out, size_t *resident);
-    size_t *resident);
 
 /*
  * Reads the PA-owned mutex stats into the output stats array, at the

dep/jemalloc/include/jemalloc/internal/pac.h

@@ -1,11 +1,15 @@
 #ifndef JEMALLOC_INTERNAL_PAC_H
 #define JEMALLOC_INTERNAL_PAC_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/decay.h"
+#include "jemalloc/internal/ecache.h"
+#include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
+#include "jemalloc/internal/lockedint.h"
 #include "jemalloc/internal/pai.h"
 #include "san_bump.h"
 
-
 /*
  * Page allocator classic; an implementation of the PAI interface that:
  * - Can be used for arenas with custom extent hooks.
@@ -91,12 +95,12 @@ struct pac_s {
 	ecache_t ecache_muzzy;
 	ecache_t ecache_retained;
 
-	base_t *base;
+	base_t        *base;
-	emap_t *emap;
+	emap_t        *emap;
 	edata_cache_t *edata_cache;
 
 	/* The grow info for the retained ecache. */
-	exp_grow_t exp_grow;
+	exp_grow_t     exp_grow;
 	malloc_mutex_t grow_mtx;
 
 	/* Special allocator for guarded frequently reused extents. */
@@ -115,12 +119,37 @@ struct pac_s {
 	decay_t decay_muzzy; /* muzzy --> retained */
 
 	malloc_mutex_t *stats_mtx;
-	pac_stats_t *stats;
+	pac_stats_t    *stats;
 
 	/* Extent serial number generator state. */
 	atomic_zu_t extent_sn_next;
 };
 
+typedef struct pac_thp_s pac_thp_t;
+struct pac_thp_s {
+	/*
+	 * opt_thp controls THP for user requested allocations. Settings
+	 * "always", "never" and "default" are available if THP is supported
+	 * by the OS and the default extent hooks are used:
+	 * - "always" and "never" are covered by pages_set_thp_state() in
+	 *   ehooks_default_alloc_impl().
+	 * - "default" makes no change for all the other auto arenas except
+	 *   the huge arena. For the huge arena, we might also look at
+	 *   opt_metadata_thp to decide whether to use THP or not.
+	 *   This is a temporary remedy before HPA is fully supported.
+	 */
+	bool thp_madvise;
+	/* Below fields are protected by the lock. */
+	malloc_mutex_t lock;
+	bool           auto_thp_switched;
+	atomic_u_t     n_thp_lazy;
+	/*
+	 * List that tracks HUGEPAGE aligned regions that're lazily hugified
+	 * in auto thp mode.
+	 */
+	edata_list_active_t thp_lazy_list;
+};
+
 bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
     edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
     ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
@@ -166,11 +195,11 @@ bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
  *
  * Returns true on error (if the new limit is not valid).
  */
-bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
+bool pac_retain_grow_limit_get_set(
-    size_t *new_limit);
+    tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit);
 
-bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
+bool    pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
-    ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
 ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
 void pac_reset(tsdn_t *tsdn, pac_t *pac);

dep/jemalloc/include/jemalloc/internal/pages.h

@@ -1,28 +1,39 @@
 #ifndef JEMALLOC_INTERNAL_PAGES_EXTERNS_H
 #define JEMALLOC_INTERNAL_PAGES_EXTERNS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+
+/* Actual operating system page size, detected during bootstrap, <= PAGE. */
+extern size_t os_page;
+
 /* Page size.  LG_PAGE is determined by the configure script. */
 #ifdef PAGE_MASK
-#  undef PAGE_MASK
+#	undef PAGE_MASK
 #endif
-#define PAGE		((size_t)(1U << LG_PAGE))
+#define PAGE ((size_t)(1U << LG_PAGE))
-#define PAGE_MASK	((size_t)(PAGE - 1))
+#define PAGE_MASK ((size_t)(PAGE - 1))
 /* Return the page base address for the page containing address a. */
-#define PAGE_ADDR2BASE(a)						\
+#define PAGE_ADDR2BASE(a) ALIGNMENT_ADDR2BASE(a, PAGE)
-	((void *)((uintptr_t)(a) & ~PAGE_MASK))
 /* Return the smallest pagesize multiple that is >= s. */
-#define PAGE_CEILING(s)							\
+#define PAGE_CEILING(s) (((s) + PAGE_MASK) & ~PAGE_MASK)
-	(((s) + PAGE_MASK) & ~PAGE_MASK)
 /* Return the largest pagesize multiple that is <=s. */
-#define PAGE_FLOOR(s) 							\
+#define PAGE_FLOOR(s) ((s) & ~PAGE_MASK)
-	((s) & ~PAGE_MASK)
 
 /* Huge page size.  LG_HUGEPAGE is determined by the configure script. */
-#define HUGEPAGE	((size_t)(1U << LG_HUGEPAGE))
+#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE))
-#define HUGEPAGE_MASK	((size_t)(HUGEPAGE - 1))
+#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1))
+
+/*
+ * Used to validate that the hugepage size is not unexpectedly high.  The huge
+ * page features (HPA, metadata_thp) are primarily designed with a 2M THP size
+ * in mind.  Much larger sizes are not tested and likely to cause issues such as
+ * bad fragmentation or simply broken.
+ */
+#define HUGEPAGE_MAX_EXPECTED_SIZE ((size_t)(16U << 20))
 
 #if LG_HUGEPAGE != 0
-#  define HUGEPAGE_PAGES (HUGEPAGE / PAGE)
+#	define HUGEPAGE_PAGES (HUGEPAGE / PAGE)
 #else
 /*
  * It's convenient to define arrays (or bitmaps) of HUGEPAGE_PAGES lengths.  If
@@ -31,19 +42,17 @@
  * that this value is at least 1.  (We won't ever run in this degraded state;
  * hpa_supported() returns false in this case.
  */
-#  define HUGEPAGE_PAGES 1
+#	define HUGEPAGE_PAGES 1
 #endif
 
 /* Return the huge page base address for the huge page containing address a. */
-#define HUGEPAGE_ADDR2BASE(a)						\
+#define HUGEPAGE_ADDR2BASE(a) ALIGNMENT_ADDR2BASE(a, HUGEPAGE)
-	((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK))
 /* Return the smallest pagesize multiple that is >= s. */
-#define HUGEPAGE_CEILING(s)						\
+#define HUGEPAGE_CEILING(s) (((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
-	(((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
 
 /* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */
 #if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE)
-#  define PAGES_CAN_PURGE_LAZY
+#	define PAGES_CAN_PURGE_LAZY
 #endif
 /*
  * PAGES_CAN_PURGE_FORCED is defined if forced purging is supported.
@@ -54,10 +63,11 @@
  * next step after purging on Windows anyway, there's no point in adding such
  * complexity.
  */
-#if !defined(_WIN32) && ((defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \
+#if !defined(_WIN32)                                                           \
-    defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)) || \
+    && ((defined(JEMALLOC_PURGE_MADVISE_DONTNEED)                              \
-    defined(JEMALLOC_MAPS_COALESCE))
+            && defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS))                 \
-#  define PAGES_CAN_PURGE_FORCED
+        || defined(JEMALLOC_MAPS_COALESCE))
+#	define PAGES_CAN_PURGE_FORCED
 #endif
 
 static const bool pages_can_purge_lazy =
@@ -76,7 +86,7 @@ static const bool pages_can_purge_forced =
     ;
 
 #if defined(JEMALLOC_HAVE_MADVISE_HUGE) || defined(JEMALLOC_HAVE_MEMCNTL)
-#  define PAGES_CAN_HUGIFY
+#	define PAGES_CAN_HUGIFY
 #endif
 
 static const bool pages_can_hugify =
@@ -87,32 +97,46 @@ static const bool pages_can_hugify =
 #endif
     ;
 
+/*
+ * thp_mode_t are values for opt.thp, while system_thp_mode_t is for kernel thp
+ * settings, i.e., init_system_thp_mode.
+ */
 typedef enum {
-	thp_mode_default       = 0, /* Do not change hugepage settings. */
+	thp_mode_do_nothing = 0, /* Respect kernel thp settings. */
-	thp_mode_always        = 1, /* Always set MADV_HUGEPAGE. */
+	thp_mode_always = 1,  /* Always set MADV_HUGEPAGE. */
-	thp_mode_never         = 2, /* Always set MADV_NOHUGEPAGE. */
+	thp_mode_never = 2,   /* Always set MADV_NOHUGEPAGE. */
 
-	thp_mode_names_limit   = 3, /* Used for option processing. */
+	thp_mode_names_limit = 3,  /* Used for option processing. */
-	thp_mode_not_supported = 3  /* No THP support detected. */
+	thp_mode_not_supported = 3 /* No THP support detected. */
 } thp_mode_t;
 
-#define THP_MODE_DEFAULT thp_mode_default
+typedef enum {
-extern thp_mode_t opt_thp;
+	system_thp_mode_madvise = 0,     /* Kernel THP mode: madvise */
-extern thp_mode_t init_system_thp_mode; /* Initial system wide state. */
+	system_thp_mode_always = 1,      /* Kernel THP mode: always */
-extern const char *thp_mode_names[];
+	system_thp_mode_never = 2,       /* Kernel THP mode: never */
+	system_thp_mode_not_supported = 3 /* No THP support detected. */
+} system_thp_mode_t;
+
+#define THP_MODE_DEFAULT thp_mode_do_nothing
+extern thp_mode_t        opt_thp;
+extern system_thp_mode_t init_system_thp_mode; /* Initial system wide state. */
+extern const char *const thp_mode_names[];
+extern const char *const system_thp_mode_names[];
 
 void *pages_map(void *addr, size_t size, size_t alignment, bool *commit);
-void pages_unmap(void *addr, size_t size);
+void  pages_unmap(void *addr, size_t size);
-bool pages_commit(void *addr, size_t size);
+bool  pages_commit(void *addr, size_t size);
-bool pages_decommit(void *addr, size_t size);
+bool  pages_decommit(void *addr, size_t size);
-bool pages_purge_lazy(void *addr, size_t size);
+bool  pages_purge_lazy(void *addr, size_t size);
-bool pages_purge_forced(void *addr, size_t size);
+bool  pages_purge_forced(void *addr, size_t size);
+bool pages_purge_process_madvise(void *vec, size_t ven_len, size_t total_bytes);
 bool pages_huge(void *addr, size_t size);
 bool pages_nohuge(void *addr, size_t size);
+bool pages_collapse(void *addr, size_t size);
 bool pages_dontdump(void *addr, size_t size);
 bool pages_dodump(void *addr, size_t size);
 bool pages_boot(void);
-void pages_set_thp_state (void *ptr, size_t size);
+void pages_set_thp_state(void *ptr, size_t size);
 void pages_mark_guards(void *head, void *tail);
 void pages_unmark_guards(void *head, void *tail);
 

dep/jemalloc/include/jemalloc/internal/pai.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_PAI_H
 #define JEMALLOC_INTERNAL_PAI_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/edata.h"
+#include "jemalloc/internal/tsd_types.h"
+
 /* An interface for page allocation. */
 
 typedef struct pai_s pai_t;
@@ -9,15 +13,6 @@ struct pai_s {
 	edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
 	    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
 	    bool *deferred_work_generated);
-	/*
-	 * Returns the number of extents added to the list (which may be fewer
-	 * than requested, in case of OOM).  The list should already be
-	 * initialized.  The only alignment guarantee is page-alignment, and
-	 * the results are not necessarily zeroed.
-	 */
-	size_t (*alloc_batch)(tsdn_t *tsdn, pai_t *self, size_t size,
-	    size_t nallocs, edata_list_active_t *results,
-	    bool *deferred_work_generated);
 	bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
 	    size_t old_size, size_t new_size, bool zero,
 	    bool *deferred_work_generated);
@@ -25,9 +20,6 @@ struct pai_s {
 	    size_t old_size, size_t new_size, bool *deferred_work_generated);
 	void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
 	    bool *deferred_work_generated);
-	/* This function empties out list as a side-effect of being called. */
-	void (*dalloc_batch)(tsdn_t *tsdn, pai_t *self,
-	    edata_list_active_t *list, bool *deferred_work_generated);
 	uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
 };
 
@@ -37,20 +29,12 @@ struct pai_s {
  */
 
 static inline edata_t *
-pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
+pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
-    bool zero, bool guarded, bool frequent_reuse,
+    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
-    bool *deferred_work_generated) {
 	return self->alloc(tsdn, self, size, alignment, zero, guarded,
 	    frequent_reuse, deferred_work_generated);
 }
 
-static inline size_t
-pai_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
-    edata_list_active_t *results, bool *deferred_work_generated) {
-	return self->alloc_batch(tsdn, self, size, nallocs, results,
-	    deferred_work_generated);
-}
-
 static inline bool
 pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool zero, bool *deferred_work_generated) {
@@ -61,35 +45,19 @@ pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
 static inline bool
 pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool *deferred_work_generated) {
-	return self->shrink(tsdn, self, edata, old_size, new_size,
+	return self->shrink(
-	    deferred_work_generated);
+	    tsdn, self, edata, old_size, new_size, deferred_work_generated);
 }
 
 static inline void
-pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+pai_dalloc(
-    bool *deferred_work_generated) {
+    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
 	self->dalloc(tsdn, self, edata, deferred_work_generated);
 }
 
-static inline void
-pai_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
-    bool *deferred_work_generated) {
-	self->dalloc_batch(tsdn, self, list, deferred_work_generated);
-}
-
 static inline uint64_t
 pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
 	return self->time_until_deferred_work(tsdn, self);
 }
 
-/*
- * An implementation of batch allocation that simply calls alloc once for
- * each item in the list.
- */
-size_t pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size,
-    size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated);
-/* Ditto, for dalloc. */
-void pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self,
-    edata_list_active_t *list, bool *deferred_work_generated);
-
 #endif /* JEMALLOC_INTERNAL_PAI_H */

dep/jemalloc/include/jemalloc/internal/peak.h

@@ -1,6 +1,8 @@
 #ifndef JEMALLOC_INTERNAL_PEAK_H
 #define JEMALLOC_INTERNAL_PEAK_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+
 typedef struct peak_s peak_t;
 struct peak_s {
 	/* The highest recorded peak value, after adjustment (see below). */
@@ -12,7 +14,8 @@ struct peak_s {
 	uint64_t adjustment;
 };
 
-#define PEAK_INITIALIZER {0, 0}
+#define PEAK_INITIALIZER                                                       \
+	{ 0, 0 }
 
 static inline uint64_t
 peak_max(peak_t *peak) {

dep/jemalloc/include/jemalloc/internal/peak_event.h

@@ -1,6 +1,17 @@
 #ifndef JEMALLOC_INTERNAL_PEAK_EVENT_H
 #define JEMALLOC_INTERNAL_PEAK_EVENT_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/*
+ * Update every 64K by default.  We're not exposing this as a configuration
+ * option for now; we don't want to bind ourselves too tightly to any particular
+ * performance requirements for small values, or guarantee that we'll even be
+ * able to provide fine-grained accuracy.
+ */
+#define PEAK_EVENT_WAIT (64 * 1024)
+
 /*
  * While peak.h contains the simple helper struct that tracks state, this
  * contains the allocator tie-ins (and knows about tsd, the event module, etc.).
@@ -9,16 +20,9 @@
 /* Update the peak with current tsd state. */
 void peak_event_update(tsd_t *tsd);
 /* Set current state to zero. */
-void peak_event_zero(tsd_t *tsd);
+void     peak_event_zero(tsd_t *tsd);
 uint64_t peak_event_max(tsd_t *tsd);
 
-/* Manual hooks. */
+extern te_base_cb_t peak_te_handler;
-/* The activity-triggered hooks. */
-uint64_t peak_alloc_new_event_wait(tsd_t *tsd);
-uint64_t peak_alloc_postponed_event_wait(tsd_t *tsd);
-void peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed);
-uint64_t peak_dalloc_new_event_wait(tsd_t *tsd);
-uint64_t peak_dalloc_postponed_event_wait(tsd_t *tsd);
-void peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed);
 
 #endif /* JEMALLOC_INTERNAL_PEAK_EVENT_H */

dep/jemalloc/include/jemalloc/internal/ph.h

@@ -1,6 +1,10 @@
 #ifndef JEMALLOC_INTERNAL_PH_H
 #define JEMALLOC_INTERNAL_PH_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/bit_util.h"
+
 /*
  * A Pairing Heap implementation.
  *
@@ -71,9 +75,19 @@ struct ph_s {
 	size_t auxcount;
 };
 
+typedef struct ph_enumerate_vars_s ph_enumerate_vars_t;
+struct ph_enumerate_vars_s {
+	uint16_t front;
+	uint16_t rear;
+	uint16_t queue_size;
+	uint16_t visited_num;
+	uint16_t max_visit_num;
+	uint16_t max_queue_size;
+};
+
 JEMALLOC_ALWAYS_INLINE phn_link_t *
 phn_link_get(void *phn, size_t offset) {
-	return (phn_link_t *)(((uintptr_t)phn) + offset);
+	return (phn_link_t *)(((char *)phn) + offset);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -115,8 +129,7 @@ phn_prev_set(void *phn, void *prev, size_t offset) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-phn_merge_ordered(void *phn0, void *phn1, size_t offset,
+phn_merge_ordered(void *phn0, void *phn1, size_t offset, ph_cmp_t cmp) {
-    ph_cmp_t cmp) {
 	void *phn0child;
 
 	assert(phn0 != NULL);
@@ -127,6 +140,7 @@ phn_merge_ordered(void *phn0, void *phn1, size_t offset,
 	phn0child = phn_lchild_get(phn0, offset);
 	phn_next_set(phn1, phn0child, offset);
 	if (phn0child != NULL) {
+		/* NOLINTNEXTLINE(readability-suspicious-call-argument) */
 		phn_prev_set(phn0child, phn1, offset);
 	}
 	phn_lchild_set(phn0, phn1, offset);
@@ -143,6 +157,7 @@ phn_merge(void *phn0, void *phn1, size_t offset, ph_cmp_t cmp) {
 		phn_merge_ordered(phn0, phn1, offset, cmp);
 		result = phn0;
 	} else {
+		/* NOLINTNEXTLINE(readability-suspicious-call-argument) */
 		phn_merge_ordered(phn1, phn0, offset, cmp);
 		result = phn1;
 	}
@@ -156,6 +171,10 @@ phn_merge_siblings(void *phn, size_t offset, ph_cmp_t cmp) {
 	void *phn0 = phn;
 	void *phn1 = phn_next_get(phn0, offset);
 
+	if (phn1 == NULL) {
+		return phn0;
+	}
+
 	/*
 	 * Multipass merge, wherein the first two elements of a FIFO
 	 * are repeatedly merged, and each result is appended to the
@@ -164,59 +183,61 @@ phn_merge_siblings(void *phn, size_t offset, ph_cmp_t cmp) {
 	 * its tail, so we do a single pass over the sibling list to
 	 * populate the FIFO.
 	 */
-	if (phn1 != NULL) {
+	void *phnrest = phn_next_get(phn1, offset);
-		void *phnrest = phn_next_get(phn1, offset);
+	if (phnrest != NULL) {
-		if (phnrest != NULL) {
+		phn_prev_set(phnrest, NULL, offset);
-			phn_prev_set(phnrest, NULL, offset);
+	}
-		}
+	phn_prev_set(phn0, NULL, offset);
-		phn_prev_set(phn0, NULL, offset);
+	phn_next_set(phn0, NULL, offset);
-		phn_next_set(phn0, NULL, offset);
+	phn_prev_set(phn1, NULL, offset);
-		phn_prev_set(phn1, NULL, offset);
+	phn_next_set(phn1, NULL, offset);
-		phn_next_set(phn1, NULL, offset);
+	phn0 = phn_merge(phn0, phn1, offset, cmp);
-		phn0 = phn_merge(phn0, phn1, offset, cmp);
+	head = tail = phn0;
-		head = tail = phn0;
+	phn0 = phnrest;
-		phn0 = phnrest;
+	while (phn0 != NULL) {
-		while (phn0 != NULL) {
-			phn1 = phn_next_get(phn0, offset);
-			if (phn1 != NULL) {
-				phnrest = phn_next_get(phn1, offset);
-				if (phnrest != NULL) {
-					phn_prev_set(phnrest, NULL, offset);
-				}
-				phn_prev_set(phn0, NULL, offset);
-				phn_next_set(phn0, NULL, offset);
-				phn_prev_set(phn1, NULL, offset);
-				phn_next_set(phn1, NULL, offset);
-				phn0 = phn_merge(phn0, phn1, offset, cmp);
-				phn_next_set(tail, phn0, offset);
-				tail = phn0;
-				phn0 = phnrest;
-			} else {
-				phn_next_set(tail, phn0, offset);
-				tail = phn0;
-				phn0 = NULL;
-			}
-		}
-		phn0 = head;
 		phn1 = phn_next_get(phn0, offset);
 		if (phn1 != NULL) {
-			while (true) {
+			phnrest = phn_next_get(phn1, offset);
-				head = phn_next_get(phn1, offset);
+			if (phnrest != NULL) {
-				assert(phn_prev_get(phn0, offset) == NULL);
+				phn_prev_set(phnrest, NULL, offset);
-				phn_next_set(phn0, NULL, offset);
-				assert(phn_prev_get(phn1, offset) == NULL);
-				phn_next_set(phn1, NULL, offset);
-				phn0 = phn_merge(phn0, phn1, offset, cmp);
-				if (head == NULL) {
-					break;
-				}
-				phn_next_set(tail, phn0, offset);
-				tail = phn0;
-				phn0 = head;
-				phn1 = phn_next_get(phn0, offset);
 			}
+			phn_prev_set(phn0, NULL, offset);
+			phn_next_set(phn0, NULL, offset);
+			phn_prev_set(phn1, NULL, offset);
+			phn_next_set(phn1, NULL, offset);
+			phn0 = phn_merge(phn0, phn1, offset, cmp);
+			/* NOLINTNEXTLINE(readability-suspicious-call-argument) */
+			phn_next_set(tail, phn0, offset);
+			tail = phn0;
+			phn0 = phnrest;
+		} else {
+			/* NOLINTNEXTLINE(readability-suspicious-call-argument) */
+			phn_next_set(tail, phn0, offset);
+			tail = phn0;
+			phn0 = NULL;
 		}
 	}
+	phn0 = head;
+	phn1 = phn_next_get(phn0, offset);
+	if (phn1 != NULL) {
+		while (true) {
+			head = phn_next_get(phn1, offset);
+			assert(phn_prev_get(phn0, offset) == NULL);
+			phn_next_set(phn0, NULL, offset);
+			assert(phn_prev_get(phn1, offset) == NULL);
+			phn_next_set(phn1, NULL, offset);
+			phn0 = phn_merge(phn0, phn1, offset, cmp);
+			if (head == NULL) {
+				break;
+			}
+			/* NOLINTNEXTLINE(readability-suspicious-call-argument) */
+			phn_next_set(tail, phn0, offset);
+			tail = phn0;
+			phn0 = head;
+			phn1 = phn_next_get(phn0, offset);
+		}
+	}
+
 	return phn0;
 }
 
@@ -230,7 +251,7 @@ ph_merge_aux(ph_t *ph, size_t offset, ph_cmp_t cmp) {
 		phn_prev_set(phn, NULL, offset);
 		phn = phn_merge_siblings(phn, offset, cmp);
 		assert(phn_next_get(phn, offset) == NULL);
-		ph->root = phn_merge(ph->root, phn, offset, cmp);
+		phn_merge_ordered(ph->root, phn, offset, cmp);
 	}
 }
 
@@ -298,6 +319,7 @@ ph_try_aux_merge_pair(ph_t *ph, size_t offset, ph_cmp_t cmp) {
 	phn0 = phn_merge(phn0, phn1, offset, cmp);
 	phn_next_set(phn0, next_phn1, offset);
 	if (next_phn1 != NULL) {
+		/* NOLINTNEXTLINE(readability-suspicious-call-argument) */
 		phn_prev_set(next_phn1, phn0, offset);
 	}
 	phn_next_set(ph->root, phn0, offset);
@@ -318,36 +340,36 @@ ph_insert(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) {
 	 */
 	if (ph->root == NULL) {
 		ph->root = phn;
-	} else {
+		return;
-		/*
-		 * As a special case, check to see if we can replace the root.
-		 * This is practically common in some important cases, and lets
-		 * us defer some insertions (hopefully, until the point where
-		 * some of the items in the aux list have been removed, savings
-		 * us from linking them at all).
-		 */
-		if (cmp(phn, ph->root) < 0) {
-			phn_lchild_set(phn, ph->root, offset);
-			phn_prev_set(ph->root, phn, offset);
-			ph->root = phn;
-			ph->auxcount = 0;
-			return;
-		}
-		ph->auxcount++;
-		phn_next_set(phn, phn_next_get(ph->root, offset), offset);
-		if (phn_next_get(ph->root, offset) != NULL) {
-			phn_prev_set(phn_next_get(ph->root, offset), phn,
-			    offset);
-		}
-		phn_prev_set(phn, ph->root, offset);
-		phn_next_set(ph->root, phn, offset);
 	}
-	if (ph->auxcount > 1) {
+
-		unsigned nmerges = ffs_zu(ph->auxcount - 1);
+	/*
-		bool done = false;
+	 * As a special case, check to see if we can replace the root.
-		for (unsigned i = 0; i < nmerges && !done; i++) {
+	 * This is practically common in some important cases, and lets
-			done = ph_try_aux_merge_pair(ph, offset, cmp);
+	 * us defer some insertions (hopefully, until the point where
-		}
+	 * some of the items in the aux list have been removed, savings
+	 * us from linking them at all).
+	 */
+	if (cmp(phn, ph->root) < 0) {
+		phn_lchild_set(phn, ph->root, offset);
+		phn_prev_set(ph->root, phn, offset);
+		ph->root = phn;
+		ph->auxcount = 0;
+		return;
+	}
+
+	phn_next_set(phn, phn_next_get(ph->root, offset), offset);
+	if (phn_next_get(ph->root, offset) != NULL) {
+		phn_prev_set(phn_next_get(ph->root, offset), phn, offset);
+	}
+	phn_prev_set(phn, ph->root, offset);
+	phn_next_set(ph->root, phn, offset);
+
+	ph->auxcount++;
+	unsigned nmerges = ffs_zu(ph->auxcount);
+	bool     done = false;
+	for (unsigned i = 0; i < nmerges && !done; i++) {
+		done = ph_try_aux_merge_pair(ph, offset, cmp);
 	}
 }
 
@@ -363,158 +385,210 @@ ph_remove_first(ph_t *ph, size_t offset, ph_cmp_t cmp) {
 	ph->root = ph_merge_children(ph->root, offset, cmp);
 
 	return ret;
-
 }
 
 JEMALLOC_ALWAYS_INLINE void
 ph_remove(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) {
-	void *replace;
-	void *parent;
-
 	if (ph->root == phn) {
-		/*
-		 * We can delete from aux list without merging it, but we need
-		 * to merge if we are dealing with the root node and it has
-		 * children.
-		 */
-		if (phn_lchild_get(phn, offset) == NULL) {
-			ph->root = phn_next_get(phn, offset);
-			if (ph->root != NULL) {
-				phn_prev_set(ph->root, NULL, offset);
-			}
-			return;
-		}
 		ph_merge_aux(ph, offset, cmp);
-		if (ph->root == phn) {
+		ph->root = ph_merge_children(phn, offset, cmp);
-			ph->root = ph_merge_children(ph->root, offset, cmp);
+		return;
-			return;
-		}
 	}
 
-	/* Get parent (if phn is leftmost child) before mutating. */
+	void *prev = phn_prev_get(phn, offset);
-	if ((parent = phn_prev_get(phn, offset)) != NULL) {
+	void *next = phn_next_get(phn, offset);
-		if (phn_lchild_get(parent, offset) != phn) {
+
-			parent = NULL;
+	/* If we have children, then we integrate them back in the heap. */
-		}
+	void *replace = ph_merge_children(phn, offset, cmp);
-	}
-	/* Find a possible replacement node, and link to parent. */
-	replace = ph_merge_children(phn, offset, cmp);
-	/* Set next/prev for sibling linked list. */
 	if (replace != NULL) {
-		if (parent != NULL) {
+		phn_next_set(replace, next, offset);
-			phn_prev_set(replace, parent, offset);
+		if (next != NULL) {
-			phn_lchild_set(parent, replace, offset);
+			phn_prev_set(next, replace, offset);
-		} else {
-			phn_prev_set(replace, phn_prev_get(phn, offset),
-			    offset);
-			if (phn_prev_get(phn, offset) != NULL) {
-				phn_next_set(phn_prev_get(phn, offset), replace,
-				    offset);
-			}
-		}
-		phn_next_set(replace, phn_next_get(phn, offset), offset);
-		if (phn_next_get(phn, offset) != NULL) {
-			phn_prev_set(phn_next_get(phn, offset), replace,
-			    offset);
 		}
+
+		next = replace;
+	}
+
+	if (next != NULL) {
+		phn_prev_set(next, prev, offset);
+	}
+
+	assert(prev != NULL);
+	if (phn_lchild_get(prev, offset) == phn) {
+		phn_lchild_set(prev, next, offset);
 	} else {
-		if (parent != NULL) {
+		phn_next_set(prev, next, offset);
-			void *next = phn_next_get(phn, offset);
-			phn_lchild_set(parent, next, offset);
-			if (next != NULL) {
-				phn_prev_set(next, parent, offset);
-			}
-		} else {
-			assert(phn_prev_get(phn, offset) != NULL);
-			phn_next_set(
-			    phn_prev_get(phn, offset),
-			    phn_next_get(phn, offset), offset);
-		}
-		if (phn_next_get(phn, offset) != NULL) {
-			phn_prev_set(
-			    phn_next_get(phn, offset),
-			    phn_prev_get(phn, offset), offset);
-		}
 	}
 }
 
-#define ph_structs(a_prefix, a_type)					\
+JEMALLOC_ALWAYS_INLINE void
-typedef struct {							\
+ph_enumerate_vars_init(ph_enumerate_vars_t *vars, uint16_t max_visit_num,
-	phn_link_t link;						\
+    uint16_t max_queue_size) {
-} a_prefix##_link_t;							\
+	vars->queue_size = 0;
-									\
+	vars->visited_num = 0;
-typedef struct {							\
+	vars->front = 0;
-	ph_t ph;							\
+	vars->rear = 0;
-} a_prefix##_t;
+	vars->max_visit_num = max_visit_num;
+	vars->max_queue_size = max_queue_size;
+	assert(vars->max_visit_num > 0);
+	/*
+	 * max_queue_size must be able to support max_visit_num, which means
+	 * the queue will not overflow before reaching max_visit_num.
+	 */
+	assert(vars->max_queue_size >= (vars->max_visit_num + 1) / 2);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+ph_enumerate_queue_push(
+    void *phn, void **bfs_queue, ph_enumerate_vars_t *vars) {
+	assert(vars->queue_size < vars->max_queue_size);
+	bfs_queue[vars->rear] = phn;
+	vars->rear = (vars->rear + 1) % vars->max_queue_size;
+	(vars->queue_size)++;
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_enumerate_queue_pop(void **bfs_queue, ph_enumerate_vars_t *vars) {
+	assert(vars->queue_size > 0);
+	assert(vars->queue_size <= vars->max_queue_size);
+	void *ret = bfs_queue[vars->front];
+	vars->front = (vars->front + 1) % vars->max_queue_size;
+	(vars->queue_size)--;
+	return ret;
+}
+
+/*
+ * The two functions below offer a solution to enumerate the pairing heap.
+ * Whe enumerating, always call ph_enumerate_prepare first to prepare the queue
+ * needed for BFS.  Next, call ph_enumerate_next to get the next element in
+ * the enumeration.  When enumeration ends, ph_enumerate_next returns NULL and
+ * should not be called again.  Enumeration ends when all elements in the heap
+ * has been enumerated or the number of visited elements exceed
+ * max_visit_num.
+ */
+JEMALLOC_ALWAYS_INLINE void
+ph_enumerate_prepare(ph_t *ph, void **bfs_queue, ph_enumerate_vars_t *vars,
+    uint16_t max_visit_num, uint16_t max_queue_size) {
+	ph_enumerate_vars_init(vars, max_visit_num, max_queue_size);
+	ph_enumerate_queue_push(ph->root, bfs_queue, vars);
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_enumerate_next(
+    ph_t *ph, size_t offset, void **bfs_queue, ph_enumerate_vars_t *vars) {
+	if (vars->queue_size == 0) {
+		return NULL;
+	}
+
+	(vars->visited_num)++;
+	if (vars->visited_num > vars->max_visit_num) {
+		return NULL;
+	}
+
+	void *ret = ph_enumerate_queue_pop(bfs_queue, vars);
+	assert(ret != NULL);
+	void *left = phn_lchild_get(ret, offset);
+	void *right = phn_next_get(ret, offset);
+	if (left) {
+		ph_enumerate_queue_push(left, bfs_queue, vars);
+	}
+	if (right) {
+		ph_enumerate_queue_push(right, bfs_queue, vars);
+	}
+	return ret;
+}
+
+#define ph_structs(a_prefix, a_type, a_max_queue_size)                         \
+	typedef struct {                                                       \
+		phn_link_t link;                                               \
+	} a_prefix##_link_t;                                                   \
+                                                                               \
+	typedef struct {                                                       \
+		ph_t ph;                                                       \
+	} a_prefix##_t;                                                        \
+                                                                               \
+	typedef struct {                                                       \
+		void               *bfs_queue[a_max_queue_size];               \
+		ph_enumerate_vars_t vars;                                      \
+	} a_prefix##_enumerate_helper_t;
 
 /*
  * The ph_proto() macro generates function prototypes that correspond to the
  * functions generated by an equivalently parameterized call to ph_gen().
  */
-#define ph_proto(a_attr, a_prefix, a_type)				\
+#define ph_proto(a_attr, a_prefix, a_type)                                     \
-									\
+                                                                               \
-a_attr void a_prefix##_new(a_prefix##_t *ph);				\
+	a_attr void    a_prefix##_new(a_prefix##_t *ph);                       \
-a_attr bool a_prefix##_empty(a_prefix##_t *ph);				\
+	a_attr bool    a_prefix##_empty(a_prefix##_t *ph);                     \
-a_attr a_type *a_prefix##_first(a_prefix##_t *ph);			\
+	a_attr a_type *a_prefix##_first(a_prefix##_t *ph);                     \
-a_attr a_type *a_prefix##_any(a_prefix##_t *ph);			\
+	a_attr a_type *a_prefix##_any(a_prefix##_t *ph);                       \
-a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn);		\
+	a_attr void    a_prefix##_insert(a_prefix##_t *ph, a_type *phn);       \
-a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph);		\
+	a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph);              \
-a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn);		\
+	a_attr void    a_prefix##_remove(a_prefix##_t *ph, a_type *phn);       \
-a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph);
+	a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph);                \
+	a_attr void    a_prefix##_enumerate_prepare(a_prefix##_t *ph,          \
+	       a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num,  \
+	       uint16_t max_queue_size);                                       \
+	a_attr a_type *a_prefix##_enumerate_next(                              \
+	    a_prefix##_t *ph, a_prefix##_enumerate_helper_t *helper);
 
 /* The ph_gen() macro generates a type-specific pairing heap implementation. */
-#define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp)		\
+#define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp)                       \
-JEMALLOC_ALWAYS_INLINE int						\
+	JEMALLOC_ALWAYS_INLINE int a_prefix##_ph_cmp(void *a, void *b) {       \
-a_prefix##_ph_cmp(void *a, void *b) {					\
+		return a_cmp((a_type *)a, (a_type *)b);                        \
-	return a_cmp((a_type *)a, (a_type *)b);				\
+	}                                                                      \
-}									\
+                                                                               \
-									\
+	a_attr void a_prefix##_new(a_prefix##_t *ph) {                         \
-a_attr void								\
+		ph_new(&ph->ph);                                               \
-a_prefix##_new(a_prefix##_t *ph) {					\
+	}                                                                      \
-	ph_new(&ph->ph);						\
+                                                                               \
-}									\
+	a_attr bool a_prefix##_empty(a_prefix##_t *ph) {                       \
-									\
+		return ph_empty(&ph->ph);                                      \
-a_attr bool								\
+	}                                                                      \
-a_prefix##_empty(a_prefix##_t *ph) {					\
+                                                                               \
-	return ph_empty(&ph->ph);					\
+	a_attr a_type *a_prefix##_first(a_prefix##_t *ph) {                    \
-}									\
+		return ph_first(                                               \
-									\
+		    &ph->ph, offsetof(a_type, a_field), &a_prefix##_ph_cmp);   \
-a_attr a_type *								\
+	}                                                                      \
-a_prefix##_first(a_prefix##_t *ph) {					\
+                                                                               \
-	return ph_first(&ph->ph, offsetof(a_type, a_field),		\
+	a_attr a_type *a_prefix##_any(a_prefix##_t *ph) {                      \
-	    &a_prefix##_ph_cmp);					\
+		return ph_any(&ph->ph, offsetof(a_type, a_field));             \
-}									\
+	}                                                                      \
-									\
+                                                                               \
-a_attr a_type *								\
+	a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn) {         \
-a_prefix##_any(a_prefix##_t *ph) {					\
+		ph_insert(&ph->ph, phn, offsetof(a_type, a_field),             \
-	return ph_any(&ph->ph, offsetof(a_type, a_field));		\
+		    a_prefix##_ph_cmp);                                        \
-}									\
+	}                                                                      \
-									\
+                                                                               \
-a_attr void								\
+	a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph) {             \
-a_prefix##_insert(a_prefix##_t *ph, a_type *phn) {			\
+		return ph_remove_first(                                        \
-	ph_insert(&ph->ph, phn, offsetof(a_type, a_field),		\
+		    &ph->ph, offsetof(a_type, a_field), a_prefix##_ph_cmp);    \
-	    a_prefix##_ph_cmp);						\
+	}                                                                      \
-}									\
+                                                                               \
-									\
+	a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn) {         \
-a_attr a_type *								\
+		ph_remove(&ph->ph, phn, offsetof(a_type, a_field),             \
-a_prefix##_remove_first(a_prefix##_t *ph) {				\
+		    a_prefix##_ph_cmp);                                        \
-	return ph_remove_first(&ph->ph, offsetof(a_type, a_field),	\
+	}                                                                      \
-	    a_prefix##_ph_cmp);						\
+                                                                               \
-}									\
+	a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph) {               \
-									\
+		a_type *ret = a_prefix##_any(ph);                              \
-a_attr void								\
+		if (ret != NULL) {                                             \
-a_prefix##_remove(a_prefix##_t *ph, a_type *phn) {			\
+			a_prefix##_remove(ph, ret);                            \
-	ph_remove(&ph->ph, phn, offsetof(a_type, a_field),		\
+		}                                                              \
-	    a_prefix##_ph_cmp);						\
+		return ret;                                                    \
-}									\
+	}                                                                      \
-									\
+                                                                               \
-a_attr a_type *								\
+	a_attr void a_prefix##_enumerate_prepare(a_prefix##_t *ph,             \
-a_prefix##_remove_any(a_prefix##_t *ph) {				\
+	    a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num,     \
-	a_type *ret = a_prefix##_any(ph);				\
+	    uint16_t max_queue_size) {                                         \
-	if (ret != NULL) {						\
+		ph_enumerate_prepare(&ph->ph, helper->bfs_queue,               \
-		a_prefix##_remove(ph, ret);				\
+		    &helper->vars, max_visit_num, max_queue_size);             \
-	}								\
+	}                                                                      \
-	return ret;							\
+                                                                               \
-}
+	a_attr a_type *a_prefix##_enumerate_next(                              \
+	    a_prefix##_t *ph, a_prefix##_enumerate_helper_t *helper) {         \
+		return ph_enumerate_next(&ph->ph, offsetof(a_type, a_field),   \
+		    helper->bfs_queue, &helper->vars);                         \
+	}
 
 #endif /* JEMALLOC_INTERNAL_PH_H */

dep/jemalloc/include/jemalloc/internal/private_namespace.h

@@ -6,17 +6,16 @@
 #define arena_migrate JEMALLOC_N(arena_migrate)
 #define arenas JEMALLOC_N(arenas)
 #define arena_set JEMALLOC_N(arena_set)
-#define arenas_lock JEMALLOC_N(arenas_lock)
 #define batch_alloc JEMALLOC_N(batch_alloc)
 #define bootstrap_calloc JEMALLOC_N(bootstrap_calloc)
 #define bootstrap_free JEMALLOC_N(bootstrap_free)
 #define bootstrap_malloc JEMALLOC_N(bootstrap_malloc)
 #define free_default JEMALLOC_N(free_default)
 #define iarena_cleanup JEMALLOC_N(iarena_cleanup)
+#define invalid_conf_abort JEMALLOC_N(invalid_conf_abort)
 #define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child)
 #define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent)
 #define jemalloc_prefork JEMALLOC_N(jemalloc_prefork)
-#define je_sdallocx_noflags JEMALLOC_N(je_sdallocx_noflags)
 #define junk_alloc_callback JEMALLOC_N(junk_alloc_callback)
 #define junk_free_callback JEMALLOC_N(junk_free_callback)
 #define malloc_default JEMALLOC_N(malloc_default)
@@ -29,14 +28,20 @@
 #define opt_abort JEMALLOC_N(opt_abort)
 #define opt_abort_conf JEMALLOC_N(opt_abort_conf)
 #define opt_cache_oblivious JEMALLOC_N(opt_cache_oblivious)
+#define opt_calloc_madvise_threshold JEMALLOC_N(opt_calloc_madvise_threshold)
 #define opt_confirm_conf JEMALLOC_N(opt_confirm_conf)
+#define opt_debug_double_free_max_scan JEMALLOC_N(opt_debug_double_free_max_scan)
+#define opt_disable_large_size_classes JEMALLOC_N(opt_disable_large_size_classes)
 #define opt_experimental_infallible_new JEMALLOC_N(opt_experimental_infallible_new)
+#define opt_experimental_tcache_gc JEMALLOC_N(opt_experimental_tcache_gc)
 #define opt_hpa JEMALLOC_N(opt_hpa)
 #define opt_hpa_opts JEMALLOC_N(opt_hpa_opts)
 #define opt_hpa_sec_opts JEMALLOC_N(opt_hpa_sec_opts)
 #define opt_junk JEMALLOC_N(opt_junk)
 #define opt_junk_alloc JEMALLOC_N(opt_junk_alloc)
 #define opt_junk_free JEMALLOC_N(opt_junk_free)
+#define opt_malloc_conf_env_var JEMALLOC_N(opt_malloc_conf_env_var)
+#define opt_malloc_conf_symlink JEMALLOC_N(opt_malloc_conf_symlink)
 #define opt_narenas JEMALLOC_N(opt_narenas)
 #define opt_narenas_ratio JEMALLOC_N(opt_narenas_ratio)
 #define opt_trust_madvise JEMALLOC_N(opt_trust_madvise)
@@ -48,15 +53,11 @@
 #define zero_realloc_count JEMALLOC_N(zero_realloc_count)
 #define zero_realloc_mode_names JEMALLOC_N(zero_realloc_mode_names)
 #define arena_basic_stats_merge JEMALLOC_N(arena_basic_stats_merge)
-#define arena_bin_choose JEMALLOC_N(arena_bin_choose)
 #define arena_binind_div_info JEMALLOC_N(arena_binind_div_info)
 #define arena_bin_offsets JEMALLOC_N(arena_bin_offsets)
 #define arena_boot JEMALLOC_N(arena_boot)
-#define arena_cache_bin_fill_small JEMALLOC_N(arena_cache_bin_fill_small)
 #define arena_choose_huge JEMALLOC_N(arena_choose_huge)
 #define arena_config_default JEMALLOC_N(arena_config_default)
-#define arena_dalloc_bin_locked_handle_newly_empty JEMALLOC_N(arena_dalloc_bin_locked_handle_newly_empty)
-#define arena_dalloc_bin_locked_handle_newly_nonempty JEMALLOC_N(arena_dalloc_bin_locked_handle_newly_nonempty)
 #define arena_dalloc_promoted JEMALLOC_N(arena_dalloc_promoted)
 #define arena_dalloc_small JEMALLOC_N(arena_dalloc_small)
 #define arena_decay JEMALLOC_N(arena_decay)
@@ -77,15 +78,15 @@
 #define arena_get_ehooks JEMALLOC_N(arena_get_ehooks)
 #define arena_handle_deferred_work JEMALLOC_N(arena_handle_deferred_work)
 #define arena_init_huge JEMALLOC_N(arena_init_huge)
-#define arena_is_huge JEMALLOC_N(arena_is_huge)
 #define arena_malloc_hard JEMALLOC_N(arena_malloc_hard)
 #define arena_muzzy_decay_ms_default_get JEMALLOC_N(arena_muzzy_decay_ms_default_get)
 #define arena_muzzy_decay_ms_default_set JEMALLOC_N(arena_muzzy_decay_ms_default_set)
+#define arena_name_get JEMALLOC_N(arena_name_get)
+#define arena_name_set JEMALLOC_N(arena_name_set)
 #define arena_new JEMALLOC_N(arena_new)
 #define arena_nthreads_dec JEMALLOC_N(arena_nthreads_dec)
 #define arena_nthreads_get JEMALLOC_N(arena_nthreads_get)
 #define arena_nthreads_inc JEMALLOC_N(arena_nthreads_inc)
-#define arena_pa_central_global JEMALLOC_N(arena_pa_central_global)
 #define arena_palloc JEMALLOC_N(arena_palloc)
 #define arena_postfork_child JEMALLOC_N(arena_postfork_child)
 #define arena_postfork_parent JEMALLOC_N(arena_postfork_parent)
@@ -99,6 +100,8 @@
 #define arena_prefork7 JEMALLOC_N(arena_prefork7)
 #define arena_prefork8 JEMALLOC_N(arena_prefork8)
 #define arena_prof_promote JEMALLOC_N(arena_prof_promote)
+#define arena_ptr_array_fill_small JEMALLOC_N(arena_ptr_array_fill_small)
+#define arena_ptr_array_flush JEMALLOC_N(arena_ptr_array_flush)
 #define arena_ralloc JEMALLOC_N(arena_ralloc)
 #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
 #define arena_reset JEMALLOC_N(arena_reset)
@@ -106,7 +109,10 @@
 #define arena_set_extent_hooks JEMALLOC_N(arena_set_extent_hooks)
 #define arena_slab_dalloc JEMALLOC_N(arena_slab_dalloc)
 #define arena_stats_merge JEMALLOC_N(arena_stats_merge)
+#define huge_arena_ind JEMALLOC_N(huge_arena_ind)
+#define huge_arena_pac_thp JEMALLOC_N(huge_arena_pac_thp)
 #define opt_dirty_decay_ms JEMALLOC_N(opt_dirty_decay_ms)
+#define opt_huge_arena_pac_thp JEMALLOC_N(opt_huge_arena_pac_thp)
 #define opt_muzzy_decay_ms JEMALLOC_N(opt_muzzy_decay_ms)
 #define opt_oversize_threshold JEMALLOC_N(opt_oversize_threshold)
 #define opt_percpu_arena JEMALLOC_N(opt_percpu_arena)
@@ -133,9 +139,12 @@
 #define opt_background_thread JEMALLOC_N(opt_background_thread)
 #define opt_max_background_threads JEMALLOC_N(opt_max_background_threads)
 #define pthread_create_wrapper JEMALLOC_N(pthread_create_wrapper)
+#define b0_alloc_tcache_stack JEMALLOC_N(b0_alloc_tcache_stack)
+#define b0_dalloc_tcache_stack JEMALLOC_N(b0_dalloc_tcache_stack)
 #define b0get JEMALLOC_N(b0get)
 #define base_alloc JEMALLOC_N(base_alloc)
 #define base_alloc_edata JEMALLOC_N(base_alloc_edata)
+#define base_alloc_rtree JEMALLOC_N(base_alloc_rtree)
 #define base_boot JEMALLOC_N(base_boot)
 #define base_delete JEMALLOC_N(base_delete)
 #define base_ehooks_get JEMALLOC_N(base_ehooks_get)
@@ -148,11 +157,28 @@
 #define base_stats_get JEMALLOC_N(base_stats_get)
 #define metadata_thp_mode_names JEMALLOC_N(metadata_thp_mode_names)
 #define opt_metadata_thp JEMALLOC_N(opt_metadata_thp)
+#define bin_choose JEMALLOC_N(bin_choose)
+#define bin_dalloc_locked_handle_newly_empty JEMALLOC_N(bin_dalloc_locked_handle_newly_empty)
+#define bin_dalloc_locked_handle_newly_nonempty JEMALLOC_N(bin_dalloc_locked_handle_newly_nonempty)
+#define bin_dalloc_slab_prepare JEMALLOC_N(bin_dalloc_slab_prepare)
+#define bin_dissociate_slab JEMALLOC_N(bin_dissociate_slab)
 #define bin_init JEMALLOC_N(bin_init)
+#define bin_lower_slab JEMALLOC_N(bin_lower_slab)
+#define bin_malloc_no_fresh_slab JEMALLOC_N(bin_malloc_no_fresh_slab)
+#define bin_malloc_with_fresh_slab JEMALLOC_N(bin_malloc_with_fresh_slab)
 #define bin_postfork_child JEMALLOC_N(bin_postfork_child)
 #define bin_postfork_parent JEMALLOC_N(bin_postfork_parent)
 #define bin_prefork JEMALLOC_N(bin_prefork)
+#define bin_refill_slabcur_no_fresh_slab JEMALLOC_N(bin_refill_slabcur_no_fresh_slab)
+#define bin_refill_slabcur_with_fresh_slab JEMALLOC_N(bin_refill_slabcur_with_fresh_slab)
 #define bin_shard_sizes_boot JEMALLOC_N(bin_shard_sizes_boot)
+#define bin_slab_reg_alloc JEMALLOC_N(bin_slab_reg_alloc)
+#define bin_slab_reg_alloc_batch JEMALLOC_N(bin_slab_reg_alloc_batch)
+#define bin_slabs_full_insert JEMALLOC_N(bin_slabs_full_insert)
+#define bin_slabs_full_remove JEMALLOC_N(bin_slabs_full_remove)
+#define bin_slabs_nonfull_insert JEMALLOC_N(bin_slabs_nonfull_insert)
+#define bin_slabs_nonfull_remove JEMALLOC_N(bin_slabs_nonfull_remove)
+#define bin_slabs_nonfull_tryget JEMALLOC_N(bin_slabs_nonfull_tryget)
 #define bin_update_shard_size JEMALLOC_N(bin_update_shard_size)
 #define bin_info_boot JEMALLOC_N(bin_info_boot)
 #define bin_infos JEMALLOC_N(bin_infos)
@@ -167,9 +193,11 @@
 #define cache_bin_info_compute_alloc JEMALLOC_N(cache_bin_info_compute_alloc)
 #define cache_bin_info_init JEMALLOC_N(cache_bin_info_init)
 #define cache_bin_init JEMALLOC_N(cache_bin_init)
+#define cache_bin_init_disabled JEMALLOC_N(cache_bin_init_disabled)
 #define cache_bin_postincrement JEMALLOC_N(cache_bin_postincrement)
 #define cache_bin_preincrement JEMALLOC_N(cache_bin_preincrement)
-#define cache_bin_still_zero_initialized JEMALLOC_N(cache_bin_still_zero_initialized)
+#define cache_bin_stack_use_thp JEMALLOC_N(cache_bin_stack_use_thp)
+#define disabled_bin JEMALLOC_N(disabled_bin)
 #define ckh_count JEMALLOC_N(ckh_count)
 #define ckh_delete JEMALLOC_N(ckh_delete)
 #define ckh_insert JEMALLOC_N(ckh_insert)
@@ -195,7 +223,6 @@
 #define ctl_postfork_child JEMALLOC_N(ctl_postfork_child)
 #define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent)
 #define ctl_prefork JEMALLOC_N(ctl_prefork)
-#define decay_deadline_init JEMALLOC_N(decay_deadline_init)
 #define decay_init JEMALLOC_N(decay_init)
 #define decay_maybe_advance_epoch JEMALLOC_N(decay_maybe_advance_epoch)
 #define decay_ms_valid JEMALLOC_N(decay_ms_valid)
@@ -209,6 +236,8 @@
 #define ecache_prefork JEMALLOC_N(ecache_prefork)
 #define edata_avail_any JEMALLOC_N(edata_avail_any)
 #define edata_avail_empty JEMALLOC_N(edata_avail_empty)
+#define edata_avail_enumerate_next JEMALLOC_N(edata_avail_enumerate_next)
+#define edata_avail_enumerate_prepare JEMALLOC_N(edata_avail_enumerate_prepare)
 #define edata_avail_first JEMALLOC_N(edata_avail_first)
 #define edata_avail_insert JEMALLOC_N(edata_avail_insert)
 #define edata_avail_new JEMALLOC_N(edata_avail_new)
@@ -217,6 +246,8 @@
 #define edata_avail_remove_first JEMALLOC_N(edata_avail_remove_first)
 #define edata_heap_any JEMALLOC_N(edata_heap_any)
 #define edata_heap_empty JEMALLOC_N(edata_heap_empty)
+#define edata_heap_enumerate_next JEMALLOC_N(edata_heap_enumerate_next)
+#define edata_heap_enumerate_prepare JEMALLOC_N(edata_heap_enumerate_prepare)
 #define edata_heap_first JEMALLOC_N(edata_heap_first)
 #define edata_heap_insert JEMALLOC_N(edata_heap_insert)
 #define edata_heap_new JEMALLOC_N(edata_heap_new)
@@ -278,11 +309,10 @@
 #define ecache_evict JEMALLOC_N(ecache_evict)
 #define extent_alloc_wrapper JEMALLOC_N(extent_alloc_wrapper)
 #define extent_boot JEMALLOC_N(extent_boot)
-#define extent_commit_wrapper JEMALLOC_N(extent_commit_wrapper)
 #define extent_commit_zero JEMALLOC_N(extent_commit_zero)
 #define extent_dalloc_gap JEMALLOC_N(extent_dalloc_gap)
 #define extent_dalloc_wrapper JEMALLOC_N(extent_dalloc_wrapper)
-#define extent_decommit_wrapper JEMALLOC_N(extent_decommit_wrapper)
+#define extent_dalloc_wrapper_purged JEMALLOC_N(extent_dalloc_wrapper_purged)
 #define extent_destroy_wrapper JEMALLOC_N(extent_destroy_wrapper)
 #define extent_gdump_add JEMALLOC_N(extent_gdump_add)
 #define extent_merge_wrapper JEMALLOC_N(extent_merge_wrapper)
@@ -292,6 +322,7 @@
 #define extent_sn_next JEMALLOC_N(extent_sn_next)
 #define extent_split_wrapper JEMALLOC_N(extent_split_wrapper)
 #define opt_lg_extent_max_active_fit JEMALLOC_N(opt_lg_extent_max_active_fit)
+#define opt_process_madvise_max_batch JEMALLOC_N(opt_process_madvise_max_batch)
 #define dss_prec_names JEMALLOC_N(dss_prec_names)
 #define extent_alloc_dss JEMALLOC_N(extent_alloc_dss)
 #define extent_dss_boot JEMALLOC_N(extent_dss_boot)
@@ -322,23 +353,32 @@
 #define hook_invoke_dalloc JEMALLOC_N(hook_invoke_dalloc)
 #define hook_invoke_expand JEMALLOC_N(hook_invoke_expand)
 #define hook_remove JEMALLOC_N(hook_remove)
-#define hpa_central_extract JEMALLOC_N(hpa_central_extract)
+#define hpa_hugepage_size_exceeds_limit JEMALLOC_N(hpa_hugepage_size_exceeds_limit)
-#define hpa_central_init JEMALLOC_N(hpa_central_init)
+#define hpa_hugify_style_names JEMALLOC_N(hpa_hugify_style_names)
 #define hpa_shard_destroy JEMALLOC_N(hpa_shard_destroy)
 #define hpa_shard_disable JEMALLOC_N(hpa_shard_disable)
 #define hpa_shard_do_deferred_work JEMALLOC_N(hpa_shard_do_deferred_work)
+#define hpa_shard_flush JEMALLOC_N(hpa_shard_flush)
 #define hpa_shard_init JEMALLOC_N(hpa_shard_init)
 #define hpa_shard_postfork_child JEMALLOC_N(hpa_shard_postfork_child)
 #define hpa_shard_postfork_parent JEMALLOC_N(hpa_shard_postfork_parent)
+#define hpa_shard_prefork2 JEMALLOC_N(hpa_shard_prefork2)
 #define hpa_shard_prefork3 JEMALLOC_N(hpa_shard_prefork3)
 #define hpa_shard_prefork4 JEMALLOC_N(hpa_shard_prefork4)
 #define hpa_shard_set_deferral_allowed JEMALLOC_N(hpa_shard_set_deferral_allowed)
 #define hpa_shard_stats_accum JEMALLOC_N(hpa_shard_stats_accum)
 #define hpa_shard_stats_merge JEMALLOC_N(hpa_shard_stats_merge)
 #define hpa_supported JEMALLOC_N(hpa_supported)
+#define opt_experimental_hpa_enforce_hugify JEMALLOC_N(opt_experimental_hpa_enforce_hugify)
+#define opt_experimental_hpa_start_huge_if_thp_always JEMALLOC_N(opt_experimental_hpa_start_huge_if_thp_always)
+#define hpa_central_extract JEMALLOC_N(hpa_central_extract)
+#define hpa_central_init JEMALLOC_N(hpa_central_init)
 #define hpa_hooks_default JEMALLOC_N(hpa_hooks_default)
+#define hpa_purge_batch JEMALLOC_N(hpa_purge_batch)
 #define hpdata_age_heap_any JEMALLOC_N(hpdata_age_heap_any)
 #define hpdata_age_heap_empty JEMALLOC_N(hpdata_age_heap_empty)
+#define hpdata_age_heap_enumerate_next JEMALLOC_N(hpdata_age_heap_enumerate_next)
+#define hpdata_age_heap_enumerate_prepare JEMALLOC_N(hpdata_age_heap_enumerate_prepare)
 #define hpdata_age_heap_first JEMALLOC_N(hpdata_age_heap_first)
 #define hpdata_age_heap_insert JEMALLOC_N(hpdata_age_heap_insert)
 #define hpdata_age_heap_new JEMALLOC_N(hpdata_age_heap_new)
@@ -365,19 +405,23 @@
 #define large_prof_tctx_reset JEMALLOC_N(large_prof_tctx_reset)
 #define large_ralloc JEMALLOC_N(large_ralloc)
 #define large_ralloc_no_move JEMALLOC_N(large_ralloc_no_move)
-#define large_salloc JEMALLOC_N(large_salloc)
 #define log_init_done JEMALLOC_N(log_init_done)
 #define log_var_names JEMALLOC_N(log_var_names)
 #define log_var_update_state JEMALLOC_N(log_var_update_state)
 #define buferror JEMALLOC_N(buferror)
 #define malloc_cprintf JEMALLOC_N(malloc_cprintf)
 #define malloc_printf JEMALLOC_N(malloc_printf)
+#define malloc_read_fd JEMALLOC_N(malloc_read_fd)
 #define malloc_snprintf JEMALLOC_N(malloc_snprintf)
 #define malloc_strtoumax JEMALLOC_N(malloc_strtoumax)
 #define malloc_vcprintf JEMALLOC_N(malloc_vcprintf)
 #define malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf)
 #define malloc_write JEMALLOC_N(malloc_write)
+#define malloc_write_fd JEMALLOC_N(malloc_write_fd)
 #define wrtmessage JEMALLOC_N(wrtmessage)
+#define had_conf_error JEMALLOC_N(had_conf_error)
+#define malloc_abort_invalid_conf JEMALLOC_N(malloc_abort_invalid_conf)
+#define malloc_conf_init JEMALLOC_N(malloc_conf_init)
 #define malloc_mutex_boot JEMALLOC_N(malloc_mutex_boot)
 #define malloc_mutex_init JEMALLOC_N(malloc_mutex_init)
 #define malloc_mutex_lock_slow JEMALLOC_N(malloc_mutex_lock_slow)
@@ -398,8 +442,11 @@
 #define nstime_init_update JEMALLOC_N(nstime_init_update)
 #define nstime_isubtract JEMALLOC_N(nstime_isubtract)
 #define nstime_monotonic JEMALLOC_N(nstime_monotonic)
-#define nstime_msec JEMALLOC_N(nstime_msec)
+#define nstime_ms JEMALLOC_N(nstime_ms)
+#define nstime_ms_between JEMALLOC_N(nstime_ms_between)
+#define nstime_ms_since JEMALLOC_N(nstime_ms_since)
 #define nstime_ns JEMALLOC_N(nstime_ns)
+#define nstime_ns_between JEMALLOC_N(nstime_ns_between)
 #define nstime_nsec JEMALLOC_N(nstime_nsec)
 #define nstime_ns_since JEMALLOC_N(nstime_ns_since)
 #define nstime_prof_init_update JEMALLOC_N(nstime_prof_init_update)
@@ -419,14 +466,17 @@
 #define pa_shard_disable_hpa JEMALLOC_N(pa_shard_disable_hpa)
 #define pa_shard_do_deferred_work JEMALLOC_N(pa_shard_do_deferred_work)
 #define pa_shard_enable_hpa JEMALLOC_N(pa_shard_enable_hpa)
+#define pa_shard_flush JEMALLOC_N(pa_shard_flush)
 #define pa_shard_init JEMALLOC_N(pa_shard_init)
 #define pa_shard_reset JEMALLOC_N(pa_shard_reset)
-#define pa_shard_retain_grow_limit_get_set JEMALLOC_N(pa_shard_retain_grow_limit_get_set)
 #define pa_shard_set_deferral_allowed JEMALLOC_N(pa_shard_set_deferral_allowed)
 #define pa_shard_time_until_deferred_work JEMALLOC_N(pa_shard_time_until_deferred_work)
 #define pa_shrink JEMALLOC_N(pa_shrink)
 #define pa_shard_basic_stats_merge JEMALLOC_N(pa_shard_basic_stats_merge)
 #define pa_shard_mtx_stats_read JEMALLOC_N(pa_shard_mtx_stats_read)
+#define pa_shard_nactive JEMALLOC_N(pa_shard_nactive)
+#define pa_shard_ndirty JEMALLOC_N(pa_shard_ndirty)
+#define pa_shard_nmuzzy JEMALLOC_N(pa_shard_nmuzzy)
 #define pa_shard_postfork_child JEMALLOC_N(pa_shard_postfork_child)
 #define pa_shard_postfork_parent JEMALLOC_N(pa_shard_postfork_parent)
 #define pa_shard_prefork0 JEMALLOC_N(pa_shard_prefork0)
@@ -435,8 +485,6 @@
 #define pa_shard_prefork4 JEMALLOC_N(pa_shard_prefork4)
 #define pa_shard_prefork5 JEMALLOC_N(pa_shard_prefork5)
 #define pa_shard_stats_merge JEMALLOC_N(pa_shard_stats_merge)
-#define pai_alloc_batch_default JEMALLOC_N(pai_alloc_batch_default)
-#define pai_dalloc_batch_default JEMALLOC_N(pai_dalloc_batch_default)
 #define pac_decay_all JEMALLOC_N(pac_decay_all)
 #define pac_decay_ms_get JEMALLOC_N(pac_decay_ms_get)
 #define pac_decay_ms_set JEMALLOC_N(pac_decay_ms_set)
@@ -447,7 +495,9 @@
 #define pac_retain_grow_limit_get_set JEMALLOC_N(pac_retain_grow_limit_get_set)
 #define init_system_thp_mode JEMALLOC_N(init_system_thp_mode)
 #define opt_thp JEMALLOC_N(opt_thp)
+#define os_page JEMALLOC_N(os_page)
 #define pages_boot JEMALLOC_N(pages_boot)
+#define pages_collapse JEMALLOC_N(pages_collapse)
 #define pages_commit JEMALLOC_N(pages_commit)
 #define pages_decommit JEMALLOC_N(pages_decommit)
 #define pages_dodump JEMALLOC_N(pages_dodump)
@@ -458,29 +508,28 @@
 #define pages_nohuge JEMALLOC_N(pages_nohuge)
 #define pages_purge_forced JEMALLOC_N(pages_purge_forced)
 #define pages_purge_lazy JEMALLOC_N(pages_purge_lazy)
+#define pages_purge_process_madvise JEMALLOC_N(pages_purge_process_madvise)
 #define pages_set_thp_state JEMALLOC_N(pages_set_thp_state)
 #define pages_unmap JEMALLOC_N(pages_unmap)
 #define pages_unmark_guards JEMALLOC_N(pages_unmark_guards)
+#define system_thp_mode_names JEMALLOC_N(system_thp_mode_names)
 #define thp_mode_names JEMALLOC_N(thp_mode_names)
-#define peak_alloc_event_handler JEMALLOC_N(peak_alloc_event_handler)
-#define peak_alloc_new_event_wait JEMALLOC_N(peak_alloc_new_event_wait)
-#define peak_alloc_postponed_event_wait JEMALLOC_N(peak_alloc_postponed_event_wait)
-#define peak_dalloc_event_handler JEMALLOC_N(peak_dalloc_event_handler)
-#define peak_dalloc_new_event_wait JEMALLOC_N(peak_dalloc_new_event_wait)
-#define peak_dalloc_postponed_event_wait JEMALLOC_N(peak_dalloc_postponed_event_wait)
 #define peak_event_max JEMALLOC_N(peak_event_max)
 #define peak_event_update JEMALLOC_N(peak_event_update)
 #define peak_event_zero JEMALLOC_N(peak_event_zero)
+#define peak_te_handler JEMALLOC_N(peak_te_handler)
 #define lg_prof_sample JEMALLOC_N(lg_prof_sample)
 #define opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval)
 #define opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample)
 #define opt_prof JEMALLOC_N(opt_prof)
 #define opt_prof_accum JEMALLOC_N(opt_prof_accum)
 #define opt_prof_active JEMALLOC_N(opt_prof_active)
+#define opt_prof_bt_max JEMALLOC_N(opt_prof_bt_max)
 #define opt_prof_final JEMALLOC_N(opt_prof_final)
 #define opt_prof_gdump JEMALLOC_N(opt_prof_gdump)
 #define opt_prof_leak JEMALLOC_N(opt_prof_leak)
 #define opt_prof_leak_error JEMALLOC_N(opt_prof_leak_error)
+#define opt_prof_pid_namespace JEMALLOC_N(opt_prof_pid_namespace)
 #define opt_prof_prefix JEMALLOC_N(opt_prof_prefix)
 #define opt_prof_sys_thread_name JEMALLOC_N(opt_prof_sys_thread_name)
 #define opt_prof_thread_active_init JEMALLOC_N(opt_prof_thread_active_init)
@@ -489,14 +538,12 @@
 #define prof_active_set JEMALLOC_N(prof_active_set)
 #define prof_active_state JEMALLOC_N(prof_active_state)
 #define prof_alloc_rollback JEMALLOC_N(prof_alloc_rollback)
-#define prof_backtrace_hook JEMALLOC_N(prof_backtrace_hook)
 #define prof_backtrace_hook_get JEMALLOC_N(prof_backtrace_hook_get)
 #define prof_backtrace_hook_set JEMALLOC_N(prof_backtrace_hook_set)
 #define prof_boot0 JEMALLOC_N(prof_boot0)
 #define prof_boot1 JEMALLOC_N(prof_boot1)
 #define prof_boot2 JEMALLOC_N(prof_boot2)
 #define prof_booted JEMALLOC_N(prof_booted)
-#define prof_dump_hook JEMALLOC_N(prof_dump_hook)
 #define prof_dump_hook_get JEMALLOC_N(prof_dump_hook_get)
 #define prof_dump_hook_set JEMALLOC_N(prof_dump_hook_set)
 #define prof_free_sampled_object JEMALLOC_N(prof_free_sampled_object)
@@ -513,8 +560,12 @@
 #define prof_prefork0 JEMALLOC_N(prof_prefork0)
 #define prof_prefork1 JEMALLOC_N(prof_prefork1)
 #define prof_sample_event_handler JEMALLOC_N(prof_sample_event_handler)
+#define prof_sample_free_hook_get JEMALLOC_N(prof_sample_free_hook_get)
+#define prof_sample_free_hook_set JEMALLOC_N(prof_sample_free_hook_set)
+#define prof_sample_hook_get JEMALLOC_N(prof_sample_hook_get)
+#define prof_sample_hook_set JEMALLOC_N(prof_sample_hook_set)
 #define prof_sample_new_event_wait JEMALLOC_N(prof_sample_new_event_wait)
-#define prof_sample_postponed_event_wait JEMALLOC_N(prof_sample_postponed_event_wait)
+#define prof_sample_te_handler JEMALLOC_N(prof_sample_te_handler)
 #define prof_tctx_create JEMALLOC_N(prof_tctx_create)
 #define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup)
 #define prof_tdata_init JEMALLOC_N(prof_tdata_init)
@@ -525,6 +576,7 @@
 #define prof_thread_active_set JEMALLOC_N(prof_thread_active_set)
 #define prof_thread_name_get JEMALLOC_N(prof_thread_name_get)
 #define prof_thread_name_set JEMALLOC_N(prof_thread_name_set)
+#define tsd_prof_sample_event_wait_get JEMALLOC_N(tsd_prof_sample_event_wait_get)
 #define bt2gctx_mtx JEMALLOC_N(bt2gctx_mtx)
 #define gctx_locks JEMALLOC_N(gctx_locks)
 #define prof_bt_count JEMALLOC_N(prof_bt_count)
@@ -541,7 +593,6 @@
 #define prof_tdata_count JEMALLOC_N(prof_tdata_count)
 #define prof_tdata_detach JEMALLOC_N(prof_tdata_detach)
 #define prof_tdata_init_impl JEMALLOC_N(prof_tdata_init_impl)
-#define prof_thread_name_alloc JEMALLOC_N(prof_thread_name_alloc)
 #define prof_thread_name_set_impl JEMALLOC_N(prof_thread_name_set_impl)
 #define prof_unbiased_sz JEMALLOC_N(prof_unbiased_sz)
 #define prof_unbias_map_init JEMALLOC_N(prof_unbias_map_init)
@@ -552,7 +603,6 @@
 #define prof_log_alloc_count JEMALLOC_N(prof_log_alloc_count)
 #define prof_log_bt_count JEMALLOC_N(prof_log_bt_count)
 #define prof_log_dummy_set JEMALLOC_N(prof_log_dummy_set)
-#define prof_logging_state JEMALLOC_N(prof_logging_state)
 #define prof_log_init JEMALLOC_N(prof_log_init)
 #define prof_log_is_logging JEMALLOC_N(prof_log_is_logging)
 #define prof_log_rep_check JEMALLOC_N(prof_log_rep_check)
@@ -574,6 +624,7 @@
 #define prof_recent_alloc_reset JEMALLOC_N(prof_recent_alloc_reset)
 #define prof_recent_dump_mtx JEMALLOC_N(prof_recent_dump_mtx)
 #define prof_recent_init JEMALLOC_N(prof_recent_init)
+#define prof_thread_stack_range JEMALLOC_N(prof_thread_stack_range)
 #define opt_prof_stats JEMALLOC_N(opt_prof_stats)
 #define prof_stats_dec JEMALLOC_N(prof_stats_dec)
 #define prof_stats_get_accum JEMALLOC_N(prof_stats_get_accum)
@@ -583,7 +634,6 @@
 #define bt_init JEMALLOC_N(bt_init)
 #define prof_backtrace JEMALLOC_N(prof_backtrace)
 #define prof_base JEMALLOC_N(prof_base)
-#define prof_do_mock JEMALLOC_N(prof_do_mock)
 #define prof_dump_filename_mtx JEMALLOC_N(prof_dump_filename_mtx)
 #define prof_dump_open_file JEMALLOC_N(prof_dump_open_file)
 #define prof_dump_open_maps JEMALLOC_N(prof_dump_open_maps)
@@ -618,7 +668,9 @@
 #define sc_boot JEMALLOC_N(sc_boot)
 #define sc_data_init JEMALLOC_N(sc_data_init)
 #define sc_data_update_slab_size JEMALLOC_N(sc_data_update_slab_size)
-#define sec_disable JEMALLOC_N(sec_disable)
+#define sec_alloc JEMALLOC_N(sec_alloc)
+#define sec_dalloc JEMALLOC_N(sec_dalloc)
+#define sec_fill JEMALLOC_N(sec_fill)
 #define sec_flush JEMALLOC_N(sec_flush)
 #define sec_init JEMALLOC_N(sec_init)
 #define sec_mutex_stats_read JEMALLOC_N(sec_mutex_stats_read)
@@ -626,16 +678,16 @@
 #define sec_postfork_parent JEMALLOC_N(sec_postfork_parent)
 #define sec_prefork2 JEMALLOC_N(sec_prefork2)
 #define sec_stats_merge JEMALLOC_N(sec_stats_merge)
-#define arena_mutex_names JEMALLOC_N(arena_mutex_names)
-#define global_mutex_names JEMALLOC_N(global_mutex_names)
 #define opt_stats_interval JEMALLOC_N(opt_stats_interval)
 #define opt_stats_interval_opts JEMALLOC_N(opt_stats_interval_opts)
 #define opt_stats_print JEMALLOC_N(opt_stats_print)
 #define opt_stats_print_opts JEMALLOC_N(opt_stats_print_opts)
 #define stats_boot JEMALLOC_N(stats_boot)
+#define stats_interval_accum_batch JEMALLOC_N(stats_interval_accum_batch)
 #define stats_interval_event_handler JEMALLOC_N(stats_interval_event_handler)
 #define stats_interval_new_event_wait JEMALLOC_N(stats_interval_new_event_wait)
 #define stats_interval_postponed_event_wait JEMALLOC_N(stats_interval_postponed_event_wait)
+#define stats_interval_te_handler JEMALLOC_N(stats_interval_te_handler)
 #define stats_postfork_child JEMALLOC_N(stats_postfork_child)
 #define stats_postfork_parent JEMALLOC_N(stats_postfork_parent)
 #define stats_prefork JEMALLOC_N(stats_prefork)
@@ -647,7 +699,8 @@
 #define sz_psz_quantize_ceil JEMALLOC_N(sz_psz_quantize_ceil)
 #define sz_psz_quantize_floor JEMALLOC_N(sz_psz_quantize_floor)
 #define sz_size2index_tab JEMALLOC_N(sz_size2index_tab)
-#define nhbins JEMALLOC_N(nhbins)
+#define global_do_not_change_tcache_maxclass JEMALLOC_N(global_do_not_change_tcache_maxclass)
+#define global_do_not_change_tcache_nbins JEMALLOC_N(global_do_not_change_tcache_nbins)
 #define opt_lg_tcache_flush_large_div JEMALLOC_N(opt_lg_tcache_flush_large_div)
 #define opt_lg_tcache_flush_small_div JEMALLOC_N(opt_lg_tcache_flush_small_div)
 #define opt_lg_tcache_nslots_mul JEMALLOC_N(opt_lg_tcache_nslots_mul)
@@ -665,18 +718,17 @@
 #define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large)
 #define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small)
 #define tcache_bin_flush_stashed JEMALLOC_N(tcache_bin_flush_stashed)
-#define tcache_bin_info JEMALLOC_N(tcache_bin_info)
+#define tcache_bin_info_default_init JEMALLOC_N(tcache_bin_info_default_init)
+#define tcache_bin_ncached_max_read JEMALLOC_N(tcache_bin_ncached_max_read)
+#define tcache_bins_ncached_max_write JEMALLOC_N(tcache_bins_ncached_max_write)
 #define tcache_boot JEMALLOC_N(tcache_boot)
 #define tcache_cleanup JEMALLOC_N(tcache_cleanup)
 #define tcache_create_explicit JEMALLOC_N(tcache_create_explicit)
+#define tcache_enabled_set JEMALLOC_N(tcache_enabled_set)
 #define tcache_flush JEMALLOC_N(tcache_flush)
-#define tcache_gc_dalloc_event_handler JEMALLOC_N(tcache_gc_dalloc_event_handler)
-#define tcache_gc_dalloc_new_event_wait JEMALLOC_N(tcache_gc_dalloc_new_event_wait)
-#define tcache_gc_dalloc_postponed_event_wait JEMALLOC_N(tcache_gc_dalloc_postponed_event_wait)
-#define tcache_gc_event_handler JEMALLOC_N(tcache_gc_event_handler)
 #define tcache_gc_new_event_wait JEMALLOC_N(tcache_gc_new_event_wait)
 #define tcache_gc_postponed_event_wait JEMALLOC_N(tcache_gc_postponed_event_wait)
-#define tcache_maxclass JEMALLOC_N(tcache_maxclass)
+#define tcache_gc_te_handler JEMALLOC_N(tcache_gc_te_handler)
 #define tcache_postfork_child JEMALLOC_N(tcache_postfork_child)
 #define tcache_postfork_parent JEMALLOC_N(tcache_postfork_parent)
 #define tcache_prefork JEMALLOC_N(tcache_prefork)
@@ -685,15 +737,22 @@
 #define tcaches_create JEMALLOC_N(tcaches_create)
 #define tcaches_destroy JEMALLOC_N(tcaches_destroy)
 #define tcaches_flush JEMALLOC_N(tcaches_flush)
+#define tcache_stack_alloc JEMALLOC_N(tcache_stack_alloc)
 #define tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
-#define tsd_tcache_data_init JEMALLOC_N(tsd_tcache_data_init)
+#define thread_tcache_max_set JEMALLOC_N(thread_tcache_max_set)
 #define tsd_tcache_enabled_data_init JEMALLOC_N(tsd_tcache_enabled_data_init)
 #define test_hooks_arena_new_hook JEMALLOC_N(test_hooks_arena_new_hook)
 #define test_hooks_libc_hook JEMALLOC_N(test_hooks_libc_hook)
+#define te_adjust_thresholds_helper JEMALLOC_N(te_adjust_thresholds_helper)
 #define te_assert_invariants_debug JEMALLOC_N(te_assert_invariants_debug)
 #define te_event_trigger JEMALLOC_N(te_event_trigger)
 #define te_recompute_fast_threshold JEMALLOC_N(te_recompute_fast_threshold)
 #define tsd_te_init JEMALLOC_N(tsd_te_init)
+#define experimental_thread_events_boot JEMALLOC_N(experimental_thread_events_boot)
+#define te_alloc_handlers JEMALLOC_N(te_alloc_handlers)
+#define te_dalloc_handlers JEMALLOC_N(te_dalloc_handlers)
+#define te_register_user_handler JEMALLOC_N(te_register_user_handler)
+#define te_user_event_enabled JEMALLOC_N(te_user_event_enabled)
 #define ticker_geom_table JEMALLOC_N(ticker_geom_table)
 #define malloc_tsd_boot0 JEMALLOC_N(malloc_tsd_boot0)
 #define malloc_tsd_boot1 JEMALLOC_N(malloc_tsd_boot1)
@@ -712,6 +771,7 @@
 #define tsd_state_set JEMALLOC_N(tsd_state_set)
 #define tsd_tls JEMALLOC_N(tsd_tls)
 #define tsd_tsd JEMALLOC_N(tsd_tsd)
+#define multi_setting_parse_next JEMALLOC_N(multi_setting_parse_next)
 #define witness_depth_error JEMALLOC_N(witness_depth_error)
 #define witnesses_cleanup JEMALLOC_N(witnesses_cleanup)
 #define witness_init JEMALLOC_N(witness_init)

dep/jemalloc/include/jemalloc/internal/prng.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_PRNG_H
 #define JEMALLOC_INTERNAL_PRNG_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bit_util.h"
 
 /*
@@ -25,11 +26,11 @@
 /******************************************************************************/
 /* INTERNAL DEFINITIONS -- IGNORE */
 /******************************************************************************/
-#define PRNG_A_32	UINT32_C(1103515241)
+#define PRNG_A_32 UINT32_C(1103515241)
-#define PRNG_C_32	UINT32_C(12347)
+#define PRNG_C_32 UINT32_C(12347)
 
-#define PRNG_A_64	UINT64_C(6364136223846793005)
+#define PRNG_A_64 UINT64_C(6364136223846793005)
-#define PRNG_C_64	UINT64_C(1442695040888963407)
+#define PRNG_C_64 UINT64_C(1442695040888963407)
 
 JEMALLOC_ALWAYS_INLINE uint32_t
 prng_state_next_u32(uint32_t state) {
@@ -48,7 +49,7 @@ prng_state_next_zu(size_t state) {
 #elif LG_SIZEOF_PTR == 3
 	return (state * PRNG_A_64) + PRNG_C_64;
 #else
-#error Unsupported pointer size
+#	error Unsupported pointer size
 #endif
 }
 

dep/jemalloc/include/jemalloc/internal/prof_data.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_PROF_DATA_H
 #define JEMALLOC_INTERNAL_PROF_DATA_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/mutex.h"
 
 extern malloc_mutex_t bt2gctx_mtx;
@@ -16,22 +17,21 @@ extern size_t prof_shifted_unbiased_cnt[PROF_SC_NSIZES];
 void prof_bt_hash(const void *key, size_t r_hash[2]);
 bool prof_bt_keycomp(const void *k1, const void *k2);
 
-bool prof_data_init(tsd_t *tsd);
+bool         prof_data_init(tsd_t *tsd);
 prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
-char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name);
+int          prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name);
-int prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name);
+void         prof_unbias_map_init(void);
-void prof_unbias_map_init();
 void prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
     prof_tdata_t *tdata, bool leakcheck);
-prof_tdata_t * prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid,
+prof_tdata_t *prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid,
     uint64_t thr_discrim, char *thread_name, bool active);
-void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
+void          prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
-void prof_reset(tsd_t *tsd, size_t lg_sample);
+void          prof_reset(tsd_t *tsd, size_t lg_sample);
-void prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
+void          prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
 
 /* Used in unit tests. */
 size_t prof_tdata_count(void);
 size_t prof_bt_count(void);
-void prof_cnt_all(prof_cnt_t *cnt_all);
+void   prof_cnt_all(prof_cnt_t *cnt_all);
 
 #endif /* JEMALLOC_INTERNAL_PROF_DATA_H */

dep/jemalloc/include/jemalloc/internal/prof_externs.h

@@ -1,28 +1,35 @@
 #ifndef JEMALLOC_INTERNAL_PROF_EXTERNS_H
 #define JEMALLOC_INTERNAL_PROF_EXTERNS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/prof_hook.h"
+#include "jemalloc/internal/thread_event_registry.h"
 
-extern bool opt_prof;
+extern bool     opt_prof;
-extern bool opt_prof_active;
+extern bool     opt_prof_active;
-extern bool opt_prof_thread_active_init;
+extern bool     opt_prof_thread_active_init;
-extern size_t opt_lg_prof_sample;    /* Mean bytes between samples. */
+extern unsigned opt_prof_bt_max;
-extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
+extern size_t   opt_lg_prof_sample; /* Mean bytes between samples. */
-extern bool opt_prof_gdump;          /* High-water memory dumping. */
+extern ssize_t opt_lg_prof_interval;    /* lg(prof_interval). */
-extern bool opt_prof_final;          /* Final profile dumping. */
+extern bool    opt_prof_gdump;          /* High-water memory dumping. */
-extern bool opt_prof_leak;           /* Dump leak summary at exit. */
+extern bool    opt_prof_final;          /* Final profile dumping. */
-extern bool opt_prof_leak_error;     /* Exit with error code if memory leaked */
+extern bool    opt_prof_leak;           /* Dump leak summary at exit. */
-extern bool opt_prof_accum;          /* Report cumulative bytes. */
+extern bool    opt_prof_leak_error; /* Exit with error code if memory leaked */
-extern bool opt_prof_log;            /* Turn logging on at boot. */
+extern bool    opt_prof_accum;      /* Report cumulative bytes. */
-extern char opt_prof_prefix[
+extern bool    opt_prof_log;        /* Turn logging on at boot. */
-    /* Minimize memory bloat for non-prof builds. */
+extern char    opt_prof_prefix[
+/* Minimize memory bloat for non-prof builds. */
 #ifdef JEMALLOC_PROF
     PATH_MAX +
 #endif
     1];
 extern bool opt_prof_unbias;
 
+/* Include pid namespace in profile file names. */
+extern bool opt_prof_pid_namespace;
+
 /* For recording recent allocations */
 extern ssize_t opt_prof_recent_alloc_max;
 
@@ -49,47 +56,110 @@ extern size_t lg_prof_sample;
 
 extern bool prof_booted;
 
-void prof_backtrace_hook_set(prof_backtrace_hook_t hook);
+void                  prof_backtrace_hook_set(prof_backtrace_hook_t hook);
-prof_backtrace_hook_t prof_backtrace_hook_get();
+prof_backtrace_hook_t prof_backtrace_hook_get(void);
 
-void prof_dump_hook_set(prof_dump_hook_t hook);
+void             prof_dump_hook_set(prof_dump_hook_t hook);
-prof_dump_hook_t prof_dump_hook_get();
+prof_dump_hook_t prof_dump_hook_get(void);
+
+void               prof_sample_hook_set(prof_sample_hook_t hook);
+prof_sample_hook_t prof_sample_hook_get(void);
+
+void                    prof_sample_free_hook_set(prof_sample_free_hook_t hook);
+prof_sample_free_hook_t prof_sample_free_hook_get(void);
 
 /* Functions only accessed in prof_inlines.h */
 prof_tdata_t *prof_tdata_init(tsd_t *tsd);
 prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
 
 void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
-void prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size,
+void prof_malloc_sample_object(
-    size_t usize, prof_tctx_t *tctx);
+    tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
-void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
+void prof_free_sampled_object(
+    tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
 prof_tctx_t *prof_tctx_create(tsd_t *tsd);
-void prof_idump(tsdn_t *tsdn);
+void         prof_idump(tsdn_t *tsdn);
-bool prof_mdump(tsd_t *tsd, const char *filename);
+bool         prof_mdump(tsd_t *tsd, const char *filename);
-void prof_gdump(tsdn_t *tsdn);
+void         prof_gdump(tsdn_t *tsdn);
 
-void prof_tdata_cleanup(tsd_t *tsd);
+void        prof_tdata_cleanup(tsd_t *tsd);
-bool prof_active_get(tsdn_t *tsdn);
+bool        prof_active_get(tsdn_t *tsdn);
-bool prof_active_set(tsdn_t *tsdn, bool active);
+bool        prof_active_set(tsdn_t *tsdn, bool active);
 const char *prof_thread_name_get(tsd_t *tsd);
-int prof_thread_name_set(tsd_t *tsd, const char *thread_name);
+int         prof_thread_name_set(tsd_t *tsd, const char *thread_name);
-bool prof_thread_active_get(tsd_t *tsd);
+bool        prof_thread_active_get(tsd_t *tsd);
-bool prof_thread_active_set(tsd_t *tsd, bool active);
+bool        prof_thread_active_set(tsd_t *tsd, bool active);
-bool prof_thread_active_init_get(tsdn_t *tsdn);
+bool        prof_thread_active_init_get(tsdn_t *tsdn);
-bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
+bool        prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
-bool prof_gdump_get(tsdn_t *tsdn);
+bool        prof_gdump_get(tsdn_t *tsdn);
-bool prof_gdump_set(tsdn_t *tsdn, bool active);
+bool        prof_gdump_set(tsdn_t *tsdn, bool active);
-void prof_boot0(void);
+void        prof_boot0(void);
-void prof_boot1(void);
+void        prof_boot1(void);
-bool prof_boot2(tsd_t *tsd, base_t *base);
+bool        prof_boot2(tsd_t *tsd, base_t *base);
-void prof_prefork0(tsdn_t *tsdn);
+void        prof_prefork0(tsdn_t *tsdn);
-void prof_prefork1(tsdn_t *tsdn);
+void        prof_prefork1(tsdn_t *tsdn);
-void prof_postfork_parent(tsdn_t *tsdn);
+void        prof_postfork_parent(tsdn_t *tsdn);
-void prof_postfork_child(tsdn_t *tsdn);
+void        prof_postfork_child(tsdn_t *tsdn);
 
-/* Only accessed by thread event. */
 uint64_t prof_sample_new_event_wait(tsd_t *tsd);
-uint64_t prof_sample_postponed_event_wait(tsd_t *tsd);
+uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
-void prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed);
+
+/*
+ * The lookahead functionality facilitates events to be able to lookahead, i.e.
+ * without touching the event counters, to determine whether an event would be
+ * triggered.  The event counters are not advanced until the end of the
+ * allocation / deallocation calls, so the lookahead can be useful if some
+ * preparation work for some event must be done early in the allocation /
+ * deallocation calls.
+ *
+ * Currently only the profiling sampling event needs the lookahead
+ * functionality, so we don't yet define general purpose lookahead functions.
+ *
+ * Surplus is a terminology referring to the amount of bytes beyond what's
+ * needed for triggering an event, which can be a useful quantity to have in
+ * general when lookahead is being called.
+ *
+ * This function returns true if allocation of usize would go above the next
+ * trigger for prof event, and false otherwise.
+ * If function returns true surplus will contain number of bytes beyond that
+ * trigger.
+ */
+
+JEMALLOC_ALWAYS_INLINE bool
+te_prof_sample_event_lookahead_surplus(
+    tsd_t *tsd, size_t usize, size_t *surplus) {
+	if (surplus != NULL) {
+		/*
+		 * This is a dead store: the surplus will be overwritten before
+		 * any read.  The initialization suppresses compiler warnings.
+		 * Meanwhile, using SIZE_MAX to initialize is good for
+		 * debugging purpose, because a valid surplus value is strictly
+		 * less than usize, which is at most SIZE_MAX.
+		 */
+		*surplus = SIZE_MAX;
+	}
+	if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) {
+		return false;
+	}
+	/* The subtraction is intentionally susceptible to underflow. */
+	uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
+	    - tsd_thread_allocated_last_event_get(tsd);
+	uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd);
+	if (accumbytes < sample_wait) {
+		return false;
+	}
+	assert(accumbytes - sample_wait < (uint64_t)usize);
+	if (surplus != NULL) {
+		*surplus = (size_t)(accumbytes - sample_wait);
+	}
+	return true;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
+	return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL);
+}
+
+extern te_base_cb_t prof_sample_te_handler;
 
 #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */

dep/jemalloc/include/jemalloc/internal/prof_hook.h

@@ -1,6 +1,8 @@
 #ifndef JEMALLOC_INTERNAL_PROF_HOOK_H
 #define JEMALLOC_INTERNAL_PROF_HOOK_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+
 /*
  * The hooks types of which are declared in this file are experimental and
  * undocumented, thus the typedefs are located in an 'internal' header.
@@ -18,4 +20,11 @@ typedef void (*prof_backtrace_hook_t)(void **, unsigned *, unsigned);
  */
 typedef void (*prof_dump_hook_t)(const char *filename);
 
+/* ptr, size, backtrace vector, backtrace vector length, usize */
+typedef void (*prof_sample_hook_t)(const void *ptr, size_t size,
+    void **backtrace, unsigned backtrace_length, size_t usize);
+
+/* ptr, size */
+typedef void (*prof_sample_free_hook_t)(const void *, size_t);
+
 #endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */

dep/jemalloc/include/jemalloc/internal/prof_inlines.h

@@ -1,12 +1,17 @@
 #ifndef JEMALLOC_INTERNAL_PROF_INLINES_H
 #define JEMALLOC_INTERNAL_PROF_INLINES_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
+#include "jemalloc/internal/prof_externs.h"
+#include "jemalloc/internal/prof_structs.h"
 #include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/thread_event.h"
 
 JEMALLOC_ALWAYS_INLINE void
-prof_active_assert() {
+prof_active_assert(void) {
 	cassert(config_prof);
 	/*
 	 * If opt_prof is off, then prof_active must always be off, regardless
@@ -37,6 +42,22 @@ prof_gdump_get_unlocked(void) {
 	return prof_gdump_val;
 }
 
+JEMALLOC_ALWAYS_INLINE void
+prof_thread_name_assert(prof_tdata_t *tdata) {
+	if (!config_debug) {
+		return;
+	}
+	prof_active_assert();
+
+	bool terminated = false;
+	for (unsigned i = 0; i < PROF_THREAD_NAME_MAX_LEN; i++) {
+		if (tdata->thread_name[i] == '\0') {
+			terminated = true;
+		}
+	}
+	assert(terminated);
+}
+
 JEMALLOC_ALWAYS_INLINE prof_tdata_t *
 prof_tdata_get(tsd_t *tsd, bool create) {
 	prof_tdata_t *tdata;
@@ -58,6 +79,10 @@ prof_tdata_get(tsd_t *tsd, bool create) {
 		assert(tdata == NULL || tdata->attached);
 	}
 
+	if (tdata != NULL) {
+		prof_thread_name_assert(tdata);
+	}
+
 	return tdata;
 }
 
@@ -81,6 +106,11 @@ prof_info_get_and_reset_recent(tsd_t *tsd, const void *ptr,
 	arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, true);
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+prof_tctx_is_valid(const prof_tctx_t *tctx) {
+	return tctx != NULL && tctx != PROF_TCTX_SENTINEL;
+}
+
 JEMALLOC_ALWAYS_INLINE void
 prof_tctx_reset(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
 	cassert(config_prof);
@@ -101,7 +131,7 @@ JEMALLOC_ALWAYS_INLINE void
 prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
 	cassert(config_prof);
 	assert(edata != NULL);
-	assert((uintptr_t)tctx > (uintptr_t)1U);
+	assert(prof_tctx_is_valid(tctx));
 
 	arena_prof_info_set(tsd, edata, tctx, size);
 }
@@ -134,9 +164,9 @@ JEMALLOC_ALWAYS_INLINE prof_tctx_t *
 prof_alloc_prep(tsd_t *tsd, bool prof_active, bool sample_event) {
 	prof_tctx_t *ret;
 
-	if (!prof_active ||
+	if (!prof_active
-	    likely(prof_sample_should_skip(tsd, sample_event))) {
+	    || likely(prof_sample_should_skip(tsd, sample_event))) {
-		ret = (prof_tctx_t *)(uintptr_t)1U;
+		ret = PROF_TCTX_SENTINEL;
 	} else {
 		ret = prof_tctx_create(tsd);
 	}
@@ -151,7 +181,7 @@ prof_malloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
 	assert(ptr != NULL);
 	assert(usize == isalloc(tsd_tsdn(tsd), ptr));
 
-	if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) {
+	if (unlikely(prof_tctx_is_valid(tctx))) {
 		prof_malloc_sample_object(tsd, ptr, size, usize, tctx);
 	} else {
 		prof_tctx_reset(tsd, ptr, alloc_ctx);
@@ -165,7 +195,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
 	bool sampled, old_sampled, moved;
 
 	cassert(config_prof);
-	assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
+	assert(ptr != NULL || !prof_tctx_is_valid(tctx));
 
 	if (prof_active && ptr != NULL) {
 		assert(usize == isalloc(tsd_tsdn(tsd), ptr));
@@ -178,12 +208,12 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
 			 * sample threshold.
 			 */
 			prof_alloc_rollback(tsd, tctx);
-			tctx = (prof_tctx_t *)(uintptr_t)1U;
+			tctx = PROF_TCTX_SENTINEL;
 		}
 	}
 
-	sampled = ((uintptr_t)tctx > (uintptr_t)1U);
+	sampled = prof_tctx_is_valid(tctx);
-	old_sampled = ((uintptr_t)old_prof_info->alloc_tctx > (uintptr_t)1U);
+	old_sampled = prof_tctx_is_valid(old_prof_info->alloc_tctx);
 	moved = (ptr != old_ptr);
 
 	if (unlikely(sampled)) {
@@ -201,7 +231,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
 	} else {
 		prof_info_t prof_info;
 		prof_info_get(tsd, ptr, NULL, &prof_info);
-		assert((uintptr_t)prof_info.alloc_tctx == (uintptr_t)1U);
+		assert(prof_info.alloc_tctx == PROF_TCTX_SENTINEL);
 	}
 
 	/*
@@ -212,31 +242,29 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
 	 * counters.
 	 */
 	if (unlikely(old_sampled)) {
-		prof_free_sampled_object(tsd, old_usize, old_prof_info);
+		prof_free_sampled_object(
+		    tsd, old_ptr, old_usize, old_prof_info);
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-prof_sample_align(size_t orig_align) {
+prof_sample_align(size_t usize, size_t orig_align) {
 	/*
-	 * Enforce page alignment, so that sampled allocations can be identified
+	 * Enforce alignment, so that sampled allocations can be identified
 	 * w/o metadata lookup.
 	 */
 	assert(opt_prof);
-	return (opt_cache_oblivious && orig_align < PAGE) ? PAGE :
+	return (orig_align < PROF_SAMPLE_ALIGNMENT
-	    orig_align;
+	           && (sz_can_use_slab(usize) || opt_cache_oblivious))
-}
+	    ? PROF_SAMPLE_ALIGNMENT
-
+	    : orig_align;
-JEMALLOC_ALWAYS_INLINE bool
-prof_sample_aligned(const void *ptr) {
-	return ((uintptr_t)ptr & PAGE_MASK) == 0;
 }
 
 JEMALLOC_ALWAYS_INLINE bool
 prof_sampled(tsd_t *tsd, const void *ptr) {
 	prof_info_t prof_info;
 	prof_info_get(tsd, ptr, NULL, &prof_info);
-	bool sampled = (uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U;
+	bool sampled = prof_tctx_is_valid(prof_info.alloc_tctx);
 	if (sampled) {
 		assert(prof_sample_aligned(ptr));
 	}
@@ -244,18 +272,32 @@ prof_sampled(tsd_t *tsd, const void *ptr) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_free(tsd_t *tsd, const void *ptr, size_t usize,
+prof_free(
-    emap_alloc_ctx_t *alloc_ctx) {
+    tsd_t *tsd, const void *ptr, size_t usize, emap_alloc_ctx_t *alloc_ctx) {
 	prof_info_t prof_info;
 	prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
 
 	cassert(config_prof);
 	assert(usize == isalloc(tsd_tsdn(tsd), ptr));
 
-	if (unlikely((uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U)) {
+	if (unlikely(prof_tctx_is_valid(prof_info.alloc_tctx))) {
 		assert(prof_sample_aligned(ptr));
-		prof_free_sampled_object(tsd, usize, &prof_info);
+		prof_free_sampled_object(tsd, ptr, usize, &prof_info);
 	}
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+prof_thread_name_empty(prof_tdata_t *tdata) {
+	prof_active_assert();
+
+	return (tdata->thread_name[0] == '\0');
+}
+
+JEMALLOC_ALWAYS_INLINE void
+prof_thread_name_clear(prof_tdata_t *tdata) {
+	prof_active_assert();
+
+	tdata->thread_name[0] = '\0';
+}
+
 #endif /* JEMALLOC_INTERNAL_PROF_INLINES_H */

dep/jemalloc/include/jemalloc/internal/prof_log.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_PROF_LOG_H
 #define JEMALLOC_INTERNAL_PROF_LOG_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/mutex.h"
 
 extern malloc_mutex_t log_mtx;
@@ -12,9 +13,9 @@ bool prof_log_init(tsd_t *tsdn);
 size_t prof_log_bt_count(void);
 size_t prof_log_alloc_count(void);
 size_t prof_log_thr_count(void);
-bool prof_log_is_logging(void);
+bool   prof_log_is_logging(void);
-bool prof_log_rep_check(void);
+bool   prof_log_rep_check(void);
-void prof_log_dummy_set(bool new_value);
+void   prof_log_dummy_set(bool new_value);
 
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);

dep/jemalloc/include/jemalloc/internal/prof_recent.h

@@ -1,13 +1,17 @@
 #ifndef JEMALLOC_INTERNAL_PROF_RECENT_H
 #define JEMALLOC_INTERNAL_PROF_RECENT_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/edata.h"
+#include "jemalloc/internal/mutex.h"
+
 extern malloc_mutex_t prof_recent_alloc_mtx;
 extern malloc_mutex_t prof_recent_dump_mtx;
 
 bool prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx);
 void prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize);
 void prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata);
-bool prof_recent_init();
+bool prof_recent_init(void);
 void edata_prof_recent_alloc_init(edata_t *edata);
 
 /* Used in unit tests. */
@@ -16,7 +20,7 @@ extern prof_recent_list_t prof_recent_alloc_list;
 edata_t *prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *node);
 prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata);
 
-ssize_t prof_recent_alloc_max_ctl_read();
+ssize_t prof_recent_alloc_max_ctl_read(void);
 ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max);
 void prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque);
 

dep/jemalloc/include/jemalloc/internal/prof_stats.h

@@ -1,6 +1,9 @@
 #ifndef JEMALLOC_INTERNAL_PROF_STATS_H
 #define JEMALLOC_INTERNAL_PROF_STATS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/mutex.h"
+
 typedef struct prof_stats_s prof_stats_t;
 struct prof_stats_s {
 	uint64_t req_sum;

dep/jemalloc/include/jemalloc/internal/prof_structs.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_PROF_STRUCTS_H
 #define JEMALLOC_INTERNAL_PROF_STRUCTS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/ckh.h"
 #include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
@@ -9,29 +10,29 @@
 
 struct prof_bt_s {
 	/* Backtrace, stored as len program counters. */
-	void		**vec;
+	void   **vec;
-	unsigned	len;
+	unsigned len;
 };
 
 #ifdef JEMALLOC_PROF_LIBGCC
 /* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
 typedef struct {
-	void 		**vec;
+	void    **vec;
-	unsigned	*len;
+	unsigned *len;
-	unsigned	max;
+	unsigned  max;
 } prof_unwind_data_t;
 #endif
 
 struct prof_cnt_s {
 	/* Profiling counters. */
-	uint64_t	curobjs;
+	uint64_t curobjs;
-	uint64_t	curobjs_shifted_unbiased;
+	uint64_t curobjs_shifted_unbiased;
-	uint64_t	curbytes;
+	uint64_t curbytes;
-	uint64_t	curbytes_unbiased;
+	uint64_t curbytes_unbiased;
-	uint64_t	accumobjs;
+	uint64_t accumobjs;
-	uint64_t	accumobjs_shifted_unbiased;
+	uint64_t accumobjs_shifted_unbiased;
-	uint64_t	accumbytes;
+	uint64_t accumbytes;
-	uint64_t	accumbytes_unbiased;
+	uint64_t accumbytes_unbiased;
 };
 
 typedef enum {
@@ -43,26 +44,26 @@ typedef enum {
 
 struct prof_tctx_s {
 	/* Thread data for thread that performed the allocation. */
-	prof_tdata_t		*tdata;
+	prof_tdata_t *tdata;
 
 	/*
 	 * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
 	 * defunct during teardown.
 	 */
-	uint64_t		thr_uid;
+	uint64_t thr_uid;
-	uint64_t		thr_discrim;
+	uint64_t thr_discrim;
 
 	/*
 	 * Reference count of how many times this tctx object is referenced in
 	 * recent allocation / deallocation records, protected by tdata->lock.
 	 */
-	uint64_t		recent_count;
+	uint64_t recent_count;
 
 	/* Profiling counters, protected by tdata->lock. */
-	prof_cnt_t		cnts;
+	prof_cnt_t cnts;
 
 	/* Associated global context. */
-	prof_gctx_t		*gctx;
+	prof_gctx_t *gctx;
 
 	/*
 	 * UID that distinguishes multiple tctx's created by the same thread,
@@ -77,40 +78,40 @@ struct prof_tctx_s {
 	 *   threshold can be hit again before the first consumer finishes
 	 *   executing prof_tctx_destroy().
 	 */
-	uint64_t		tctx_uid;
+	uint64_t tctx_uid;
 
 	/* Linkage into gctx's tctxs. */
-	rb_node(prof_tctx_t)	tctx_link;
+	rb_node(prof_tctx_t) tctx_link;
 
 	/*
 	 * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
 	 * sample vs destroy race.
 	 */
-	bool			prepared;
+	bool prepared;
 
 	/* Current dump-related state, protected by gctx->lock. */
-	prof_tctx_state_t	state;
+	prof_tctx_state_t state;
 
 	/*
 	 * Copy of cnts snapshotted during early dump phase, protected by
 	 * dump_mtx.
 	 */
-	prof_cnt_t		dump_cnts;
+	prof_cnt_t dump_cnts;
 };
 typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
 
 struct prof_info_s {
 	/* Time when the allocation was made. */
-	nstime_t		alloc_time;
+	nstime_t alloc_time;
 	/* Points to the prof_tctx_t corresponding to the allocation. */
-	prof_tctx_t		*alloc_tctx;
+	prof_tctx_t *alloc_tctx;
 	/* Allocation request size. */
-	size_t			alloc_size;
+	size_t alloc_size;
 };
 
 struct prof_gctx_s {
 	/* Protects nlimbo, cnt_summed, and tctxs. */
-	malloc_mutex_t		*lock;
+	malloc_mutex_t *lock;
 
 	/*
 	 * Number of threads that currently cause this gctx to be in a state of
@@ -122,54 +123,48 @@ struct prof_gctx_s {
 	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
 	 * gctx.
 	 */
-	unsigned		nlimbo;
+	unsigned nlimbo;
 
 	/*
 	 * Tree of profile counters, one for each thread that has allocated in
 	 * this context.
 	 */
-	prof_tctx_tree_t	tctxs;
+	prof_tctx_tree_t tctxs;
 
 	/* Linkage for tree of contexts to be dumped. */
-	rb_node(prof_gctx_t)	dump_link;
+	rb_node(prof_gctx_t) dump_link;
 
 	/* Temporary storage for summation during dump. */
-	prof_cnt_t		cnt_summed;
+	prof_cnt_t cnt_summed;
 
 	/* Associated backtrace. */
-	prof_bt_t		bt;
+	prof_bt_t bt;
 
 	/* Backtrace vector, variable size, referred to by bt. */
-	void			*vec[1];
+	void *vec[1];
 };
 typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
 
 struct prof_tdata_s {
-	malloc_mutex_t		*lock;
+	malloc_mutex_t *lock;
 
 	/* Monotonically increasing unique thread identifier. */
-	uint64_t		thr_uid;
+	uint64_t thr_uid;
 
 	/*
 	 * Monotonically increasing discriminator among tdata structures
 	 * associated with the same thr_uid.
 	 */
-	uint64_t		thr_discrim;
+	uint64_t thr_discrim;
 
-	/* Included in heap profile dumps if non-NULL. */
+	rb_node(prof_tdata_t) tdata_link;
-	char			*thread_name;
-
-	bool			attached;
-	bool			expired;
-
-	rb_node(prof_tdata_t)	tdata_link;
 
 	/*
 	 * Counter used to initialize prof_tctx_t's tctx_uid.  No locking is
 	 * necessary when incrementing this field, because only one thread ever
 	 * does so.
 	 */
-	uint64_t		tctx_uid_next;
+	uint64_t tctx_uid_next;
 
 	/*
 	 * Hash of (prof_bt_t *)-->(prof_tctx_t *).  Each thread tracks
@@ -177,12 +172,15 @@ struct prof_tdata_s {
 	 * associated with thread-specific prof_tctx_t objects.  Other threads
 	 * may write to prof_tctx_t contents when freeing associated objects.
 	 */
-	ckh_t			bt2tctx;
+	ckh_t bt2tctx;
+
+	/* Included in heap profile dumps if has content. */
+	char thread_name[PROF_THREAD_NAME_MAX_LEN];
 
 	/* State used to avoid dumping while operating on prof internals. */
-	bool			enq;
+	bool enq;
-	bool			enq_idump;
+	bool enq_idump;
-	bool			enq_gdump;
+	bool enq_gdump;
 
 	/*
 	 * Set to true during an early dump phase for tdata's which are
@@ -190,19 +188,22 @@ struct prof_tdata_s {
 	 * to false so that they aren't accidentally included in later dump
 	 * phases.
 	 */
-	bool			dumping;
+	bool dumping;
 
 	/*
 	 * True if profiling is active for this tdata's thread
 	 * (thread.prof.active mallctl).
 	 */
-	bool			active;
+	bool active;
+
+	bool attached;
+	bool expired;
 
 	/* Temporary storage for summation during dump. */
-	prof_cnt_t		cnt_summed;
+	prof_cnt_t cnt_summed;
 
 	/* Backtrace vector, used for calls to prof_backtrace(). */
-	void			*vec[PROF_BT_MAX];
+	void **vec;
 };
 typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
 
@@ -211,9 +212,9 @@ struct prof_recent_s {
 	nstime_t dalloc_time;
 
 	ql_elm(prof_recent_t) link;
-	size_t size;
+	size_t       size;
-	size_t usize;
+	size_t       usize;
-	atomic_p_t alloc_edata; /* NULL means allocation has been freed. */
+	atomic_p_t   alloc_edata; /* NULL means allocation has been freed. */
 	prof_tctx_t *alloc_tctx;
 	prof_tctx_t *dalloc_tctx;
 };

dep/jemalloc/include/jemalloc/internal/prof_sys.h

@@ -1,30 +1,35 @@
 #ifndef JEMALLOC_INTERNAL_PROF_SYS_H
 #define JEMALLOC_INTERNAL_PROF_SYS_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
+#include "jemalloc/internal/mutex.h"
+
 extern malloc_mutex_t prof_dump_filename_mtx;
-extern base_t *prof_base;
+extern base_t        *prof_base;
 
 void bt_init(prof_bt_t *bt, void **vec);
 void prof_backtrace(tsd_t *tsd, prof_bt_t *bt);
-void prof_hooks_init();
+void prof_hooks_init(void);
-void prof_unwind_init();
+void prof_unwind_init(void);
 void prof_sys_thread_name_fetch(tsd_t *tsd);
-int prof_getpid(void);
+int  prof_getpid(void);
 void prof_get_default_filename(tsdn_t *tsdn, char *filename, uint64_t ind);
 bool prof_prefix_set(tsdn_t *tsdn, const char *prefix);
 void prof_fdump_impl(tsd_t *tsd);
 void prof_idump_impl(tsd_t *tsd);
 bool prof_mdump_impl(tsd_t *tsd, const char *filename);
 void prof_gdump_impl(tsd_t *tsd);
+int  prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high);
 
 /* Used in unit tests. */
-typedef int (prof_sys_thread_name_read_t)(char *buf, size_t limit);
+typedef int(prof_sys_thread_name_read_t)(char *buf, size_t limit);
 extern prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read;
-typedef int (prof_dump_open_file_t)(const char *, int);
+typedef int(prof_dump_open_file_t)(const char *, int);
 extern prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file;
-typedef ssize_t (prof_dump_write_file_t)(int, const void *, size_t);
+typedef ssize_t(prof_dump_write_file_t)(int, const void *, size_t);
 extern prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file;
-typedef int (prof_dump_open_maps_t)();
+typedef int(prof_dump_open_maps_t)(void);
 extern prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps;
 
 #endif /* JEMALLOC_INTERNAL_PROF_SYS_H */

dep/jemalloc/include/jemalloc/internal/prof_types.h

@@ -1,75 +1,94 @@
 #ifndef JEMALLOC_INTERNAL_PROF_TYPES_H
 #define JEMALLOC_INTERNAL_PROF_TYPES_H
 
-typedef struct prof_bt_s prof_bt_t;
+typedef struct prof_bt_s     prof_bt_t;
-typedef struct prof_cnt_s prof_cnt_t;
+typedef struct prof_cnt_s    prof_cnt_t;
-typedef struct prof_tctx_s prof_tctx_t;
+typedef struct prof_tctx_s   prof_tctx_t;
-typedef struct prof_info_s prof_info_t;
+typedef struct prof_info_s   prof_info_t;
-typedef struct prof_gctx_s prof_gctx_t;
+typedef struct prof_gctx_s   prof_gctx_t;
-typedef struct prof_tdata_s prof_tdata_t;
+typedef struct prof_tdata_s  prof_tdata_t;
 typedef struct prof_recent_s prof_recent_t;
 
 /* Option defaults. */
 #ifdef JEMALLOC_PROF
-#  define PROF_PREFIX_DEFAULT		"jeprof"
+#	define PROF_PREFIX_DEFAULT "jeprof"
 #else
-#  define PROF_PREFIX_DEFAULT		""
+#	define PROF_PREFIX_DEFAULT ""
 #endif
-#define LG_PROF_SAMPLE_DEFAULT		19
+#define LG_PROF_SAMPLE_DEFAULT 19
-#define LG_PROF_INTERVAL_DEFAULT	-1
+#define LG_PROF_INTERVAL_DEFAULT -1
 
 /*
  * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
  * is based on __builtin_return_address() necessarily has a hard-coded number
  * of backtrace frame handlers, and should be kept in sync with this setting.
  */
-#define PROF_BT_MAX			128
+#ifdef JEMALLOC_PROF_GCC
+#	define PROF_BT_MAX_LIMIT 256
+#else
+#	define PROF_BT_MAX_LIMIT UINT_MAX
+#endif
+#define PROF_BT_MAX_DEFAULT 128
 
 /* Initial hash table size. */
-#define PROF_CKH_MINITEMS		64
+#define PROF_CKH_MINITEMS 64
 
 /* Size of memory buffer to use when writing dump files. */
 #ifndef JEMALLOC_PROF
 /* Minimize memory bloat for non-prof builds. */
-#  define PROF_DUMP_BUFSIZE		1
+#	define PROF_DUMP_BUFSIZE 1
 #elif defined(JEMALLOC_DEBUG)
 /* Use a small buffer size in debug build, mainly to facilitate testing. */
-#  define PROF_DUMP_BUFSIZE		16
+#	define PROF_DUMP_BUFSIZE 16
 #else
-#  define PROF_DUMP_BUFSIZE		65536
+#	define PROF_DUMP_BUFSIZE 65536
 #endif
 
 /* Size of size class related tables */
 #ifdef JEMALLOC_PROF
-#  define PROF_SC_NSIZES		SC_NSIZES
+#	define PROF_SC_NSIZES SC_NSIZES
 #else
 /* Minimize memory bloat for non-prof builds. */
-#  define PROF_SC_NSIZES		1
+#	define PROF_SC_NSIZES 1
 #endif
 
 /* Size of stack-allocated buffer used by prof_printf(). */
-#define PROF_PRINTF_BUFSIZE		128
+#define PROF_PRINTF_BUFSIZE 128
 
 /*
  * Number of mutexes shared among all gctx's.  No space is allocated for these
  * unless profiling is enabled, so it's okay to over-provision.
  */
-#define PROF_NCTX_LOCKS			1024
+#define PROF_NCTX_LOCKS 1024
 
 /*
  * Number of mutexes shared among all tdata's.  No space is allocated for these
  * unless profiling is enabled, so it's okay to over-provision.
  */
-#define PROF_NTDATA_LOCKS		256
+#define PROF_NTDATA_LOCKS 256
 
 /* Minimize memory bloat for non-prof builds. */
 #ifdef JEMALLOC_PROF
-#define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
+#	define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
 #else
-#define PROF_DUMP_FILENAME_LEN 1
+#	define PROF_DUMP_FILENAME_LEN 1
 #endif
 
 /* Default number of recent allocations to record. */
 #define PROF_RECENT_ALLOC_MAX_DEFAULT 0
 
+/* Thread name storage size limit. */
+#define PROF_THREAD_NAME_MAX_LEN 16
+
+/*
+ * Minimum required alignment for sampled allocations. Over-aligning sampled
+ * allocations allows us to quickly identify them on the dalloc path without
+ * resorting to metadata lookup.
+ */
+#define PROF_SAMPLE_ALIGNMENT PAGE
+#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
+
+/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
+#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
+
 #endif /* JEMALLOC_INTERNAL_PROF_TYPES_H */

dep/jemalloc/include/jemalloc/internal/psset.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_PSSET_H
 #define JEMALLOC_INTERNAL_PSSET_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/hpdata.h"
 
 /*
@@ -20,6 +21,12 @@
  */
 #define PSSET_NPSIZES 64
 
+/*
+ * We store non-hugefied and hugified pageslabs metadata separately.
+ * [0] corresponds to non-hugified and [1] to hugified pageslabs.
+ */
+#define PSSET_NHUGE 2
+
 /*
  * We keep two purge lists per page size class; one for hugified hpdatas (at
  * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
@@ -43,21 +50,36 @@ struct psset_bin_stats_s {
 
 typedef struct psset_stats_s psset_stats_t;
 struct psset_stats_s {
+	/*
+	 * Merged stats for all pageslabs in psset.  This lets us quickly
+	 * answer queries for the number of dirty and active pages in the
+	 * entire set.
+	 */
+	psset_bin_stats_t merged;
+
+	/*
+	 * Below are the same stats, but aggregated by different
+	 * properties of pageslabs: huginess or fullness.
+	 */
+
+	/* Non-huge and huge slabs. */
+	psset_bin_stats_t slabs[PSSET_NHUGE];
+
 	/*
 	 * The second index is huge stats; nonfull_slabs[pszind][0] contains
 	 * stats for the non-huge slabs in bucket pszind, while
 	 * nonfull_slabs[pszind][1] contains stats for the huge slabs.
 	 */
-	psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2];
+	psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][PSSET_NHUGE];
 
 	/*
 	 * Full slabs don't live in any edata heap, but we still track their
 	 * stats.
 	 */
-	psset_bin_stats_t full_slabs[2];
+	psset_bin_stats_t full_slabs[PSSET_NHUGE];
 
 	/* Empty slabs are similar. */
-	psset_bin_stats_t empty_slabs[2];
+	psset_bin_stats_t empty_slabs[PSSET_NHUGE];
 };
 
 typedef struct psset_s psset_t;
@@ -68,13 +90,7 @@ struct psset_s {
 	 */
 	hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
 	/* Bitmap for which set bits correspond to non-empty heaps. */
-	fb_group_t pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)];
+	fb_group_t    pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)];
-	/*
-	 * The sum of all bin stats in stats.  This lets us quickly answer
-	 * queries for the number of dirty, active, and retained pages in the
-	 * entire set.
-	 */
-	psset_bin_stats_t merged_stats;
 	psset_stats_t stats;
 	/*
 	 * Slabs with no active allocations, but which are allowed to serve new
@@ -105,8 +121,12 @@ void psset_update_end(psset_t *psset, hpdata_t *ps);
 
 /* Analogous to the eset_fit; pick a hpdata to serve the request. */
 hpdata_t *psset_pick_alloc(psset_t *psset, size_t size);
-/* Pick one to purge. */
+/*
-hpdata_t *psset_pick_purge(psset_t *psset);
+ * Pick one to purge that is purgable before given time (inclusive).  If now
+ * is NULL then time is not considered.
+ */
+hpdata_t *psset_pick_purge(psset_t *psset, const nstime_t *now);
+
 /* Pick one to hugify. */
 hpdata_t *psset_pick_hugify(psset_t *psset);
 
@@ -115,17 +135,17 @@ void psset_remove(psset_t *psset, hpdata_t *ps);
 
 static inline size_t
 psset_npageslabs(psset_t *psset) {
-	return psset->merged_stats.npageslabs;
+	return psset->stats.merged.npageslabs;
 }
 
 static inline size_t
 psset_nactive(psset_t *psset) {
-	return psset->merged_stats.nactive;
+	return psset->stats.merged.nactive;
 }
 
 static inline size_t
 psset_ndirty(psset_t *psset) {
-	return psset->merged_stats.ndirty;
+	return psset->stats.merged.ndirty;
 }
 
 #endif /* JEMALLOC_INTERNAL_PSSET_H */

dep/jemalloc/include/jemalloc/internal/public_namespace.h

@@ -2,6 +2,8 @@
 #define je_calloc JEMALLOC_N(calloc)
 #define je_dallocx JEMALLOC_N(dallocx)
 #define je_free JEMALLOC_N(free)
+#define je_free_sized JEMALLOC_N(free_sized)
+#define je_free_aligned_sized JEMALLOC_N(free_aligned_sized)
 #define je_mallctl JEMALLOC_N(mallctl)
 #define je_mallctlbymib JEMALLOC_N(mallctlbymib)
 #define je_mallctlnametomib JEMALLOC_N(mallctlnametomib)
@@ -12,7 +14,7 @@
 #define je_malloc_stats_print JEMALLOC_N(malloc_stats_print)
 #define je_malloc_usable_size JEMALLOC_N(malloc_usable_size)
 #define je_mallocx JEMALLOC_N(mallocx)
-#define je_smallocx_54eaed1d8b56b1aa528be3bdd1877e59c56fa90c JEMALLOC_N(smallocx_54eaed1d8b56b1aa528be3bdd1877e59c56fa90c)
+#define je_smallocx_ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396 JEMALLOC_N(smallocx_ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396)
 #define je_nallocx JEMALLOC_N(nallocx)
 #define je_posix_memalign JEMALLOC_N(posix_memalign)
 #define je_rallocx JEMALLOC_N(rallocx)
@@ -22,4 +24,5 @@
 #define je_xallocx JEMALLOC_N(xallocx)
 #define je_memalign JEMALLOC_N(memalign)
 #define je_valloc JEMALLOC_N(valloc)
+#define je_pvalloc JEMALLOC_N(pvalloc)
 #define je_malloc_size JEMALLOC_N(malloc_size)

dep/jemalloc/include/jemalloc/internal/public_unnamespace.h

@@ -2,6 +2,8 @@
 #undef je_calloc
 #undef je_dallocx
 #undef je_free
+#undef je_free_sized
+#undef je_free_aligned_sized
 #undef je_mallctl
 #undef je_mallctlbymib
 #undef je_mallctlnametomib
@@ -12,7 +14,7 @@
 #undef je_malloc_stats_print
 #undef je_malloc_usable_size
 #undef je_mallocx
-#undef je_smallocx_54eaed1d8b56b1aa528be3bdd1877e59c56fa90c
+#undef je_smallocx_ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396
 #undef je_nallocx
 #undef je_posix_memalign
 #undef je_rallocx
@@ -22,4 +24,5 @@
 #undef je_xallocx
 #undef je_memalign
 #undef je_valloc
+#undef je_pvalloc
 #undef je_malloc_size

dep/jemalloc/include/jemalloc/internal/ql.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_QL_H
 #define JEMALLOC_INTERNAL_QL_H
 
+#include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/qr.h"
 
 /*
@@ -27,33 +28,36 @@
  */
 
 /* List definitions. */
-#define ql_head(a_type)							\
+#define ql_head(a_type)                                                        \
-struct {								\
+	struct {                                                               \
-	a_type *qlh_first;						\
+		a_type *qlh_first;                                             \
-}
+	}
 
 /* Static initializer for an empty list. */
-#define ql_head_initializer(a_head) {NULL}
+#define ql_head_initializer(a_head)                                            \
+	{ NULL }
 
 /* The field definition. */
-#define ql_elm(a_type)	qr(a_type)
+#define ql_elm(a_type) qr(a_type)
 
 /* A pointer to the first element in the list, or NULL if the list is empty. */
 #define ql_first(a_head) ((a_head)->qlh_first)
 
 /* Dynamically initializes a list. */
-#define ql_new(a_head) do {						\
+#define ql_new(a_head)                                                         \
-	ql_first(a_head) = NULL;					\
+	do {                                                                   \
-} while (0)
+		ql_first(a_head) = NULL;                                       \
+	} while (0)
 
 /*
  * Sets dest to be the contents of src (overwriting any elements there), leaving
  * src empty.
  */
-#define ql_move(a_head_dest, a_head_src) do {				\
+#define ql_move(a_head_dest, a_head_src)                                       \
-	ql_first(a_head_dest) = ql_first(a_head_src);			\
+	do {                                                                   \
-	ql_new(a_head_src);						\
+		ql_first(a_head_dest) = ql_first(a_head_src);                  \
-} while (0)
+		ql_new(a_head_src);                                            \
+	} while (0)
 
 /* True if the list is empty, otherwise false. */
 #define ql_empty(a_head) (ql_first(a_head) == NULL)
@@ -67,85 +71,91 @@ struct {								\
 /*
  * Obtains the last item in the list.
  */
-#define ql_last(a_head, a_field)					\
+#define ql_last(a_head, a_field)                                               \
 	(ql_empty(a_head) ? NULL : qr_prev(ql_first(a_head), a_field))
 
 /*
  * Gets a pointer to the next/prev element in the list.  Trying to advance past
  * the end or retreat before the beginning of the list returns NULL.
  */
-#define ql_next(a_head, a_elm, a_field)					\
+#define ql_next(a_head, a_elm, a_field)                                        \
-	((ql_last(a_head, a_field) != (a_elm))				\
+	((ql_last(a_head, a_field) != (a_elm)) ? qr_next((a_elm), a_field)     \
-	    ? qr_next((a_elm), a_field)	: NULL)
+	                                       : NULL)
-#define ql_prev(a_head, a_elm, a_field)					\
+#define ql_prev(a_head, a_elm, a_field)                                        \
-	((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field)	\
+	((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) : NULL)
-				       : NULL)
 
 /* Inserts a_elm before a_qlelm in the list. */
-#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do {		\
+#define ql_before_insert(a_head, a_qlelm, a_elm, a_field)                      \
-	qr_before_insert((a_qlelm), (a_elm), a_field);			\
+	do {                                                                   \
-	if (ql_first(a_head) == (a_qlelm)) {				\
+		qr_before_insert((a_qlelm), (a_elm), a_field);                 \
-		ql_first(a_head) = (a_elm);				\
+		if (ql_first(a_head) == (a_qlelm)) {                           \
-	}								\
+			ql_first(a_head) = (a_elm);                            \
-} while (0)
+		}                                                              \
+	} while (0)
 
 /* Inserts a_elm after a_qlelm in the list. */
-#define ql_after_insert(a_qlelm, a_elm, a_field)			\
+#define ql_after_insert(a_qlelm, a_elm, a_field)                               \
 	qr_after_insert((a_qlelm), (a_elm), a_field)
 
 /* Inserts a_elm as the first item in the list. */
-#define ql_head_insert(a_head, a_elm, a_field) do {			\
+#define ql_head_insert(a_head, a_elm, a_field)                                 \
-	if (!ql_empty(a_head)) {					\
+	do {                                                                   \
-		qr_before_insert(ql_first(a_head), (a_elm), a_field);	\
+		if (!ql_empty(a_head)) {                                       \
-	}								\
+			qr_before_insert(ql_first(a_head), (a_elm), a_field);  \
-	ql_first(a_head) = (a_elm);					\
+		}                                                              \
-} while (0)
+		ql_first(a_head) = (a_elm);                                    \
+	} while (0)
 
 /* Inserts a_elm as the last item in the list. */
-#define ql_tail_insert(a_head, a_elm, a_field) do {			\
+#define ql_tail_insert(a_head, a_elm, a_field)                                 \
-	if (!ql_empty(a_head)) {					\
+	do {                                                                   \
-		qr_before_insert(ql_first(a_head), (a_elm), a_field);	\
+		if (!ql_empty(a_head)) {                                       \
-	}								\
+			qr_before_insert(ql_first(a_head), (a_elm), a_field);  \
-	ql_first(a_head) = qr_next((a_elm), a_field);			\
+		}                                                              \
-} while (0)
+		ql_first(a_head) = qr_next((a_elm), a_field);                  \
+	} while (0)
 
 /*
  * Given lists a = [a_1, ..., a_n] and [b_1, ..., b_n], results in:
  * a = [a1, ..., a_n, b_1, ..., b_n] and b = [].
  */
-#define ql_concat(a_head_a, a_head_b, a_field) do {			\
+#define ql_concat(a_head_a, a_head_b, a_field)                                 \
-	if (ql_empty(a_head_a)) {					\
+	do {                                                                   \
-		ql_move(a_head_a, a_head_b);				\
+		if (ql_empty(a_head_a)) {                                      \
-	} else if (!ql_empty(a_head_b)) {				\
+			ql_move(a_head_a, a_head_b);                           \
-		qr_meld(ql_first(a_head_a), ql_first(a_head_b),		\
+		} else if (!ql_empty(a_head_b)) {                              \
-		    a_field);						\
+			qr_meld(                                               \
-		ql_new(a_head_b);					\
+			    ql_first(a_head_a), ql_first(a_head_b), a_field);  \
-	}								\
+			ql_new(a_head_b);                                      \
-} while (0)
+		}                                                              \
+	} while (0)
 
 /* Removes a_elm from the list. */
-#define ql_remove(a_head, a_elm, a_field) do {				\
+#define ql_remove(a_head, a_elm, a_field)                                      \
-	if (ql_first(a_head) == (a_elm)) {				\
+	do {                                                                   \
-		ql_first(a_head) = qr_next(ql_first(a_head), a_field);	\
+		if (ql_first(a_head) == (a_elm)) {                             \
-	}								\
+			ql_first(a_head) = qr_next(ql_first(a_head), a_field); \
-	if (ql_first(a_head) != (a_elm)) {				\
+		}                                                              \
-		qr_remove((a_elm), a_field);				\
+		if (ql_first(a_head) != (a_elm)) {                             \
-	} else {							\
+			qr_remove((a_elm), a_field);                           \
-		ql_new(a_head);						\
+		} else {                                                       \
-	}								\
+			ql_new(a_head);                                        \
-} while (0)
+		}                                                              \
+	} while (0)
 
 /* Removes the first item in the list. */
-#define ql_head_remove(a_head, a_type, a_field) do {			\
+#define ql_head_remove(a_head, a_type, a_field)                                \
-	a_type *t = ql_first(a_head);					\
+	do {                                                                   \
-	ql_remove((a_head), t, a_field);				\
+		a_type *t = ql_first(a_head);                                  \
-} while (0)
+		ql_remove((a_head), t, a_field);                               \
+	} while (0)
 
 /* Removes the last item in the list. */
-#define ql_tail_remove(a_head, a_type, a_field) do {			\
+#define ql_tail_remove(a_head, a_type, a_field)                                \
-	a_type *t = ql_last(a_head, a_field);				\
+	do {                                                                   \
-	ql_remove((a_head), t, a_field);				\
+		a_type *t = ql_last(a_head, a_field);                          \
-} while (0)
+		ql_remove((a_head), t, a_field);                               \
+	} while (0)
 
 /*
  * Given a = [a_1, a_2, ..., a_n-1, a_n, a_n+1, ...],
@@ -154,14 +164,15 @@ struct {								\
  * and replaces b's contents with:
  *   b = [a_n, a_n+1, ...]
  */
-#define ql_split(a_head_a, a_elm, a_head_b, a_field) do {		\
+#define ql_split(a_head_a, a_elm, a_head_b, a_field)                           \
-	if (ql_first(a_head_a) == (a_elm)) {				\
+	do {                                                                   \
-		ql_move(a_head_b, a_head_a);				\
+		if (ql_first(a_head_a) == (a_elm)) {                           \
-	} else {							\
+			ql_move(a_head_b, a_head_a);                           \
-		qr_split(ql_first(a_head_a), (a_elm), a_field);		\
+		} else {                                                       \
-		ql_first(a_head_b) = (a_elm);				\
+			qr_split(ql_first(a_head_a), (a_elm), a_field);        \
-	}								\
+			ql_first(a_head_b) = (a_elm);                          \
-} while (0)
+		}                                                              \
+	} while (0)
 
 /*
  * An optimized version of:
@@ -169,9 +180,10 @@ struct {								\
  *	ql_remove((a_head), t, a_field);
  *	ql_tail_insert((a_head), t, a_field);
  */
-#define ql_rotate(a_head, a_field) do {					\
+#define ql_rotate(a_head, a_field)                                             \
-	ql_first(a_head) = qr_next(ql_first(a_head), a_field);		\
+	do {                                                                   \
-} while (0)
+		ql_first(a_head) = qr_next(ql_first(a_head), a_field);         \
+	} while (0)
 
 /*
  * Helper macro to iterate over each element in a list in order, starting from
@@ -188,10 +200,10 @@ struct {								\
  * }
  */
 
-#define ql_foreach(a_var, a_head, a_field)				\
+#define ql_foreach(a_var, a_head, a_field)                                     \
-	qr_foreach((a_var), ql_first(a_head), a_field)
+	qr_foreach ((a_var), ql_first(a_head), a_field)
 
-#define ql_reverse_foreach(a_var, a_head, a_field)			\
+#define ql_reverse_foreach(a_var, a_head, a_field)                             \
 	qr_reverse_foreach((a_var), ql_first(a_head), a_field)
 
 #endif /* JEMALLOC_INTERNAL_QL_H */