diff --git a/benchmarks/malloc_benchmark.cpp b/benchmarks/malloc_benchmark.cpp index 18ba52386..e733cd055 100644 --- a/benchmarks/malloc_benchmark.cpp +++ b/benchmarks/malloc_benchmark.cpp @@ -36,11 +36,11 @@ #if defined(__BIONIC__) -static void BM_mallopt_purge(benchmark::State& state) { +static void RunMalloptPurge(benchmark::State& state, int purge_value) { static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576}; static int pagesize = getpagesize(); mallopt(M_DECAY_TIME, 1); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); for (auto _ : state) { state.PauseTiming(); std::vector ptrs; @@ -63,10 +63,19 @@ static void BM_mallopt_purge(benchmark::State& state) { ptrs.clear(); state.ResumeTiming(); - mallopt(M_PURGE, 0); + mallopt(purge_value, 0); } mallopt(M_DECAY_TIME, 0); } + +static void BM_mallopt_purge(benchmark::State& state) { + RunMalloptPurge(state, M_PURGE); +} BIONIC_BENCHMARK(BM_mallopt_purge); +static void BM_mallopt_purge_all(benchmark::State& state) { + RunMalloptPurge(state, M_PURGE_ALL); +} +BIONIC_BENCHMARK(BM_mallopt_purge_all); + #endif diff --git a/benchmarks/malloc_map_benchmark.cpp b/benchmarks/malloc_map_benchmark.cpp index ba4d62c0b..575732539 100644 --- a/benchmarks/malloc_map_benchmark.cpp +++ b/benchmarks/malloc_map_benchmark.cpp @@ -69,7 +69,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) { for (auto _ : state) { #if defined(__BIONIC__) state.PauseTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); uint64_t rss_bytes_before = 0; Gather(&rss_bytes_before); state.ResumeTiming(); @@ -80,7 +80,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) { } #if defined(__BIONIC__) state.PauseTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); Gather(&rss_bytes); // Try and record only the memory used in the map. rss_bytes -= rss_bytes_before; diff --git a/benchmarks/malloc_rss_benchmark.cpp b/benchmarks/malloc_rss_benchmark.cpp index 58f61d971..4b34e72f7 100644 --- a/benchmarks/malloc_rss_benchmark.cpp +++ b/benchmarks/malloc_rss_benchmark.cpp @@ -112,7 +112,7 @@ void StressSizeClass(size_t numThreads, size_t allocSize) { // Do an explicit purge to ensure we will be more likely to get the actual // in-use memory. - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); android::meminfo::ProcMemInfo proc_mem(getpid()); const std::vector& maps = proc_mem.MapsWithoutUsageStats(); diff --git a/libc/bionic/jemalloc_wrapper.cpp b/libc/bionic/jemalloc_wrapper.cpp index ef488eecc..ce3f31420 100644 --- a/libc/bionic/jemalloc_wrapper.cpp +++ b/libc/bionic/jemalloc_wrapper.cpp @@ -102,7 +102,7 @@ int je_mallopt(int param, int value) { } } return 1; - } else if (param == M_PURGE) { + } else if (param == M_PURGE || param == M_PURGE_ALL) { // Only clear the current thread cache since there is no easy way to // clear the caches of other threads. // This must be done first so that cleared allocations get purged diff --git a/libc/include/malloc.h b/libc/include/malloc.h index 02bda608c..6a2d38080 100644 --- a/libc/include/malloc.h +++ b/libc/include/malloc.h @@ -183,7 +183,15 @@ int malloc_info(int __must_be_zero, FILE* _Nonnull __fp) __INTRODUCED_IN(23); * Available since API level 28. */ #define M_PURGE (-101) - +/** + * mallopt() option to immediately purge all possible memory back to + * the kernel. This call can take longer than a normal purge since it + * examines everything. In some cases, it can take more than twice the + * time of a M_PURGE call. The value is ignored. + * + * Available since API level 34. + */ +#define M_PURGE_ALL (-104) /** * mallopt() option to tune the allocator's choice of memory tags to diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp index 63ad99d3d..4e7eb7b05 100644 --- a/tests/malloc_test.cpp +++ b/tests/malloc_test.cpp @@ -36,7 +36,10 @@ #include #include #include +#include #include +#include +#include #include #include @@ -695,6 +698,44 @@ TEST(malloc, mallopt_purge) { #endif } +TEST(malloc, mallopt_purge_all) { +#if defined(__BIONIC__) + SKIP_WITH_HWASAN << "hwasan does not implement mallopt"; + errno = 0; + ASSERT_EQ(1, mallopt(M_PURGE_ALL, 0)); +#else + GTEST_SKIP() << "bionic-only test"; +#endif +} + +// Verify that all of the mallopt values are unique. +TEST(malloc, mallopt_unique_params) { +#if defined(__BIONIC__) + std::vector> params{ + std::make_pair(M_DECAY_TIME, "M_DECAY_TIME"), + std::make_pair(M_PURGE, "M_PURGE"), + std::make_pair(M_PURGE_ALL, "M_PURGE_ALL"), + std::make_pair(M_MEMTAG_TUNING, "M_MEMTAG_TUNING"), + std::make_pair(M_THREAD_DISABLE_MEM_INIT, "M_THREAD_DISABLE_MEM_INIT"), + std::make_pair(M_CACHE_COUNT_MAX, "M_CACHE_COUNT_MAX"), + std::make_pair(M_CACHE_SIZE_MAX, "M_CACHE_SIZE_MAX"), + std::make_pair(M_TSDS_COUNT_MAX, "M_TSDS_COUNT_MAX"), + std::make_pair(M_BIONIC_ZERO_INIT, "M_BIONIC_ZERO_INIT"), + std::make_pair(M_BIONIC_SET_HEAP_TAGGING_LEVEL, "M_BIONIC_SET_HEAP_TAGGING_LEVEL"), + }; + + std::unordered_map all_params; + for (const auto& param : params) { + EXPECT_TRUE(all_params.count(param.first) == 0) + << "mallopt params " << all_params[param.first] << " and " << param.second + << " have the same value " << param.first; + all_params.insert(param); + } +#else + GTEST_SKIP() << "bionic-only test"; +#endif +} + #if defined(__BIONIC__) static void GetAllocatorVersion(bool* allocator_scudo) { TemporaryFile tf;