From d86eb8665c4783b8c941b307e98589136ffc7293 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 28 Feb 2023 12:45:54 -0800 Subject: [PATCH] Add support for M_PURGE_ALL. This is a new mallopt option that will force purge absolutely everything no matter how long it takes to purge. Wrote a unit test for the new mallopt, and added a test to help verify that new mallopt parameters do not conflict with each other. Modified some benchmarks to use this new parameter so that we can get better RSS data. Added a new M_PURGE_ALL benchmark. Bug: 243851006 Test: All unit tests pass. Test: Ran changed benchmarks. Change-Id: I1b46a5e6253538108e052d11ee46fd513568adec --- benchmarks/malloc_benchmark.cpp | 15 ++++++++--- benchmarks/malloc_map_benchmark.cpp | 4 +-- benchmarks/malloc_rss_benchmark.cpp | 2 +- libc/bionic/jemalloc_wrapper.cpp | 2 +- libc/include/malloc.h | 10 ++++++- tests/malloc_test.cpp | 41 +++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 8 deletions(-) diff --git a/benchmarks/malloc_benchmark.cpp b/benchmarks/malloc_benchmark.cpp index 18ba52386..e733cd055 100644 --- a/benchmarks/malloc_benchmark.cpp +++ b/benchmarks/malloc_benchmark.cpp @@ -36,11 +36,11 @@ #if defined(__BIONIC__) -static void BM_mallopt_purge(benchmark::State& state) { +static void RunMalloptPurge(benchmark::State& state, int purge_value) { static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576}; static int pagesize = getpagesize(); mallopt(M_DECAY_TIME, 1); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); for (auto _ : state) { state.PauseTiming(); std::vector ptrs; @@ -63,10 +63,19 @@ static void BM_mallopt_purge(benchmark::State& state) { ptrs.clear(); state.ResumeTiming(); - mallopt(M_PURGE, 0); + mallopt(purge_value, 0); } mallopt(M_DECAY_TIME, 0); } + +static void BM_mallopt_purge(benchmark::State& state) { + RunMalloptPurge(state, M_PURGE); +} BIONIC_BENCHMARK(BM_mallopt_purge); +static void BM_mallopt_purge_all(benchmark::State& state) { + RunMalloptPurge(state, M_PURGE_ALL); +} +BIONIC_BENCHMARK(BM_mallopt_purge_all); + #endif diff --git a/benchmarks/malloc_map_benchmark.cpp b/benchmarks/malloc_map_benchmark.cpp index ba4d62c0b..575732539 100644 --- a/benchmarks/malloc_map_benchmark.cpp +++ b/benchmarks/malloc_map_benchmark.cpp @@ -69,7 +69,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) { for (auto _ : state) { #if defined(__BIONIC__) state.PauseTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); uint64_t rss_bytes_before = 0; Gather(&rss_bytes_before); state.ResumeTiming(); @@ -80,7 +80,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) { } #if defined(__BIONIC__) state.PauseTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); Gather(&rss_bytes); // Try and record only the memory used in the map. rss_bytes -= rss_bytes_before; diff --git a/benchmarks/malloc_rss_benchmark.cpp b/benchmarks/malloc_rss_benchmark.cpp index 58f61d971..4b34e72f7 100644 --- a/benchmarks/malloc_rss_benchmark.cpp +++ b/benchmarks/malloc_rss_benchmark.cpp @@ -112,7 +112,7 @@ void StressSizeClass(size_t numThreads, size_t allocSize) { // Do an explicit purge to ensure we will be more likely to get the actual // in-use memory. - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); android::meminfo::ProcMemInfo proc_mem(getpid()); const std::vector& maps = proc_mem.MapsWithoutUsageStats(); diff --git a/libc/bionic/jemalloc_wrapper.cpp b/libc/bionic/jemalloc_wrapper.cpp index ef488eecc..ce3f31420 100644 --- a/libc/bionic/jemalloc_wrapper.cpp +++ b/libc/bionic/jemalloc_wrapper.cpp @@ -102,7 +102,7 @@ int je_mallopt(int param, int value) { } } return 1; - } else if (param == M_PURGE) { + } else if (param == M_PURGE || param == M_PURGE_ALL) { // Only clear the current thread cache since there is no easy way to // clear the caches of other threads. // This must be done first so that cleared allocations get purged diff --git a/libc/include/malloc.h b/libc/include/malloc.h index 02bda608c..6a2d38080 100644 --- a/libc/include/malloc.h +++ b/libc/include/malloc.h @@ -183,7 +183,15 @@ int malloc_info(int __must_be_zero, FILE* _Nonnull __fp) __INTRODUCED_IN(23); * Available since API level 28. */ #define M_PURGE (-101) - +/** + * mallopt() option to immediately purge all possible memory back to + * the kernel. This call can take longer than a normal purge since it + * examines everything. In some cases, it can take more than twice the + * time of a M_PURGE call. The value is ignored. + * + * Available since API level 34. + */ +#define M_PURGE_ALL (-104) /** * mallopt() option to tune the allocator's choice of memory tags to diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp index 63ad99d3d..4e7eb7b05 100644 --- a/tests/malloc_test.cpp +++ b/tests/malloc_test.cpp @@ -36,7 +36,10 @@ #include #include #include +#include #include +#include +#include #include #include @@ -695,6 +698,44 @@ TEST(malloc, mallopt_purge) { #endif } +TEST(malloc, mallopt_purge_all) { +#if defined(__BIONIC__) + SKIP_WITH_HWASAN << "hwasan does not implement mallopt"; + errno = 0; + ASSERT_EQ(1, mallopt(M_PURGE_ALL, 0)); +#else + GTEST_SKIP() << "bionic-only test"; +#endif +} + +// Verify that all of the mallopt values are unique. +TEST(malloc, mallopt_unique_params) { +#if defined(__BIONIC__) + std::vector> params{ + std::make_pair(M_DECAY_TIME, "M_DECAY_TIME"), + std::make_pair(M_PURGE, "M_PURGE"), + std::make_pair(M_PURGE_ALL, "M_PURGE_ALL"), + std::make_pair(M_MEMTAG_TUNING, "M_MEMTAG_TUNING"), + std::make_pair(M_THREAD_DISABLE_MEM_INIT, "M_THREAD_DISABLE_MEM_INIT"), + std::make_pair(M_CACHE_COUNT_MAX, "M_CACHE_COUNT_MAX"), + std::make_pair(M_CACHE_SIZE_MAX, "M_CACHE_SIZE_MAX"), + std::make_pair(M_TSDS_COUNT_MAX, "M_TSDS_COUNT_MAX"), + std::make_pair(M_BIONIC_ZERO_INIT, "M_BIONIC_ZERO_INIT"), + std::make_pair(M_BIONIC_SET_HEAP_TAGGING_LEVEL, "M_BIONIC_SET_HEAP_TAGGING_LEVEL"), + }; + + std::unordered_map all_params; + for (const auto& param : params) { + EXPECT_TRUE(all_params.count(param.first) == 0) + << "mallopt params " << all_params[param.first] << " and " << param.second + << " have the same value " << param.first; + all_params.insert(param); + } +#else + GTEST_SKIP() << "bionic-only test"; +#endif +} + #if defined(__BIONIC__) static void GetAllocatorVersion(bool* allocator_scudo) { TemporaryFile tf;