Merge "Add support for M_PURGE_ALL."

benchmarks/malloc_benchmark.cpp

@@ -36,11 +36,11 @@
 
 #if defined(__BIONIC__)
 
-static void BM_mallopt_purge(benchmark::State& state) {
+static void RunMalloptPurge(benchmark::State& state, int purge_value) {
   static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576};
   static int pagesize = getpagesize();
   mallopt(M_DECAY_TIME, 1);
-  mallopt(M_PURGE, 0);
+  mallopt(M_PURGE_ALL, 0);
   for (auto _ : state) {
     state.PauseTiming();
     std::vector<void*> ptrs;
@@ -63,10 +63,19 @@ static void BM_mallopt_purge(benchmark::State& state) {
     ptrs.clear();
     state.ResumeTiming();
 
-    mallopt(M_PURGE, 0);
+    mallopt(purge_value, 0);
   }
   mallopt(M_DECAY_TIME, 0);
 }
+
+static void BM_mallopt_purge(benchmark::State& state) {
+  RunMalloptPurge(state, M_PURGE);
+}
 BIONIC_BENCHMARK(BM_mallopt_purge);
 
+static void BM_mallopt_purge_all(benchmark::State& state) {
+  RunMalloptPurge(state, M_PURGE_ALL);
+}
+BIONIC_BENCHMARK(BM_mallopt_purge_all);
+
 #endif

benchmarks/malloc_map_benchmark.cpp

@@ -69,7 +69,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) {
   for (auto _ : state) {
 #if defined(__BIONIC__)
     state.PauseTiming();
-    mallopt(M_PURGE, 0);
+    mallopt(M_PURGE_ALL, 0);
     uint64_t rss_bytes_before = 0;
     Gather(&rss_bytes_before);
     state.ResumeTiming();
@@ -80,7 +80,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) {
     }
 #if defined(__BIONIC__)
     state.PauseTiming();
-    mallopt(M_PURGE, 0);
+    mallopt(M_PURGE_ALL, 0);
     Gather(&rss_bytes);
     // Try and record only the memory used in the map.
     rss_bytes -= rss_bytes_before;

benchmarks/malloc_rss_benchmark.cpp

@@ -112,7 +112,7 @@ void StressSizeClass(size_t numThreads, size_t allocSize) {
 
   // Do an explicit purge to ensure we will be more likely to get the actual
   // in-use memory.
-  mallopt(M_PURGE, 0);
+  mallopt(M_PURGE_ALL, 0);
 
   android::meminfo::ProcMemInfo proc_mem(getpid());
   const std::vector<android::meminfo::Vma>& maps = proc_mem.MapsWithoutUsageStats();

libc/bionic/jemalloc_wrapper.cpp

@@ -102,7 +102,7 @@ int je_mallopt(int param, int value) {
       }
     }
     return 1;
-  } else if (param == M_PURGE) {
+  } else if (param == M_PURGE || param == M_PURGE_ALL) {
     // Only clear the current thread cache since there is no easy way to
     // clear the caches of other threads.
     // This must be done first so that cleared allocations get purged

libc/include/malloc.h

@@ -183,7 +183,15 @@ int malloc_info(int __must_be_zero, FILE* _Nonnull __fp) __INTRODUCED_IN(23);
  * Available since API level 28.
  */
 #define M_PURGE (-101)
-
+/**
+ * mallopt() option to immediately purge all possible memory back to
+ * the kernel. This call can take longer than a normal purge since it
+ * examines everything. In some cases, it can take more than twice the
+ * time of a M_PURGE call. The value is ignored.
+ *
+ * Available since API level 34.
+ */
+#define M_PURGE_ALL (-104)
 
 /**
  * mallopt() option to tune the allocator's choice of memory tags to

tests/malloc_test.cpp

@@ -36,7 +36,10 @@
 #include <algorithm>
 #include <atomic>
 #include <functional>
+#include <string>
 #include <thread>
+#include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include <tinyxml2.h>
@@ -695,6 +698,44 @@ TEST(malloc, mallopt_purge) {
 #endif
 }
 
+TEST(malloc, mallopt_purge_all) {
+#if defined(__BIONIC__)
+  SKIP_WITH_HWASAN << "hwasan does not implement mallopt";
+  errno = 0;
+  ASSERT_EQ(1, mallopt(M_PURGE_ALL, 0));
+#else
+  GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
+// Verify that all of the mallopt values are unique.
+TEST(malloc, mallopt_unique_params) {
+#if defined(__BIONIC__)
+  std::vector<std::pair<int, std::string>> params{
+      std::make_pair(M_DECAY_TIME, "M_DECAY_TIME"),
+      std::make_pair(M_PURGE, "M_PURGE"),
+      std::make_pair(M_PURGE_ALL, "M_PURGE_ALL"),
+      std::make_pair(M_MEMTAG_TUNING, "M_MEMTAG_TUNING"),
+      std::make_pair(M_THREAD_DISABLE_MEM_INIT, "M_THREAD_DISABLE_MEM_INIT"),
+      std::make_pair(M_CACHE_COUNT_MAX, "M_CACHE_COUNT_MAX"),
+      std::make_pair(M_CACHE_SIZE_MAX, "M_CACHE_SIZE_MAX"),
+      std::make_pair(M_TSDS_COUNT_MAX, "M_TSDS_COUNT_MAX"),
+      std::make_pair(M_BIONIC_ZERO_INIT, "M_BIONIC_ZERO_INIT"),
+      std::make_pair(M_BIONIC_SET_HEAP_TAGGING_LEVEL, "M_BIONIC_SET_HEAP_TAGGING_LEVEL"),
+  };
+
+  std::unordered_map<int, std::string> all_params;
+  for (const auto& param : params) {
+    EXPECT_TRUE(all_params.count(param.first) == 0)
+        << "mallopt params " << all_params[param.first] << " and " << param.second
+        << " have the same value " << param.first;
+    all_params.insert(param);
+  }
+#else
+  GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
 #if defined(__BIONIC__)
 static void GetAllocatorVersion(bool* allocator_scudo) {
   TemporaryFile tf;