Merge "Add android_mallopt M_GET_DECAY_TIME_ENABLED." into main am: 35aaed9e1b am: be90376e0c

Original change: https://android-review.googlesource.com/c/platform/bionic/+/2810911

Change-Id: Ic944aaa8c76e0dcc19ccb56b39ddcc853d08fd4a
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

benchmarks/Android.bp

@@ -72,6 +72,7 @@ cc_defaults {
 
     target: {
         android: {
+            header_libs: ["bionic_libc_platform_headers"],
             static_libs: [
                 "libmeminfo",
                 "libprocinfo",

benchmarks/ScopedDecayTimeRestorer.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <malloc.h>
+
+#if defined(__BIONIC__)
+
+#include "platform/bionic/malloc.h"
+
+class ScopedDecayTimeRestorer {
+ public:
+  ScopedDecayTimeRestorer() {
+    bool value;
+    if (android_mallopt(M_GET_DECAY_TIME_ENABLED, &value, sizeof(value))) {
+      saved_value_ = value ? 1 : 0;
+    }
+  }
+
+  virtual ~ScopedDecayTimeRestorer() { mallopt(M_DECAY_TIME, saved_value_); }
+
+ private:
+  int saved_value_ = 0;
+};
+
+#endif

benchmarks/malloc_benchmark.cpp

@@ -36,11 +36,14 @@
 #include <vector>
 
 #include <benchmark/benchmark.h>
+#include "ScopedDecayTimeRestorer.h"
 #include "util.h"
 
 #if defined(__BIONIC__)
 
 static void RunMalloptPurge(benchmark::State& state, int purge_value) {
+  ScopedDecayTimeRestorer restorer;
+
   static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576};
   static int pagesize = getpagesize();
   mallopt(M_DECAY_TIME, 1);
@@ -69,7 +72,6 @@ static void RunMalloptPurge(benchmark::State& state, int purge_value) {
 
     mallopt(purge_value, 0);
   }
-  mallopt(M_DECAY_TIME, 0);
 }
 
 static void RunThreadsThroughput(benchmark::State& state, size_t size, size_t num_threads) {

benchmarks/malloc_sql_benchmark.cpp

@@ -31,6 +31,7 @@
 #include <unistd.h>
 
 #include <benchmark/benchmark.h>
+#include "ScopedDecayTimeRestorer.h"
 #include "util.h"
 
 #if defined(__BIONIC__)
@@ -104,6 +105,8 @@ void BenchmarkMalloc(MallocEntry entries[], size_t total_entries, size_t max_all
 #include "malloc_sql.h"
 
 static void BM_malloc_sql_trace_default(benchmark::State& state) {
+  ScopedDecayTimeRestorer restorer;
+
   // The default is expected to be a zero decay time.
   mallopt(M_DECAY_TIME, 0);
 
@@ -115,14 +118,14 @@ static void BM_malloc_sql_trace_default(benchmark::State& state) {
 BIONIC_BENCHMARK(BM_malloc_sql_trace_default);
 
 static void BM_malloc_sql_trace_decay1(benchmark::State& state) {
+  ScopedDecayTimeRestorer restorer;
+
   mallopt(M_DECAY_TIME, 1);
 
   for (auto _ : state) {
     BenchmarkMalloc(g_sql_entries, sizeof(g_sql_entries) / sizeof(MallocEntry),
                     kMaxSqlAllocSlots);
   }
-
-  mallopt(M_DECAY_TIME, 0);
 }
 BIONIC_BENCHMARK(BM_malloc_sql_trace_decay1);
 

benchmarks/stdlib_benchmark.cpp

@@ -22,6 +22,7 @@
 #include <unistd.h>
 
 #include <benchmark/benchmark.h>
+#include "ScopedDecayTimeRestorer.h"
 #include "util.h"
 
 static void MallocFree(benchmark::State& state) {
@@ -40,6 +41,8 @@ static void MallocFree(benchmark::State& state) {
 
 static void BM_stdlib_malloc_free_default(benchmark::State& state) {
 #if defined(__BIONIC__)
+  ScopedDecayTimeRestorer restorer;
+
   // The default is expected to be a zero decay time.
   mallopt(M_DECAY_TIME, 0);
 #endif
@@ -50,11 +53,11 @@ BIONIC_BENCHMARK_WITH_ARG(BM_stdlib_malloc_free_default, "AT_COMMON_SIZES");
 
 #if defined(__BIONIC__)
 static void BM_stdlib_malloc_free_decay1(benchmark::State& state) {
+  ScopedDecayTimeRestorer restorer;
+
   mallopt(M_DECAY_TIME, 1);
 
   MallocFree(state);
-
-  mallopt(M_DECAY_TIME, 0);
 }
 BIONIC_BENCHMARK_WITH_ARG(BM_stdlib_malloc_free_decay1, "AT_COMMON_SIZES");
 #endif
@@ -75,6 +78,8 @@ static void CallocFree(benchmark::State& state) {
 
 static void BM_stdlib_calloc_free_default(benchmark::State& state) {
 #if defined(__BIONIC__)
+  ScopedDecayTimeRestorer restorer;
+
   // The default is expected to be a zero decay time.
   mallopt(M_DECAY_TIME, 0);
 #endif
@@ -113,8 +118,9 @@ static void MallocMultiple(benchmark::State& state, size_t nbytes, size_t numAll
 }
 
 void BM_stdlib_malloc_forty_default(benchmark::State& state) {
-
 #if defined(__BIONIC__)
+  ScopedDecayTimeRestorer restorer;
+
   // The default is expected to be a zero decay time.
   mallopt(M_DECAY_TIME, 0);
 #endif
@@ -125,17 +131,19 @@ BIONIC_BENCHMARK_WITH_ARG(BM_stdlib_malloc_forty_default, "AT_COMMON_SIZES");
 
 #if defined(__BIONIC__)
 void BM_stdlib_malloc_forty_decay1(benchmark::State& state) {
+  ScopedDecayTimeRestorer restorer;
+
   mallopt(M_DECAY_TIME, 1);
 
   MallocMultiple(state, state.range(0), 40);
-
-  mallopt(M_DECAY_TIME, 0);
 }
 BIONIC_BENCHMARK_WITH_ARG(BM_stdlib_malloc_forty_decay1, "AT_COMMON_SIZES");
 #endif
 
 void BM_stdlib_malloc_multiple_8192_allocs_default(benchmark::State& state) {
 #if defined(__BIONIC__)
+  ScopedDecayTimeRestorer restorer;
+
   // The default is expected to be a zero decay time.
   mallopt(M_DECAY_TIME, 0);
 #endif
@@ -146,11 +154,11 @@ BIONIC_BENCHMARK_WITH_ARG(BM_stdlib_malloc_multiple_8192_allocs_default, "AT_SMA
 
 #if defined(__BIONIC__)
 void BM_stdlib_malloc_multiple_8192_allocs_decay1(benchmark::State& state) {
+  ScopedDecayTimeRestorer restorer;
+
   mallopt(M_DECAY_TIME, 1);
 
   MallocMultiple(state, 8192, state.range(0));
-
-  mallopt(M_DECAY_TIME, 0);
 }
 BIONIC_BENCHMARK_WITH_ARG(BM_stdlib_malloc_multiple_8192_allocs_decay1, "AT_SMALL_SIZES");
 #endif

libc/bionic/malloc_common.cpp

@@ -110,12 +110,27 @@ extern "C" int mallopt(int param, int value) {
   if (param == M_BIONIC_ZERO_INIT) {
     return SetHeapZeroInitialize(value);
   }
+
   // The rest we pass on...
+  int retval;
   auto dispatch_table = GetDispatchTable();
   if (__predict_false(dispatch_table != nullptr)) {
-    return dispatch_table->mallopt(param, value);
+    retval = dispatch_table->mallopt(param, value);
+  } else {
+    retval = Malloc(mallopt)(param, value);
   }
-  return Malloc(mallopt)(param, value);
+
+  // Track the M_DECAY_TIME mallopt calls.
+  if (param == M_DECAY_TIME && retval == 1) {
+    __libc_globals.mutate([value](libc_globals* globals) {
+      if (value == 0) {
+        atomic_store(&globals->decay_time_enabled, false);
+      } else {
+        atomic_store(&globals->decay_time_enabled, true);
+      }
+    });
+  }
+  return retval;
 }
 
 extern "C" void* malloc(size_t bytes) {
@@ -341,6 +356,14 @@ extern "C" bool android_mallopt(int opcode, void* arg, size_t arg_size) {
     *reinterpret_cast<bool*>(arg) = atomic_load(&__libc_globals->memtag_stack);
     return true;
   }
+  if (opcode == M_GET_DECAY_TIME_ENABLED) {
+    if (arg == nullptr || arg_size != sizeof(bool)) {
+      errno = EINVAL;
+      return false;
+    }
+    *reinterpret_cast<bool*>(arg) = atomic_load(&__libc_globals->decay_time_enabled);
+    return true;
+  }
   errno = ENOTSUP;
   return false;
 }

libc/bionic/malloc_common_dynamic.cpp

@@ -543,6 +543,14 @@ extern "C" bool android_mallopt(int opcode, void* arg, size_t arg_size) {
     *reinterpret_cast<bool*>(arg) = atomic_load(&__libc_globals->memtag_stack);
     return true;
   }
+  if (opcode == M_GET_DECAY_TIME_ENABLED) {
+    if (arg == nullptr || arg_size != sizeof(bool)) {
+      errno = EINVAL;
+      return false;
+    }
+    *reinterpret_cast<bool*>(arg) = atomic_load(&__libc_globals->decay_time_enabled);
+    return true;
+  }
   // Try heapprofd's mallopt, as it handles options not covered here.
   return HeapprofdMallopt(opcode, arg, arg_size);
 }

libc/platform/bionic/malloc.h

@@ -104,6 +104,13 @@ enum {
   // Query whether memtag stack is enabled for this process.
   M_MEMTAG_STACK_IS_ON = 11,
 #define M_MEMTAG_STACK_IS_ON M_MEMTAG_STACK_IS_ON
+  // Query whether the current process has the decay time enabled so that
+  // the memory from allocations are not immediately released to the OS.
+  // Result is assigned to the arg pointer's destination.
+  //   arg = bool*
+  //   arg_size = sizeof(bool)
+  M_GET_DECAY_TIME_ENABLED = 12,
+#define M_GET_DECAY_TIME_ENABLED M_GET_DECAY_TIME_ENABLED
 };
 
 #pragma clang diagnostic push

libc/private/bionic_globals.h

@@ -49,6 +49,7 @@ struct libc_globals {
   long setjmp_cookie;
   uintptr_t heap_pointer_tag;
   _Atomic(bool) memtag_stack;
+  _Atomic(bool) decay_time_enabled;
 
   // In order to allow a complete switch between dispatch tables without
   // the need for copying each function by function in the structure,

tests/malloc_test.cpp

@@ -1734,3 +1734,36 @@ TEST(malloc, zeroed_allocations_realloc) {
     }
   }
 }
+
+TEST(android_mallopt, get_decay_time_enabled_errors) {
+#if defined(__BIONIC__)
+  errno = 0;
+  EXPECT_FALSE(android_mallopt(M_GET_DECAY_TIME_ENABLED, nullptr, sizeof(bool)));
+  EXPECT_ERRNO(EINVAL);
+
+  errno = 0;
+  int value;
+  EXPECT_FALSE(android_mallopt(M_GET_DECAY_TIME_ENABLED, &value, sizeof(value)));
+  EXPECT_ERRNO(EINVAL);
+#else
+  GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
+TEST(android_mallopt, get_decay_time_enabled) {
+#if defined(__BIONIC__)
+  SKIP_WITH_HWASAN << "hwasan does not implement mallopt";
+
+  EXPECT_EQ(1, mallopt(M_DECAY_TIME, 0));
+
+  bool value;
+  EXPECT_TRUE(android_mallopt(M_GET_DECAY_TIME_ENABLED, &value, sizeof(value)));
+  EXPECT_FALSE(value);
+
+  EXPECT_EQ(1, mallopt(M_DECAY_TIME, 1));
+  EXPECT_TRUE(android_mallopt(M_GET_DECAY_TIME_ENABLED, &value, sizeof(value)));
+  EXPECT_TRUE(value);
+#else
+  GTEST_SKIP() << "bionic-only test";
+#endif
+}