Merge changes I32e726c7,I1dc9a708,I09cc335b,Ifb8a66ab,I0e2c25bc, ...

* changes:
  versioner: whitelist atexit, turn on symbol checking by default.
  versioner: add missing test.
  versioner: fix false positive with functions only available as inlines.
  versioner: improve error output slightly.
  versioner: merge stdout and stderr in the test runner.
  versioner: clean up tests, test runner.
  versioner: ignore functions that are __INTRODUCED_IN_FUTURE.
  versioner: autodetect paths when no specified.
  versioner: improve usage messages.
  Remove __cachectl.

libc/include/sys/cachectl.h

@@ -30,6 +30,5 @@
 
 #ifdef __mips__
 #include <asm/cachectl.h>
-extern int __cachectl (void *addr, __const int nbytes, __const int op);
 #endif
 #endif /* sys/cachectl.h */

tools/versioner/run_tests.py

@@ -20,13 +20,13 @@ def indent(text, spaces=4):
 def run_test(test_name, path):
     os.chdir(path)
     process = subprocess.Popen(
-        ["/bin/sh", "run.sh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (output, error) = process.communicate()
+        ["/bin/sh", "run.sh"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    (output, _) = process.communicate()
 
     if os.path.exists("expected_fail"):
         with open("expected_fail") as f:
             expected_output = f.read()
-            if output != expected_output:
+            if not output.endswith(expected_output):
                 print("{} {}: expected output mismatch".format(
                     prefix_fail, test_name))
                 print("")

tools/versioner/src/DeclarationDatabase.h

@@ -19,6 +19,7 @@
 #include <iostream>
 #include <map>
 #include <set>
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -72,6 +73,31 @@ struct DeclarationAvailability {
   int obsoleted = 0;
 
   void dump(std::ostream& out = std::cout) const {
+    out << describe();
+  }
+
+  bool empty() const {
+    return !(introduced || deprecated || obsoleted);
+  }
+
+  auto tie() const {
+    return std::tie(introduced, deprecated, obsoleted);
+  }
+
+  bool operator==(const DeclarationAvailability& rhs) const {
+    return this->tie() == rhs.tie();
+  }
+
+  bool operator!=(const DeclarationAvailability& rhs) const {
+    return !(*this == rhs);
+  }
+
+  std::string describe() const {
+    if (!(introduced || deprecated || obsoleted)) {
+      return "no availability";
+    }
+
+    std::stringstream out;
     bool need_comma = false;
     auto comma = [&out, &need_comma]() {
       if (!need_comma) {
@@ -93,27 +119,8 @@ struct DeclarationAvailability {
       comma();
       out << "obsoleted = " << obsoleted;
     }
-  }
 
-  bool empty() const {
-    return !(introduced || deprecated || obsoleted);
-  }
-
-  auto tie() const {
-    return std::tie(introduced, deprecated, obsoleted);
-  }
-
-  bool operator==(const DeclarationAvailability& rhs) const {
-    return this->tie() == rhs.tie();
-  }
-
-  bool operator!=(const DeclarationAvailability& rhs) const {
-    return !(*this == rhs);
-  }
-
-  std::string describe() const {
-    return std::string("[") + std::to_string(introduced) + "," + std::to_string(deprecated) + "," +
-           std::to_string(obsoleted) + "]";
+    return out.str();
   }
 };
 
@@ -137,6 +144,26 @@ struct DeclarationLocation {
   bool operator==(const DeclarationLocation& other) const {
     return tie() == other.tie();
   }
+
+  void dump(const std::string& base_path = "", std::ostream& out = std::cout) const {
+    const char* var_type = declarationTypeName(type);
+    const char* declaration_type = is_definition ? "definition" : "declaration";
+    const char* linkage = is_extern ? "extern" : "static";
+
+    std::string stripped_path;
+    if (llvm::StringRef(filename).startswith(base_path)) {
+      stripped_path = filename.substr(base_path.size());
+    } else {
+      stripped_path = filename;
+    }
+
+    out << "        " << linkage << " " << var_type << " " << declaration_type << " @ "
+        << stripped_path << ":" << line_number << ":" << column;
+
+    out << "\t[";
+    availability.dump(out);
+    out << "]\n";
+  }
 };
 
 struct Declaration {
@@ -165,29 +192,7 @@ struct Declaration {
   void dump(const std::string& base_path = "", std::ostream& out = std::cout) const {
     out << "    " << name << " declared in " << locations.size() << " locations:\n";
     for (const DeclarationLocation& location : locations) {
-      const char* var_type = declarationTypeName(location.type);
-      const char* declaration_type = location.is_definition ? "definition" : "declaration";
-      const char* linkage = location.is_extern ? "extern" : "static";
-
-      std::string filename;
-      if (llvm::StringRef(location.filename).startswith(base_path)) {
-        filename = location.filename.substr(base_path.size());
-      } else {
-        filename = location.filename;
-      }
-
-      out << "        " << linkage << " " << var_type << " " << declaration_type << " @ "
-          << filename << ":" << location.line_number << ":" << location.column;
-
-      if (!location.availability.empty()) {
-        out << "\t[";
-        location.availability.dump(out);
-        out << "]";
-      } else {
-        out << "\t[no availability]";
-      }
-
-      out << "\n";
+      location.dump(base_path, out);
     }
   }
 };

tools/versioner/src/versioner.cpp

@@ -272,6 +272,8 @@ static DeclarationDatabase compileHeaders(const std::set<CompilationType>& types
 static bool sanityCheck(const std::set<CompilationType>& types,
                         const DeclarationDatabase& database) {
   bool error = false;
+  std::string cwd = getWorkingDir() + "/";
+
   for (auto outer : database) {
     const std::string& symbol_name = outer.first;
     CompilationType last_type;
@@ -290,7 +292,7 @@ static bool sanityCheck(const std::set<CompilationType>& types,
       bool availability_mismatch = false;
       DeclarationAvailability current_availability;
 
-      // Make sure that all of the availability declarations for this symbol match.
+      // Ensure that all of the availability declarations for this symbol match.
       for (const DeclarationLocation& location : declaration.locations) {
         if (!found_availability) {
           found_availability = true;
@@ -306,7 +308,7 @@ static bool sanityCheck(const std::set<CompilationType>& types,
 
       if (availability_mismatch) {
         printf("%s: availability mismatch for %s\n", symbol_name.c_str(), type.describe().c_str());
-        declaration.dump(getWorkingDir() + "/");
+        declaration.dump(cwd);
       }
 
       if (type.arch != last_type.arch) {
@@ -315,14 +317,31 @@ static bool sanityCheck(const std::set<CompilationType>& types,
         continue;
       }
 
-      // Make sure that availability declarations are consistent across API levels for a given arch.
+      // Ensure that availability declarations are consistent across API levels for a given arch.
       if (last_availability != current_availability) {
         error = true;
-        printf("%s: availability mismatch between %s and %s: %s before, %s after\n",
+        printf("%s: availability mismatch between %s and %s: [%s] before, [%s] after\n",
                symbol_name.c_str(), last_type.describe().c_str(), type.describe().c_str(),
                last_availability.describe().c_str(), current_availability.describe().c_str());
       }
 
+      // Ensure that at most one inline definition of a function exists.
+      std::set<DeclarationLocation> inline_definitions;
+
+      for (const DeclarationLocation& location : declaration.locations) {
+        if (location.is_definition) {
+          inline_definitions.insert(location);
+        }
+      }
+
+      if (inline_definitions.size() > 1) {
+        error = true;
+        printf("%s: multiple inline definitions found:\n", symbol_name.c_str());
+        for (const DeclarationLocation& location : declaration.locations) {
+          location.dump(cwd);
+        }
+      }
+
       last_type = type;
     }
   }
@@ -341,17 +360,15 @@ static bool checkVersions(const std::set<CompilationType>& types,
     arch_types[type.arch].insert(type);
   }
 
+  std::set<std::string> completely_unavailable;
+
   for (const auto& outer : declaration_database) {
     const std::string& symbol_name = outer.first;
     const auto& compilations = outer.second;
 
     auto platform_availability_it = symbol_database.find(symbol_name);
     if (platform_availability_it == symbol_database.end()) {
-      // This currently has lots of false positives (__INTRODUCED_IN_FUTURE, __errordecl, functions
-      // that come from crtbegin, etc.). Only print them with verbose, because of this.
-      if (verbose) {
-        printf("%s: not available in any platform\n", symbol_name.c_str());
-      }
+      completely_unavailable.insert(symbol_name);
       continue;
     }
 
@@ -380,13 +397,13 @@ static bool checkVersions(const std::set<CompilationType>& types,
       bool symbol_available = symbol_availability_it != platform_availability.end();
       if (decl_available) {
         if (!symbol_available) {
-          // Make sure that either it exists in the platform, or an inline definition is visible.
+          // Ensure that either it exists in the platform, or an inline definition is visible.
           if (!declaration.hasDefinition()) {
             missing_symbol.insert(type);
             continue;
           }
         } else {
-          // Make sure that symbols declared as functions/variables actually are.
+          // Ensure that symbols declared as functions/variables actually are.
           switch (declaration.type()) {
             case DeclarationType::inconsistent:
               printf("%s: inconsistent declaration type\n", symbol_name.c_str());
@@ -411,7 +428,7 @@ static bool checkVersions(const std::set<CompilationType>& types,
           }
         }
       } else {
-        // Make sure it's not available in the platform.
+        // Ensure that it's not available in the platform.
         if (symbol_availability_it != platform_availability.end()) {
           printf("%s: symbol should be unavailable in %s (declared with availability %s)\n",
                  symbol_name.c_str(), type.describe().c_str(), availability.describe().c_str());
@@ -467,24 +484,65 @@ static bool checkVersions(const std::set<CompilationType>& types,
     }
   }
 
+  for (const std::string& symbol_name : completely_unavailable) {
+    bool found_inline_definition = false;
+    bool future = false;
+
+    auto symbol_it = declaration_database.find(symbol_name);
+
+    // Ignore inline functions and functions that are tagged as __INTRODUCED_IN_FUTURE.
+    // Ensure that all of the declarations of that function satisfy that.
+    for (const auto& declaration_pair : symbol_it->second) {
+      const Declaration& declaration = declaration_pair.second;
+      DeclarationAvailability availability = declaration.locations.begin()->availability;
+
+      if (availability.introduced >= 10000) {
+        future = true;
+      }
+
+      if (declaration.hasDefinition()) {
+        found_inline_definition = true;
+      }
+    }
+
+    if (future || found_inline_definition) {
+      continue;
+    }
+
+    if (missing_symbol_whitelist.count(symbol_name) != 0) {
+      continue;
+    }
+
+    printf("%s: not available in any platform\n", symbol_name.c_str());
+    failed = true;
+  }
+
   return !failed;
 }
 
-static void usage() {
-  fprintf(stderr, "Usage: versioner [OPTION]... HEADER_PATH [DEPS_PATH]\n");
-  fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n");
-  fprintf(stderr, "\n");
-  fprintf(stderr, "Target specification (defaults to all):\n");
-  fprintf(stderr, "  -a API_LEVEL\tbuild with specified API level (can be repeated)\n");
-  fprintf(stderr, "    \t\tvalid levels are %s\n", Join(supported_levels).c_str());
-  fprintf(stderr, "  -r ARCH\tbuild with specified architecture (can be repeated)\n");
-  fprintf(stderr, "    \t\tvalid architectures are %s\n", Join(supported_archs).c_str());
-  fprintf(stderr, "\n");
-  fprintf(stderr, "Validation:\n");
-  fprintf(stderr, "  -p PATH\tcompare against NDK platform at PATH\n");
-  fprintf(stderr, "  -d\t\tdump symbol availability in libraries\n");
-  fprintf(stderr, "  -v\t\tenable verbose warnings\n");
-  exit(1);
+static void usage(bool help = false) {
+  fprintf(stderr, "Usage: versioner [OPTION]... [HEADER_PATH] [DEPS_PATH]\n");
+  if (!help) {
+    printf("Try 'versioner -h' for more information.\n");
+    exit(1);
+  } else {
+    fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n");
+    fprintf(stderr, "Autodetects paths if HEADER_PATH and DEPS_PATH are not specified\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "Target specification (defaults to all):\n");
+    fprintf(stderr, "  -a API_LEVEL\tbuild with specified API level (can be repeated)\n");
+    fprintf(stderr, "    \t\tvalid levels are %s\n", Join(supported_levels).c_str());
+    fprintf(stderr, "  -r ARCH\tbuild with specified architecture (can be repeated)\n");
+    fprintf(stderr, "    \t\tvalid architectures are %s\n", Join(supported_archs).c_str());
+    fprintf(stderr, "\n");
+    fprintf(stderr, "Validation:\n");
+    fprintf(stderr, "  -p PATH\tcompare against NDK platform at PATH\n");
+    fprintf(stderr, "  -v\t\tenable verbose warnings\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "Miscellaneous:\n");
+    fprintf(stderr, "  -h\t\tdisplay this message\n");
+    exit(0);
+  }
 }
 
 int main(int argc, char** argv) {
@@ -495,7 +553,7 @@ int main(int argc, char** argv) {
   std::set<int> selected_levels;
 
   int c;
-  while ((c = getopt(argc, argv, "a:r:p:n:duv")) != -1) {
+  while ((c = getopt(argc, argv, "a:r:p:vh")) != -1) {
     default_args = false;
     switch (c) {
       case 'a': {
@@ -542,16 +600,45 @@ int main(int argc, char** argv) {
         verbose = true;
         break;
 
+      case 'h':
+        usage(true);
+        break;
+
       default:
         usage();
         break;
     }
   }
 
-  if (argc - optind > 2 || optind >= argc) {
+  if (argc - optind > 2 || optind > argc) {
     usage();
   }
 
+  std::string header_dir;
+  std::string dependency_dir;
+
+  if (optind == argc) {
+    // Neither HEADER_PATH nor DEPS_PATH were specified, so try to figure them out.
+    const char* top = getenv("ANDROID_BUILD_TOP");
+    if (!top) {
+      fprintf(stderr, "versioner: failed to autodetect bionic paths. Is ANDROID_BUILD_TOP set?\n");
+      usage();
+    }
+
+    std::string versioner_dir = std::to_string(top) + "/bionic/tools/versioner";
+    header_dir = versioner_dir + "/current";
+    dependency_dir = versioner_dir + "/dependencies";
+    if (platform_dir.empty()) {
+      platform_dir = versioner_dir + "/platforms";
+    }
+  } else {
+    header_dir = argv[optind];
+
+    if (argc - optind == 2) {
+      dependency_dir = argv[optind + 1];
+    }
+  }
+
   if (selected_levels.empty()) {
     selected_levels = supported_levels;
   }
@@ -560,14 +647,12 @@ int main(int argc, char** argv) {
     selected_architectures = supported_archs;
   }
 
-  std::string dependencies = (argc - optind == 2) ? argv[optind + 1] : "";
-  const char* header_dir = argv[optind];
 
   struct stat st;
-  if (stat(header_dir, &st) != 0) {
-    err(1, "failed to stat '%s'", header_dir);
+  if (stat(header_dir.c_str(), &st) != 0) {
+    err(1, "failed to stat '%s'", header_dir.c_str());
   } else if (!S_ISDIR(st.st_mode)) {
-    errx(1, "'%s' is not a directory", header_dir);
+    errx(1, "'%s' is not a directory", header_dir.c_str());
   }
 
   std::set<CompilationType> compilation_types;
@@ -583,7 +668,7 @@ int main(int argc, char** argv) {
   }
 
   bool failed = false;
-  declaration_database = compileHeaders(compilation_types, header_dir, dependencies, &failed);
+  declaration_database = compileHeaders(compilation_types, header_dir, dependency_dir, &failed);
 
   if (!sanityCheck(compilation_types, declaration_database)) {
     printf("versioner: sanity check failed\n");

tools/versioner/src/versioner.h

@@ -20,6 +20,7 @@
 #include <set>
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 
 extern bool verbose;
 
@@ -55,3 +56,8 @@ static const std::unordered_map<std::string, std::set<std::string>> header_black
   // time64.h #errors when included on LP64 archs.
   { "time64.h", { "arm64", "mips64", "x86_64" } },
 };
+
+static const std::unordered_set<std::string> missing_symbol_whitelist = {
+  // atexit comes from crtbegin.
+  "atexit",
+};

tools/versioner/tests/arch_specific/dependencies/common/foo/foodep.h

@@ -1 +0,0 @@
-typedef int foo_t;

tools/versioner/tests/arch_specific/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -r x86 -a 9
+versioner headers -p platforms -r arm -r x86 -a 9

tools/versioner/tests/compilation_error/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9
+versioner headers -p platforms -r arm -a 9

tools/versioner/tests/errordecl/headers/foo.h

@@ -0,0 +1 @@
+int foo() __attribute__((unavailable));

tools/versioner/tests/errordecl/run.sh

@@ -0,0 +1 @@
+versioner -v headers -p platforms -r arm -a 9

tools/versioner/tests/future/headers/foo.h

@@ -0,0 +1 @@
+int foo() __attribute__((availability(android, introduced = 10000)));

tools/versioner/tests/future/run.sh

@@ -0,0 +1 @@
+versioner -v headers -p platforms -r arm -a 9

tools/versioner/tests/future_arch/headers/foo.h

@@ -0,0 +1,5 @@
+#if defined(__arm__)
+int foo() __attribute__((availability(android, introduced = 9)));
+#else
+int foo() __attribute__((availability(android, introduced = 10000)));
+#endif

tools/versioner/tests/future_arch/platforms/android-9/arch-arm/symbols/libc.so.functions.txt

@@ -0,0 +1 @@
+foo

tools/versioner/tests/future_arch/run.sh

@@ -0,0 +1 @@
+versioner -v headers -p platforms -r arm -r x86 -a 9

tools/versioner/tests/inline/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9 -a 12
+versioner headers -p platforms -r arm -a 9 -a 12

tools/versioner/tests/inline_unavailable/headers/foo.h

@@ -0,0 +1,3 @@
+static int foo() __attribute__((availability(android, introduced = 9))) {
+  return 0;
+}

tools/versioner/tests/inline_unavailable/run.sh

@@ -0,0 +1 @@
+versioner -v headers -p platforms -r arm -a 9

tools/versioner/tests/inline_version_mismatch/expected_fail

@@ -1,2 +1,2 @@
-foo: availability mismatch between arm-9 and arm-12: [9,0,0] before, [10,0,0] after
+foo: availability mismatch between arm-9 and arm-12: [introduced = 9] before, [introduced = 10] after
 versioner: sanity check failed

tools/versioner/tests/inline_version_mismatch/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9 -a 12
+versioner headers -p platforms -r arm -a 9 -a 12

tools/versioner/tests/missing_api/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9 -a 12
+versioner headers -p platforms -r arm -a 9 -a 12

tools/versioner/tests/missing_arch/dependencies/common/foo/foodep.h

@@ -1 +0,0 @@
-typedef int foo_t;

tools/versioner/tests/missing_arch/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -r x86 -a 9
+versioner headers -p platforms -r arm -r x86 -a 9

tools/versioner/tests/multiple_decl/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9
+versioner headers -p platforms -r arm -a 9

tools/versioner/tests/multiple_decl_mismatch/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9
+versioner headers -p platforms -r arm -a 9

tools/versioner/tests/obsoleted/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9 -a 12
+versioner headers -p platforms -r arm -a 9 -a 12

tools/versioner/tests/smoke/run.sh

@@ -1 +1 @@
-versioner headers dependencies -p platforms -r arm -a 9
+versioner headers -p platforms -r arm -a 9

tools/versioner/tests/version_mismatch/expected_fail

@@ -0,0 +1,2 @@
+foo: availability mismatch between arm-9 and arm-12: [introduced = 9] before, [introduced = 10] after
+versioner: sanity check failed

tools/versioner/tests/version_mismatch/headers/foo.h

@@ -0,0 +1,5 @@
+#if __ANDROID_API__ <= 9
+int foo() __attribute__((availability(android, introduced = 9)));
+#else
+int foo() __attribute__((availability(android, introduced = 10)));
+#endif

tools/versioner/tests/version_mismatch/platforms/android-12/arch-arm/symbols/libc.so.functions.txt

@@ -0,0 +1 @@
+foo

tools/versioner/tests/version_mismatch/platforms/android-9/arch-arm/symbols/libc.so.functions.txt

@@ -0,0 +1 @@
+foo

tools/versioner/tests/version_mismatch/run.sh

@@ -0,0 +1 @@
+versioner headers -p platforms -r arm -a 9 -a 12