versioner: use a virtual filesystem for input files.

Use an InMemoryFileSystem to store and share input files across
compilations.

This improves the result of `time versioner` further, from:
    versioner  109.12s user 17.43s system 2433% cpu 5.201 total
to:
    versioner  112.20s user 1.38s system 2416% cpu 4.700 total

Bug: http://b/32748936
Test: python run_tests.py
Change-Id: I72d37b7c30850b8399cc40338247700fe3e7b2f9

tools/versioner/src/Android.mk

@@ -31,7 +31,8 @@ LOCAL_SRC_FILES := \
   Driver.cpp \
   Preprocessor.cpp \
   SymbolDatabase.cpp \
-  Utils.cpp
+  Utils.cpp \
+  VFS.cpp
 
 LOCAL_SHARED_LIBRARIES := libclang libLLVM libbase
 

tools/versioner/src/Driver.cpp

@@ -29,6 +29,7 @@
 #include <clang/AST/ASTConsumer.h>
 #include <clang/Basic/Diagnostic.h>
 #include <clang/Basic/TargetInfo.h>
+#include <clang/Basic/VirtualFileSystem.h>
 #include <clang/Driver/Compilation.h>
 #include <clang/Driver/Driver.h>
 #include <clang/Frontend/CompilerInstance.h>
@@ -92,7 +93,8 @@ static IntrusiveRefCntPtr<DiagnosticsEngine> constructDiags() {
 // Run it once to generate flags for each target, and memoize the results.
 static std::unordered_map<CompilationType, std::vector<std::string>> cc1_flags;
 static const char* filename_placeholder = "__VERSIONER_PLACEHOLDER__";
-static void generateTargetCC1Flags(CompilationType type,
+static void generateTargetCC1Flags(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+                                   CompilationType type,
                                    const std::vector<std::string>& include_dirs) {
   std::vector<std::string> cmd = { "versioner" };
   cmd.push_back("-std=c11");
@@ -122,12 +124,8 @@ static void generateTargetCC1Flags(CompilationType type,
   cmd.push_back("-nostdinc");
 
   if (add_include) {
-    const char* top = getenv("ANDROID_BUILD_TOP");
-    if (!top) {
-      errx(1, "-i passed, but ANDROID_BUILD_TOP is unset");
-    }
     cmd.push_back("-include");
-    cmd.push_back(to_string(top) + "/bionic/libc/include/android/versioning.h");
+    cmd.push_back("android/versioning.h");
   }
 
   for (const auto& dir : include_dirs) {
@@ -138,7 +136,7 @@ static void generateTargetCC1Flags(CompilationType type,
   cmd.push_back(filename_placeholder);
 
   auto diags = constructDiags();
-  driver::Driver driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags);
+  driver::Driver driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags, vfs);
   driver.setCheckInputsExist(false);
 
   llvm::SmallVector<const char*, 32> driver_args;
@@ -192,7 +190,8 @@ static std::vector<const char*> getCC1Command(CompilationType type, const std::s
   return result;
 }
 
-void initializeTargetCC1FlagCache(const std::set<CompilationType>& types,
+void initializeTargetCC1FlagCache(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+                                  const std::set<CompilationType>& types,
                                   const std::unordered_map<Arch, CompilationRequirements>& reqs) {
   if (!cc1_flags.empty()) {
     errx(1, "reinitializing target CC1 flag cache?");
@@ -201,14 +200,14 @@ void initializeTargetCC1FlagCache(const std::set<CompilationType>& types,
   auto start = std::chrono::high_resolution_clock::now();
   std::vector<std::thread> threads;
   for (const CompilationType type : types) {
-    threads.emplace_back([type, &reqs]() {
+    threads.emplace_back([type, &vfs, &reqs]() {
       const auto& arch_req_it = reqs.find(type.arch);
       if (arch_req_it == reqs.end()) {
         errx(1, "CompilationRequirement map missing entry for CompilationType %s",
              to_string(type).c_str());
       }
 
-      generateTargetCC1Flags(type, arch_req_it->second.dependencies);
+      generateTargetCC1Flags(vfs, type, arch_req_it->second.dependencies);
     });
   }
   for (auto& thread : threads) {
@@ -226,7 +225,8 @@ void initializeTargetCC1FlagCache(const std::set<CompilationType>& types,
   }
 }
 
-void compileHeader(HeaderDatabase* header_database, CompilationType type,
+void compileHeader(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+                   HeaderDatabase* header_database, CompilationType type,
                    const std::string& filename) {
   auto diags = constructDiags();
   std::vector<const char*> cc1_flags = getCC1Command(type, filename);
@@ -239,6 +239,7 @@ void compileHeader(HeaderDatabase* header_database, CompilationType type,
   clang::CompilerInstance Compiler;
   Compiler.setInvocation(invocation.release());
   Compiler.setDiagnostics(diags.get());
+  Compiler.setVirtualFileSystem(vfs);
 
   VersionerASTAction versioner_action(header_database, type);
   if (!Compiler.ExecuteAction(versioner_action)) {

tools/versioner/src/Driver.h

@@ -20,16 +20,21 @@
 #include <string>
 #include <unordered_map>
 
+#include <llvm/ADT/IntrusiveRefCntPtr.h>
+
 #include "Arch.h"
 #include "DeclarationDatabase.h"
+#include "VFS.h"
 
 struct CompilationRequirements {
   std::vector<std::string> headers;
   std::vector<std::string> dependencies;
 };
 
-void initializeTargetCC1FlagCache(const std::set<CompilationType>& types,
+void initializeTargetCC1FlagCache(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+                                  const std::set<CompilationType>& types,
                                   const std::unordered_map<Arch, CompilationRequirements>& reqs);
 
-void compileHeader(HeaderDatabase* header_database, CompilationType type,
+void compileHeader(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+                   HeaderDatabase* header_database, CompilationType type,
                    const std::string& filename);

tools/versioner/src/VFS.cpp

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <err.h>
+#include <fcntl.h>
+#include <fts.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+
+#include <android-base/unique_fd.h>
+#include <clang/Basic/VirtualFileSystem.h>
+#include <llvm/ADT/IntrusiveRefCntPtr.h>
+#include <llvm/Support/MemoryBuffer.h>
+
+#include "Utils.h"
+
+using android::base::unique_fd;
+using namespace clang::vfs;
+
+static void addDirectoryToVFS(InMemoryFileSystem* vfs, const std::string& path) {
+  char* paths[] = { const_cast<char*>(path.c_str()), nullptr };
+  FTS* fts = fts_open(paths, FTS_COMFOLLOW | FTS_LOGICAL | FTS_NOCHDIR, nullptr);
+  if (!fts) {
+    err(1, "failed to open directory %s", path.c_str());
+  }
+
+  while (FTSENT* ent = fts_read(fts)) {
+    if ((ent->fts_info & FTS_F) == 0) {
+      continue;
+    }
+
+    const char* file_path = ent->fts_accpath;
+    unique_fd fd(open(file_path, O_RDONLY | O_CLOEXEC));
+    if (fd == -1) {
+      err(1, "failed to open header '%s'", file_path);
+    }
+
+    auto buffer_opt = llvm::MemoryBuffer::getOpenFileSlice(fd, file_path, -1, 0);
+    if (!buffer_opt) {
+      errx(1, "failed to map header '%s'", file_path);
+    }
+
+    if (!vfs->addFile(file_path, ent->fts_statp->st_mtim.tv_sec, std::move(buffer_opt.get()))) {
+      errx(1, "failed to add file '%s'", file_path);
+    }
+  }
+
+  fts_close(fts);
+}
+
+llvm::IntrusiveRefCntPtr<FileSystem> createCommonVFS(const std::string& header_dir,
+                                                     const std::string& dependency_dir,
+                                                     bool add_versioning_header) {
+  auto vfs = std::make_unique<InMemoryFileSystem>();
+  addDirectoryToVFS(vfs.get(), header_dir);
+  if (!dependency_dir.empty()) {
+    addDirectoryToVFS(vfs.get(), dependency_dir);
+  }
+
+  if (add_versioning_header) {
+    const char* top = getenv("ANDROID_BUILD_TOP");
+    if (!top) {
+      errx(1, "-i passed, but ANDROID_BUILD_TOP is unset");
+    }
+
+    std::string header_path = std::string(top) + "/bionic/libc/include/android/versioning.h";
+    auto buffer_opt = llvm::MemoryBuffer::getFile(header_path);
+    if (!buffer_opt) {
+      err(1, "failed to open %s", header_path.c_str());
+    }
+    vfs->addFile("android/versioning.h", 0, std::move(buffer_opt.get()));
+  }
+
+  return llvm::IntrusiveRefCntPtr<FileSystem>(vfs.release());
+}

tools/versioner/src/VFS.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <llvm/ADT/IntrusiveRefCntPtr.h>
+#include <clang/Basic/VirtualFileSystem.h>
+
+llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> createCommonVFS(const std::string& header_dir,
+                                                                 const std::string& dependency_dir,
+                                                                 bool add_versioning_header);

tools/versioner/src/versioner.cpp

@@ -44,6 +44,8 @@
 #include "Preprocessor.h"
 #include "SymbolDatabase.h"
 #include "Utils.h"
+#include "VFS.h"
+
 #include "versioner.h"
 
 using namespace std::string_literals;
@@ -138,6 +140,8 @@ static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<Compilation
     errx(1, "compileHeaders received no CompilationTypes");
   }
 
+  auto vfs = createCommonVFS(header_dir, dependency_dir, add_include);
+
   size_t thread_count = max_thread_count;
   std::vector<std::thread> threads;
 
@@ -151,7 +155,7 @@ static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<Compilation
     }
   }
 
-  initializeTargetCC1FlagCache(types, requirements);
+  initializeTargetCC1FlagCache(vfs, types, requirements);
 
   std::vector<std::pair<CompilationType, const std::string&>> jobs;
   for (CompilationType type : types) {
@@ -165,16 +169,16 @@ static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<Compilation
 
   if (thread_count == 1) {
     for (const auto& job : jobs) {
-      compileHeader(result.get(), job.first, job.second);
+      compileHeader(vfs, result.get(), job.first, job.second);
     }
   } else {
     // Spawn threads.
     for (size_t i = 0; i < thread_count; ++i) {
-      threads.emplace_back([&jobs, &result, &header_dir, thread_count, i]() {
+      threads.emplace_back([&jobs, &result, &header_dir, vfs, thread_count, i]() {
         size_t index = i;
         while (index < jobs.size()) {
           const auto& job = jobs[index];
-          compileHeader(result.get(), job.first, job.second);
+          compileHeader(vfs, result.get(), job.first, job.second);
           index += thread_count;
         }
       });