tequilaOS/platform_bionic
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

versioner: refactor to use CompilerInstance directly.

Browse source 
am: 16016df79f

Change-Id: I142f75a381307b71c0893b04e943d65e0dbea844
This commit is contained in:
Josh Gao 2016-11-15 01:02:51 +00:00 committed by android-build-merger
commit cdfd128911
15 changed files with 251 additions and 162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
libc/include/android/legacy_termios_inlines.h
View file

@ -29,11 +29,12 @@
#ifndef _ANDROID_LEGACY_TERMIOS_INLINES_H_
#define _ANDROID_LEGACY_TERMIOS_INLINES_H_
#include <linux/termios.h>
#include <sys/cdefs.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <linux/termios.h>
#if __ANDROID_API__ < 21
__BEGIN_DECLS

3
libc/include/elf.h
View file

@ -29,11 +29,12 @@
#ifndef _ELF_H
#define _ELF_H
#include <sys/cdefs.h>
#include <linux/auxvec.h>
#include <linux/elf.h>
#include <linux/elf-em.h>
#include <machine/elf_machdep.h>
#include <sys/cdefs.h>
#define ELF32_R_INFO(sym, type) ((((Elf32_Word)sym) << 8) | ((type) & 0xff))
#define ELF64_R_INFO(sym, type) ((((Elf64_Xword)sym) << 32) | ((type) & 0xffffffff))

3
libc/include/link.h
View file

@ -28,10 +28,11 @@
#ifndef _LINK_H_
#define _LINK_H_
#include <elf.h>
#include <sys/cdefs.h>
#include <sys/types.h>
#include <elf.h>
__BEGIN_DECLS
#if defined(__LP64__)

5
libc/include/signal.h
View file

@ -29,12 +29,13 @@
#ifndef _SIGNAL_H_
#define _SIGNAL_H_
#include <sys/cdefs.h>
#include <sys/types.h>
#include <asm/sigcontext.h>
#include <bits/pthread_types.h>
#include <bits/timespec.h>
#include <limits.h>
#include <sys/cdefs.h>
#include <sys/types.h>
#if defined(__LP64__) || defined(__mips__)
/* For 64-bit (and mips), the kernel's struct sigaction doesn't match the POSIX one,

5
libc/include/sys/select.h
View file

@ -29,11 +29,12 @@
#ifndef _SYS_SELECT_H_
#define _SYS_SELECT_H_
#include <linux/time.h>
#include <signal.h>
#include <sys/cdefs.h>
#include <sys/types.h>
#include <linux/time.h>
#include <signal.h>
__BEGIN_DECLS
#define FD_SETSIZE 1024

3
libc/include/sys/signalfd.h
View file

@ -29,9 +29,10 @@
#ifndef _SYS_SIGNALFD_H_
#define _SYS_SIGNALFD_H_
#include <sys/cdefs.h>
#include <linux/signalfd.h>
#include <signal.h>
#include <sys/cdefs.h>
__BEGIN_DECLS

3
libc/include/sys/ucontext.h
View file

@ -29,8 +29,9 @@
#ifndef _SYS_UCONTEXT_H_
#define _SYS_UCONTEXT_H_
#include <signal.h>
#include <sys/cdefs.h>
#include <signal.h>
#include <sys/user.h>
__BEGIN_DECLS

4
libc/include/syslog.h
View file

@ -89,7 +89,11 @@ void closelog(void);
void openlog(const char* _Nullable, int, int);
int setlogmask(int);
void syslog(int, const char* _Nonnull, ...) __printflike(2, 3);
#if defined(__arm__) || defined(__aarch64__) || defined(__x86_64__)
void vsyslog(int, const char* _Nonnull, va_list) __printflike(2, 0);
#else /* defined(__mips__) || defined(__i386__) */
void vsyslog(int, const char* _Nonnull, va_list _Nonnull) __printflike(2, 0);
#endif
__END_DECLS

2
tools/versioner/src/Android.mk
View file

@ -31,7 +31,7 @@ LOCAL_SRC_FILES := \
SymbolDatabase.cpp \
Utils.cpp
LOCAL_SHARED_LIBRARIES := libclang libLLVM
LOCAL_SHARED_LIBRARIES := libclang libLLVM libbase
include $(CLANG_HOST_BUILD_MK)
include $(CLANG_TBLGEN_RULES_MK)

3
tools/versioner/src/DeclarationDatabase.cpp
View file

@ -284,9 +284,8 @@ bool Symbol::hasDeclaration(const CompilationType& type) const {
return false;
}
void HeaderDatabase::parseAST(CompilationType type, ASTUnit* ast) {
void HeaderDatabase::parseAST(CompilationType type, ASTContext& ctx) {
std::unique_lock<std::mutex> lock(this->mutex);
ASTContext& ctx = ast->getASTContext();
Visitor visitor(*this, type, ctx);
visitor.TraverseDecl(ctx.getTranslationUnitDecl());
}

4
tools/versioner/src/DeclarationDatabase.h
View file

@ -30,7 +30,7 @@
#include "Utils.h"
namespace clang {
class ASTUnit;
class ASTContext;
class Decl;
}
@ -219,7 +219,7 @@ class HeaderDatabase {
public:
std::map<std::string, Symbol> symbols;
void parseAST(CompilationType type, clang::ASTUnit* ast);
void parseAST(CompilationType type, clang::ASTContext& ast);
void dump(const std::string& base_path = "", FILE* out = stdout) const {
fprintf(out, "HeaderDatabase contains %zu symbols:\n", symbols.size());

27
tools/versioner/src/Preprocessor.cpp
View file

@ -121,33 +121,6 @@ static std::deque<std::string> readFileLines(const std::string& path) {
return result;
}
static std::string dirname(const std::string& path) {
std::unique_ptr<char, decltype(&free)> path_copy(strdup(path.c_str()), free);
return dirname(path_copy.get());
}
static bool mkdirs(const std::string& path) {
struct stat st;
if (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode)) {
return true;
}
std::string parent = dirname(path);
if (parent == path) {
return false;
}
if (!mkdirs(parent)) {
return false;
}
if (mkdir(path.c_str(), 0700) != 0) {
return false;
}
return true;
}
static void writeFileLines(const std::string& path, const std::deque<std::string>& lines) {
if (!mkdirs(dirname(path))) {
err(1, "failed to create directory '%s'", dirname(path).c_str());

48
tools/versioner/src/Utils.h
View file

@ -16,6 +16,11 @@
#pragma once
#include <errno.h>
#include <libgen.h>
#include <sys/stat.h>
#include <unistd.h>
#include <string>
#include <vector>
@ -24,16 +29,53 @@
std::string getWorkingDir();
std::vector<std::string> collectFiles(const std::string& directory);
static __attribute__((unused)) std::string to_string(const char* c) {
static inline std::string dirname(const std::string& path) {
std::unique_ptr<char, decltype(&free)> path_copy(strdup(path.c_str()), free);
return dirname(path_copy.get());
}
static inline bool is_directory(const std::string& path) {
struct stat st;
if (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode)) {
return true;
}
return false;
}
static inline bool mkdirs(const std::string& path) {
if (is_directory(path)) {
return true;
}
std::string parent = dirname(path);
if (parent == path) {
return false;
}
if (!mkdirs(parent)) {
return false;
}
if (mkdir(path.c_str(), 0700) != 0) {
if (errno != EEXIST) {
return false;
}
return is_directory(path);
}
return true;
}
static inline std::string to_string(const char* c) {
return c;
}
static __attribute__((unused)) const std::string& to_string(const std::string& str) {
static inline const std::string& to_string(const std::string& str) {
return str;
}
template <typename Collection>
static std::string Join(Collection c, const std::string& delimiter = ", ") {
static inline std::string Join(Collection c, const std::string& delimiter = ", ") {
std::string result;
for (const auto& item : c) {
using namespace std;

298
tools/versioner/src/versioner.cpp
View file

@ -23,6 +23,7 @@
#include <unistd.h>
#include <atomic>
#include <functional>
#include <iostream>
#include <map>
#include <memory>
@ -33,10 +34,22 @@
#include <unordered_map>
#include <vector>
#include <clang/AST/ASTConsumer.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/Driver/Compilation.h>
#include <clang/Driver/Driver.h>
#include <clang/Frontend/CompilerInstance.h>
#include <clang/Frontend/CompilerInvocation.h>
#include <clang/Frontend/FrontendAction.h>
#include <clang/Frontend/FrontendActions.h>
#include <clang/Frontend/TextDiagnosticPrinter.h>
#include <clang/Frontend/Utils.h>
#include <clang/FrontendTool/Utils.h>
#include <clang/Tooling/Tooling.h>
#include <llvm/ADT/StringRef.h>
#include <android-base/parseint.h>
#include "Arch.h"
#include "DeclarationDatabase.h"
#include "Preprocessor.h"
@ -46,80 +59,130 @@
using namespace std::string_literals;
using namespace clang;
using namespace clang::tooling;
using namespace tooling;
bool verbose;
static bool add_include;
static int max_thread_count = 48;
class HeaderCompilationDatabase : public CompilationDatabase {
CompilationType type;
std::string cwd;
std::vector<std::string> headers;
std::vector<std::string> include_dirs;
static std::vector<std::string> generateCompileCommand(CompilationType& type,
const std::string& filename,
const std::string& header_dir,
const std::vector<std::string>& include_dirs) {
std::vector<std::string> cmd = { "versioner" };
cmd.push_back("-std=c11");
cmd.push_back("-Wall");
cmd.push_back("-Wextra");
cmd.push_back("-Werror");
cmd.push_back("-Wundef");
cmd.push_back("-Wno-unused-macros");
cmd.push_back("-Wno-unused-function");
cmd.push_back("-Wno-unused-variable");
cmd.push_back("-Wno-unknown-attributes");
cmd.push_back("-Wno-pragma-once-outside-header");
cmd.push_back("-target");
cmd.push_back(arch_targets[type.arch]);
cmd.push_back("-DANDROID");
cmd.push_back("-D__ANDROID_API__="s + std::to_string(type.api_level));
cmd.push_back("-D_FORTIFY_SOURCE=2");
cmd.push_back("-D_GNU_SOURCE");
cmd.push_back("-D_FILE_OFFSET_BITS="s + std::to_string(type.file_offset_bits));
cmd.push_back("-nostdinc");
std::string header_path;
if (add_include) {
const char* top = getenv("ANDROID_BUILD_TOP");
header_path = to_string(top) + "/bionic/libc/include/android/versioning.h";
cmd.push_back("-include");
cmd.push_back(header_path);
}
for (const auto& dir : include_dirs) {
cmd.push_back("-isystem");
cmd.push_back(dir);
}
cmd.push_back(filename);
return cmd;
}
class VersionerASTConsumer : public clang::ASTConsumer {
public:
HeaderCompilationDatabase(CompilationType type, std::string cwd, std::vector<std::string> headers,
std::vector<std::string> include_dirs)
: type(type),
cwd(std::move(cwd)),
headers(std::move(headers)),
include_dirs(std::move(include_dirs)) {
HeaderDatabase* header_database;
CompilationType type;
VersionerASTConsumer(HeaderDatabase* header_database, CompilationType type)
: header_database(header_database), type(type) {
}
CompileCommand generateCompileCommand(const std::string& filename) const {
std::vector<std::string> command = { "clang-tool", filename, "-nostdlibinc" };
for (const auto& dir : include_dirs) {
command.push_back("-isystem");
command.push_back(dir);
}
command.push_back("-std=c11");
command.push_back("-DANDROID");
command.push_back("-D__ANDROID_API__="s + std::to_string(type.api_level));
command.push_back("-D_FORTIFY_SOURCE=2");
command.push_back("-D_GNU_SOURCE");
command.push_back("-Wall");
command.push_back("-Wextra");
command.push_back("-Werror");
command.push_back("-Wundef");
command.push_back("-Wno-unused-macros");
command.push_back("-Wno-unused-function");
command.push_back("-Wno-unused-variable");
command.push_back("-Wno-unknown-attributes");
command.push_back("-Wno-pragma-once-outside-header");
command.push_back("-target");
command.push_back(arch_targets[type.arch]);
if (add_include) {
const char* top = getenv("ANDROID_BUILD_TOP");
std::string header_path = to_string(top) + "/bionic/libc/include/android/versioning.h";
command.push_back("-include");
command.push_back(std::move(header_path));
}
command.push_back("-D_FILE_OFFSET_BITS="s + std::to_string(type.file_offset_bits));
return CompileCommand(cwd, filename, command);
}
std::vector<CompileCommand> getAllCompileCommands() const override {
std::vector<CompileCommand> commands;
for (const std::string& file : headers) {
commands.push_back(generateCompileCommand(file));
}
return commands;
}
std::vector<CompileCommand> getCompileCommands(StringRef file) const override {
std::vector<CompileCommand> commands;
commands.push_back(generateCompileCommand(file));
return commands;
}
std::vector<std::string> getAllFiles() const override {
return headers;
virtual void HandleTranslationUnit(ASTContext& ctx) override {
header_database->parseAST(type, ctx);
}
};
class VersionerASTAction : public clang::ASTFrontendAction {
public:
HeaderDatabase* header_database;
CompilationType type;
VersionerASTAction(HeaderDatabase* header_database, CompilationType type)
: header_database(header_database), type(type) {
}
virtual std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance& Compiler,
llvm::StringRef InFile) override {
return std::make_unique<VersionerASTConsumer>(header_database, type);
}
};
static void compileHeader(HeaderDatabase* header_database, CompilationType type,
const std::string& filename, const std::string& header_dir,
const std::vector<std::string>& include_dirs) {
DiagnosticOptions diagnostic_options;
auto diagnostic_printer = new TextDiagnosticPrinter(llvm::errs(), &diagnostic_options);
IntrusiveRefCntPtr<DiagnosticIDs> diagnostic_ids(new DiagnosticIDs());
IntrusiveRefCntPtr<DiagnosticsEngine> diags(
new DiagnosticsEngine(diagnostic_ids, &diagnostic_options, diagnostic_printer, false));
driver::Driver Driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags);
std::vector<std::string> cmd = generateCompileCommand(type, filename, header_dir, include_dirs);
llvm::SmallVector<const char*, 32> Args;
for (const std::string& str : cmd) {
Args.push_back(str.c_str());
}
std::unique_ptr<driver::Compilation> Compilation(Driver.BuildCompilation(Args));
const driver::Command &Cmd = llvm::cast<driver::Command>(*Compilation->getJobs().begin());
const driver::ArgStringList &CCArgs = Cmd.getArguments();
auto invocation = std::make_unique<CompilerInvocation>();
if (!CompilerInvocation::CreateFromArgs(
*invocation.get(), const_cast<const char**>(CCArgs.data()),
const_cast<const char**>(CCArgs.data()) + CCArgs.size(), *diags)) {
errx(1, "failed to create CompilerInvocation");
}
clang::CompilerInstance Compiler;
Compiler.setInvocation(invocation.release());
Compiler.setDiagnostics(diags.get());
VersionerASTAction versioner_action(header_database, type);
if (!Compiler.ExecuteAction(versioner_action)) {
errx(1, "compilation generated warnings or errors");
}
if (diags->getNumWarnings() || diags->hasErrorOccurred()) {
errx(1, "compilation generated warnings or errors");
}
}
struct CompilationRequirements {
std::vector<std::string> headers;
std::vector<std::string> dependencies;
@ -205,20 +268,51 @@ static std::set<CompilationType> generateCompilationTypes(const std::set<Arch> s
return result;
}
struct Job {
Job(CompilationType type, const std::string& header, const std::vector<std::string>& dependencies)
: type(type), header(header), dependencies(dependencies) {
}
CompilationType type;
const std::string& header;
const std::vector<std::string>& dependencies;
};
static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<CompilationType>& types,
const std::string& header_dir,
const std::string& dependency_dir,
bool* failed) {
constexpr size_t thread_count = 8;
size_t threads_created = 0;
std::mutex mutex;
std::vector<std::thread> threads(thread_count);
const std::string& dependency_dir) {
if (types.empty()) {
errx(1, "compileHeaders received no CompilationTypes");
}
size_t thread_count = max_thread_count;
std::vector<std::thread> threads;
std::vector<Job> jobs;
std::map<CompilationType, HeaderDatabase> header_databases;
std::unordered_map<Arch, CompilationRequirements> requirements;
std::string cwd = getWorkingDir();
bool errors = false;
auto result = std::make_unique<HeaderDatabase>();
auto spawn_threads = [&]() {
thread_count = std::min(thread_count, jobs.size());
for (size_t i = 0; i < thread_count; ++i) {
threads.emplace_back([&jobs, &result, &header_dir, thread_count, i]() {
size_t index = i;
while (index < jobs.size()) {
const auto& job = jobs[index];
compileHeader(result.get(), job.type, job.header, header_dir, job.dependencies);
index += thread_count;
}
});
}
};
auto reap_threads = [&]() {
for (auto& thread : threads) {
thread.join();
}
threads.clear();
};
for (const auto& type : types) {
if (requirements.count(type.arch) == 0) {
@ -226,52 +320,15 @@ static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<Compilation
}
}
auto result = std::make_unique<HeaderDatabase>();
for (const auto& type : types) {
size_t thread_id = threads_created++;
if (thread_id >= thread_count) {
thread_id = thread_id % thread_count;
threads[thread_id].join();
for (CompilationType type : types) {
CompilationRequirements& req = requirements[type.arch];
for (const std::string& header : req.headers) {
jobs.emplace_back(type, header, req.dependencies);
}
threads[thread_id] = std::thread(
[&](CompilationType type) {
const auto& req = requirements[type.arch];
HeaderCompilationDatabase compilation_database(type, cwd, req.headers, req.dependencies);
ClangTool tool(compilation_database, req.headers);
clang::DiagnosticOptions diagnostic_options;
std::vector<std::unique_ptr<ASTUnit>> asts;
tool.buildASTs(asts);
for (const auto& ast : asts) {
clang::DiagnosticsEngine& diagnostics_engine = ast->getDiagnostics();
if (diagnostics_engine.getNumWarnings() || diagnostics_engine.hasErrorOccurred()) {
std::unique_lock<std::mutex> l(mutex);
errors = true;
printf("versioner: compilation failure for %s in %s\n", to_string(type).c_str(),
ast->getOriginalSourceFileName().str().c_str());
}
result->parseAST(type, ast.get());
}
},
type);
}
if (threads_created < thread_count) {
threads.resize(threads_created);
}
for (auto& thread : threads) {
thread.join();
}
if (errors) {
printf("versioner: compilation generated warnings or errors\n");
*failed = errors;
}
spawn_threads();
reap_threads();
return result;
}
@ -492,6 +549,7 @@ static void usage(bool help = false) {
fprintf(stderr, "\n");
fprintf(stderr, "Miscellaneous:\n");
fprintf(stderr, " -d\t\tdump function availability\n");
fprintf(stderr, " -j THREADS\tmaximum number of threads to use\n");
fprintf(stderr, " -h\t\tdisplay this message\n");
exit(0);
}
@ -503,12 +561,12 @@ int main(int argc, char** argv) {
std::string platform_dir;
std::set<Arch> selected_architectures;
std::set<int> selected_levels;
bool dump = false;
std::string preprocessor_output_path;
bool force = false;
bool dump = false;
int c;
while ((c = getopt(argc, argv, "a:r:p:vo:fdhi")) != -1) {
while ((c = getopt(argc, argv, "a:r:p:vo:fdj:hi")) != -1) {
default_args = false;
switch (c) {
case 'a': {
@ -575,6 +633,12 @@ int main(int argc, char** argv) {
dump = true;
break;
case 'j':
if (!android::base::ParseInt<int>(optarg, &max_thread_count, 1)) {
usage();
}
break;
case 'h':
usage(true);
break;
@ -647,10 +711,10 @@ int main(int argc, char** argv) {
symbol_database = parsePlatforms(compilation_types, platform_dir);
}
bool failed = false;
std::unique_ptr<HeaderDatabase> declaration_database =
compileHeaders(compilation_types, header_dir, dependency_dir, &failed);
compileHeaders(compilation_types, header_dir, dependency_dir);
bool failed = false;
if (dump) {
declaration_database->dump(header_dir + "/");
} else {

2
tools/versioner/tests/compilation_error/run.sh
View file

@ -1 +1 @@
versioner headers -p platforms -r arm -a 9 -i
versioner headers -p platforms -r arm -a 9 -i -j1