tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Generate ninja files for api assembly and run ninja.

Browse source 
Change-Id: I003536e4ed4481acbdc041a6450106f8459d5978
This commit is contained in:
Joe Onorato 2022-05-12 12:58:10 -07:00
parent 69b1b0c10a
commit 51497185fd
10 changed files with 360 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
orchestrator/core/api_assembly.py
View file

@ -14,11 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import json
import os
import sys
def assemble_apis(inner_trees):
import api_assembly_cc
import ninja_tools
ContributionData = collections.namedtuple("ContributionData", ("inner_tree", "json_data"))
def assemble_apis(context, inner_trees):
# Find all of the contributions from the inner tree
contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)
@ -27,25 +34,33 @@ def assemble_apis(inner_trees):
contributions = []
for tree_key, filenames in contribution_files_dict.items():
for filename in filenames:
contribution_data = load_contribution_file(filename)
if not contribution_data:
json_data = load_contribution_file(filename)
if not json_data:
continue
# TODO: Validate the configs, especially that the domains match what we asked for
# from the lunch config.
contributions.append(contribution_data)
contributions.append(ContributionData(inner_trees.get(tree_key), json_data))
# Group contributions by language and API surface
stub_libraries = collate_contributions(contributions)
# Iterate through all of the stub libraries and generate rules to assemble them
# and Android.bp/BUILD files to make those available to inner trees.
# TODO: Parallelize? Skip unnecessary work?
ninja_file = NinjaFile() # TODO: parameters?
build_file = BuildFile() # TODO: parameters?
for stub_library in stub_libraries:
STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)
# Initialize the ninja file writer
with open(context.out.api_ninja_file(), "w") as ninja_file:
ninja = ninja_tools.Ninja(context, ninja_file)
# TODO: Handle host_executables separately or as a StubLibrary language?
# Initialize the build file writer
build_file = BuildFile() # TODO: parameters?
# Iterate through all of the stub libraries and generate rules to assemble them
# and Android.bp/BUILD files to make those available to inner trees.
# TODO: Parallelize? Skip unnecessary work?
for stub_library in stub_libraries:
STUB_LANGUAGE_HANDLERS[stub_library.language](context, ninja, build_file, stub_library)
# TODO: Handle host_executables separately or as a StubLibrary language?
# Finish writing the ninja file
ninja.write()
def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
@ -72,7 +87,8 @@ def load_contribution_file(filename):
class StubLibraryContribution(object):
def __init__(self, api_domain, library_contribution):
def __init__(self, inner_tree, api_domain, library_contribution):
self.inner_tree = inner_tree
self.api_domain = api_domain
self.library_contribution = library_contribution
@ -96,54 +112,42 @@ def collate_contributions(contributions):
grouped = {}
for contribution in contributions:
for language in STUB_LANGUAGE_HANDLERS.keys():
for library in contribution.get(language, []):
key = (language, contribution["name"], contribution["version"], library["name"])
for library in contribution.json_data.get(language, []):
key = (language, contribution.json_data["name"],
contribution.json_data["version"], library["name"])
stub_library = grouped.get(key)
if not stub_library:
stub_library = StubLibrary(language, contribution["name"],
contribution["version"], library["name"])
stub_library = StubLibrary(language, contribution.json_data["name"],
contribution.json_data["version"], library["name"])
grouped[key] = stub_library
stub_library.add_contribution(StubLibraryContribution(
contribution["api_domain"], library))
stub_library.add_contribution(StubLibraryContribution(contribution.inner_tree,
contribution.json_data["api_domain"], library))
return list(grouped.values())
def assemble_cc_api_library(ninja_file, build_file, stub_library):
print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
stub_library.name))
def assemble_java_api_library(context, ninja, build_file, stub_library):
print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface,
stub_library.api_surface_version, stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
# TODO: Implement me
def assemble_java_api_library(ninja_file, build_file, stub_library):
print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
# TODO: Implement me
def assemble_resource_api_library(ninja_file, build_file, stub_library):
print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
stub_library.name))
def assemble_resource_api_library(context, ninja, build_file, stub_library):
print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface,
stub_library.api_surface_version, stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
# TODO: Implement me
STUB_LANGUAGE_HANDLERS = {
"cc_libraries": assemble_cc_api_library,
"cc_libraries": api_assembly_cc.assemble_cc_api_library,
"java_libraries": assemble_java_api_library,
"resource_libraries": assemble_resource_api_library,
}
class NinjaFile(object):
"Generator for build actions and dependencies."
pass
class BuildFile(object):
"Abstract generator for Android.bp files and BUILD files."
pass

55
orchestrator/core/api_assembly_cc.py Normal file
View file

@ -0,0 +1,55 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
def assemble_cc_api_library(context, ninja, build_file, stub_library):
print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,
stub_library.api_surface_version, stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution))
staging_dir = context.out.api_library_dir(stub_library.api_surface,
stub_library.api_surface_version, stub_library.name)
work_dir = context.out.api_library_work_dir(stub_library.api_surface,
stub_library.api_surface_version, stub_library.name)
print("staging_dir=%s" % (staging_dir))
print("work_dir=%s" % (work_dir))
# Generate rules to copy headers
includes = []
include_dir = os.path.join(staging_dir, "include")
for contrib in stub_library.contributions:
for headers in contrib.library_contribution["headers"]:
root = headers["root"]
for file in headers["files"]:
# TODO: Deal with collisions of the same name from multiple contributions
include = os.path.join(include_dir, file)
ninja.add_copy_file(include, os.path.join(contrib.inner_tree.root, root, file))
includes.append(include)
# Generate rule to run ndkstubgen
# Generate rule to compile stubs to library
# Generate phony rule to build the library
# TODO: This name probably conflictgs with something
ninja.add_phony("-".join((stub_library.api_surface, str(stub_library.api_surface_version),
stub_library.name)), includes)
# Generate build files

29
orchestrator/core/final_packaging.py Normal file
View file

@ -0,0 +1,29 @@
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ninja_tools
import ninja_syntax # Has to be after ninja_tools because of the path hack
def final_packaging(context):
"""Pull together all of the previously defined rules into the final build stems."""
with open(context.out.outer_ninja_file(), "w") as ninja_file:
ninja = ninja_tools.Ninja(context, ninja_file)
# Add the api surfaces file
ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))
# Finish writing the ninja file
ninja.write()

11
orchestrator/core/inner_tree.py
View file

@ -56,13 +56,13 @@ class InnerTreeKey(object):
class InnerTree(object):
def __init__(self, root, product):
def __init__(self, context, root, product):
"""Initialize with the inner tree root (relative to the workspace root)"""
self.root = root
self.product = product
self.domains = {}
# TODO: Base directory on OUT_DIR
self.out = OutDirLayout(os.path.join("out", "trees", root))
self.out = OutDirLayout(context.out.inner_tree_dir(root))
def __str__(self):
return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
@ -134,7 +134,14 @@ class InnerTrees(object):
return result
def get(self, tree_key):
"""Get an inner tree for tree_key"""
return self.trees.get(tree_key)
class OutDirLayout(object):
"""Encapsulates the logic about the layout of the inner tree out directories.
See also context.OutDir for outer tree out dir contents."""
def __init__(self, root):
"Initialize with the root of the OUT_DIR for the inner tree."
self._root = root

36
orchestrator/core/ninja_runner.py Normal file
View file

@ -0,0 +1,36 @@
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess
def run_ninja(context, targets):
"""Run ninja.
"""
# Construct the command
cmd = [
context.tools.ninja(),
"-f",
context.out.outer_ninja_file(),
] + targets
# Run the command
process = subprocess.run(cmd, shell=False)
# TODO: Probably want better handling of inner tree failures
if process.returncode:
sys.stderr.write("Build error in outer tree.\nstopping multitree build.\n")
sys.exit(1)

46
orchestrator/core/ninja_tools.py Normal file
View file

@ -0,0 +1,46 @@
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
# Workaround for python include path
_ninja_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "ninja"))
if _ninja_dir not in sys.path:
sys.path.append(_ninja_dir)
import ninja_writer
from ninja_syntax import Variable, BuildAction, Rule, Pool, Subninja, Line
class Ninja(ninja_writer.Writer):
"""Some higher level constructs on top of raw ninja writing.
TODO: Not sure where these should be."""
def __init__(self, context, file):
super(Ninja, self).__init__(file)
self._context = context
self._did_copy_file = False
def add_copy_file(self, copy_to, copy_from):
if not self._did_copy_file:
self._did_copy_file = True
rule = Rule("copy_file")
rule.add_variable("command", "mkdir -p ${out_dir} && " + self._context.tools.acp()
+ " -f ${in} ${out}")
self.add_rule(rule)
build_action = BuildAction(copy_to, "copy_file", inputs=[copy_from,],
implicits=[self._context.tools.acp()])
build_action.add_variable("out_dir", os.path.dirname(copy_to))
self.add_build_action(build_action)

22
orchestrator/core/orchestrator.py
View file

@ -22,9 +22,12 @@ sys.dont_write_bytecode = True
import api_assembly
import api_domain
import api_export
import final_packaging
import inner_tree
import interrogate
import lunch
import ninja_runner
import utils
EXIT_STATUS_OK = 0
EXIT_STATUS_ERROR = 1
@ -33,14 +36,14 @@ API_DOMAIN_SYSTEM = "system"
API_DOMAIN_VENDOR = "vendor"
API_DOMAIN_MODULE = "module"
def process_config(lunch_config):
def process_config(context, lunch_config):
"""Returns a InnerTrees object based on the configuration requested in the lunch config."""
def add(domain_name, tree_root, product):
tree_key = inner_tree.InnerTreeKey(tree_root, product)
if tree_key in trees:
tree = trees[tree_key]
else:
tree = inner_tree.InnerTree(tree_root, product)
tree = inner_tree.InnerTree(context, tree_root, product)
trees[tree_key] = tree
domain = api_domain.ApiDomain(domain_name, tree, product)
domains[domain_name] = domain
@ -68,6 +71,9 @@ def build():
# Load lunch combo
#
# Choose the out directory, set up error handling, etc.
context = utils.Context(utils.choose_out_dir(), utils.Errors(sys.stderr))
# Read the config file
try:
config_file, config, variant = lunch.load_current_config()
@ -77,7 +83,7 @@ def build():
sys.stdout.write(lunch.make_config_header(config_file, config, variant))
# Construct the trees and domains dicts
inner_trees = process_config(config)
inner_trees = process_config(context, config)
#
# 1. Interrogate the trees
@ -93,7 +99,7 @@ def build():
#
# 2b. API Surface Assembly
#
api_assembly.assemble_apis(inner_trees)
api_assembly.assemble_apis(context, inner_trees)
#
# 3a. API Domain Analysis
@ -102,11 +108,17 @@ def build():
#
# 3b. Final Packaging Rules
#
final_packaging.final_packaging(context)
#
# 4. Build Execution
#
# TODO: Decide what we want the UX for selecting targets to be across
# branches... since there are very likely to be conflicting soong short
# names.
print("Running ninja...")
targets = ["public_api-1-libhwui", "public_api-1-libc"]
ninja_runner.run_ninja(context, targets)
#
# Success!

120
orchestrator/core/utils.py Normal file
View file

@ -0,0 +1,120 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import platform
class Context(object):
"""Mockable container for global state."""
def __init__(self, out_root, errors):
self.out = OutDir(out_root)
self.errors = errors
self.tools = HostTools()
class TestContext(Context):
"Context for testing. The real Context is manually constructed in orchestrator.py."
def __init__(self, test_work_dir, test_name):
super(MockContext, self).__init__(os.path.join(test_work_dir, test_name),
Errors(None))
class OutDir(object):
"""Encapsulates the logic about the out directory at the outer-tree level.
See also inner_tree.OutDirLayout for inner tree out dir contents."""
def __init__(self, root):
"Initialize with the root of the OUT_DIR for the outer tree."
self._root = root
self._intermediates = "intermediates"
def root(self):
return self._root
def inner_tree_dir(self, tree_root):
"""Root directory for inner tree inside the out dir."""
return os.path.join(self._root, "trees", tree_root)
def api_ninja_file(self):
"""The ninja file that assembles API surfaces."""
return os.path.join(self._root, "api_surfaces.ninja")
def api_library_dir(self, surface, version, library):
"""Directory for all the contents of a library inside an API surface, including
the build files. Any intermediates should go in api_library_work_dir."""
return os.path.join(self._root, "api_surfaces", surface, str(version), library)
def api_library_work_dir(self, surface, version, library):
"""Intermediates / scratch directory for library inside an API surface."""
return os.path.join(self._root, self._intermediates, "api_surfaces", surface, str(version),
library)
def outer_ninja_file(self):
return os.path.join(self._root, "multitree.ninja")
class Errors(object):
"""Class for reporting and tracking errors."""
def __init__(self, stream):
"""Initialize Error reporter with a file-like object."""
self._stream = stream
self._all = []
def error(self, message):
"""Record the error message."""
s = str(s)
if s[-1] != "\n":
s += "\n"
self._all.append(s)
if self._stream:
self._stream.write(s)
def had_error(self):
"""Return if there were any errors reported."""
return len(self._all)
def get_errors(self):
"""Get all errors that were reported."""
return self._all
class HostTools(object):
def __init__(self):
if platform.system() == "Linux":
self._arch = "linux-x86"
else:
raise Exception("Orchestrator running on an unknown system: %s" % platform.system())
# Some of these are called a lot, so pre-compute the strings to save memory
self._prebuilts = os.path.join("build", "prebuilts", "build-tools", self._arch, "bin")
self._acp = os.path.join(self._prebuilts, "acp")
self._ninja = os.path.join(self._prebuilts, "ninja")
def acp(self):
return self._acp
def ninja(self):
return self._ninja
def choose_out_dir():
"""Get the root of the out dir, either from the environment or by picking
a default."""
result = os.environ.get("OUT_DIR")
if result:
return result
else:
return "out"

2
orchestrator/ninja/ninja_syntax.py
View file

@ -159,7 +159,7 @@ class Subninja(Node):
self.chDir = chDir
# TODO(spandandas): Update the syntax when aosp/2064612 lands
def stream() -> Iterator[str]:
def stream(self) -> Iterator[str]:
yield f"subninja {self.subninja}"
class Line(Node):

4
orchestrator/ninja/ninja_writer.py
View file

@ -49,6 +49,10 @@ class Writer:
def add_subninja(self, subninja: Subninja):
self.nodes.append(subninja)
def add_phony(self, name, deps):
build_action = BuildAction(name, "phony", inputs=deps)
self.add_build_action(build_action)
def write(self):
for node in self.nodes:
for line in node.stream():