tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Checkpoint new build orchestrator"

Browse source
This commit is contained in:
Treehugger Robot 2022-05-12 22:55:33 +00:00 committed by Gerrit Code Review
commit a96be433c4
14 changed files with 957 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
envsetup.sh
View file

@ -456,7 +456,7 @@ function multitree_lunch()
if $(echo "$1" | grep -q '^-') ; then
# Calls starting with a -- argument are passed directly and the function
# returns with the lunch.py exit code.
build/make/orchestrator/core/lunch.py "$@"
build/build/make/orchestrator/core/lunch.py "$@"
code=$?
if [[ $code -eq 2 ]] ; then
echo 1>&2
@ -467,7 +467,7 @@ function multitree_lunch()
fi
else
# All other calls go through the --lunch variant of lunch.py
results=($(build/make/orchestrator/core/lunch.py --lunch "$@"))
results=($(build/build/make/orchestrator/core/lunch.py --lunch "$@"))
code=$?
if [[ $code -eq 2 ]] ; then
echo 1>&2
@ -944,6 +944,34 @@ function gettop
fi
}
# TODO: Merge into gettop as part of launching multitree
function multitree_gettop
{
local TOPFILE=build/build/make/core/envsetup.mk
if [ -n "$TOP" -a -f "$TOP/$TOPFILE" ] ; then
# The following circumlocution ensures we remove symlinks from TOP.
(cd "$TOP"; PWD= /bin/pwd)
else
if [ -f $TOPFILE ] ; then
# The following circumlocution (repeated below as well) ensures
# that we record the true directory name and not one that is
# faked up with symlink names.
PWD= /bin/pwd
else
local HERE=$PWD
local T=
while [ \( ! \( -f $TOPFILE \) \) -a \( "$PWD" != "/" \) ]; do
\cd ..
T=`PWD= /bin/pwd -P`
done
\cd "$HERE"
if [ -f "$T/$TOPFILE" ]; then
echo "$T"
fi
fi
fi
}
function croot()
{
local T=$(gettop)
@ -1826,6 +1854,21 @@ function make()
_wrap_build $(get_make_command "$@") "$@"
}
function _multitree_lunch_error()
{
>&2 echo "Couldn't locate the top of the tree. Please run \'source build/envsetup.sh\' and multitree_lunch from the root of your workspace."
}
function multitree_build()
{
if T="$(multitree_gettop)"; then
"$T/build/build/orchestrator/core/orchestrator.py" "$@"
else
_multitree_lunch_error
return 1
fi
}
function provision()
{
if [ ! "$ANDROID_PRODUCT_OUT" ]; then

7
orchestrator/README Normal file
View file

@ -0,0 +1,7 @@
DEMO
from the root of the workspace
ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build
ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build

151
orchestrator/core/api_assembly.py Normal file
View file

@ -0,0 +1,151 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
def assemble_apis(inner_trees):
# Find all of the contributions from the inner tree
contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)
# Load and validate the contribution files
# TODO: Check timestamps and skip unnecessary work
contributions = []
for tree_key, filenames in contribution_files_dict.items():
for filename in filenames:
contribution_data = load_contribution_file(filename)
if not contribution_data:
continue
# TODO: Validate the configs, especially that the domains match what we asked for
# from the lunch config.
contributions.append(contribution_data)
# Group contributions by language and API surface
stub_libraries = collate_contributions(contributions)
# Iterate through all of the stub libraries and generate rules to assemble them
# and Android.bp/BUILD files to make those available to inner trees.
# TODO: Parallelize? Skip unnecessary work?
ninja_file = NinjaFile() # TODO: parameters?
build_file = BuildFile() # TODO: parameters?
for stub_library in stub_libraries:
STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)
# TODO: Handle host_executables separately or as a StubLibrary language?
def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
"Scan an inner_tree's out dir for the api contribution files."
directory = inner_tree.out.api_contributions_dir()
result = []
with os.scandir(directory) as it:
for dirent in it:
if not dirent.is_file():
break
if dirent.name.endswith(".json"):
result.append(os.path.join(directory, dirent.name))
return result
def load_contribution_file(filename):
"Load and return the API contribution at filename. On error report error and return None."
with open(filename) as f:
try:
return json.load(f)
except json.decoder.JSONDecodeError as ex:
# TODO: Error reporting
raise ex
class StubLibraryContribution(object):
def __init__(self, api_domain, library_contribution):
self.api_domain = api_domain
self.library_contribution = library_contribution
class StubLibrary(object):
def __init__(self, language, api_surface, api_surface_version, name):
self.language = language
self.api_surface = api_surface
self.api_surface_version = api_surface_version
self.name = name
self.contributions = []
def add_contribution(self, contrib):
self.contributions.append(contrib)
def collate_contributions(contributions):
"""Take the list of parsed API contribution files, and group targets by API Surface, version,
language and library name, and return a StubLibrary object for each of those.
"""
grouped = {}
for contribution in contributions:
for language in STUB_LANGUAGE_HANDLERS.keys():
for library in contribution.get(language, []):
key = (language, contribution["name"], contribution["version"], library["name"])
stub_library = grouped.get(key)
if not stub_library:
stub_library = StubLibrary(language, contribution["name"],
contribution["version"], library["name"])
grouped[key] = stub_library
stub_library.add_contribution(StubLibraryContribution(
contribution["api_domain"], library))
return list(grouped.values())
def assemble_cc_api_library(ninja_file, build_file, stub_library):
print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
# TODO: Implement me
def assemble_java_api_library(ninja_file, build_file, stub_library):
print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
# TODO: Implement me
def assemble_resource_api_library(ninja_file, build_file, stub_library):
print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
# TODO: Implement me
STUB_LANGUAGE_HANDLERS = {
"cc_libraries": assemble_cc_api_library,
"java_libraries": assemble_java_api_library,
"resource_libraries": assemble_resource_api_library,
}
class NinjaFile(object):
"Generator for build actions and dependencies."
pass
class BuildFile(object):
"Abstract generator for Android.bp files and BUILD files."
pass

28
orchestrator/core/api_domain.py Normal file
View file

@ -0,0 +1,28 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class ApiDomain(object):
def __init__(self, name, tree, product):
# Product will be null for modules
self.name = name
self.tree = tree
self.product = product
def __str__(self):
return "ApiDomain(name=\"%s\" tree.root=\"%s\" product=%s)" % (
self.name, self.tree.root,
"None" if self.product is None else "\"%s\"" % self.product)

20
orchestrator/core/api_export.py Normal file
View file

@ -0,0 +1,20 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def export_apis_from_tree(tree_key, inner_tree, cookie):
inner_tree.invoke(["export_api_contributions"])

155
orchestrator/core/inner_tree.py Normal file
View file

@ -0,0 +1,155 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import sys
import textwrap
class InnerTreeKey(object):
"""Trees are identified uniquely by their root and the TARGET_PRODUCT they will use to build.
If a single tree uses two different prdoucts, then we won't make assumptions about
them sharing _anything_.
TODO: This is true for soong. It's more likely that bazel could do analysis for two
products at the same time in a single tree, so there's an optimization there to do
eventually."""
def __init__(self, root, product):
self.root = root
self.product = product
def __str__(self):
return "TreeKey(root=%s product=%s)" % (enquote(self.root), enquote(self.product))
def __hash__(self):
return hash((self.root, self.product))
def __eq__(self, other):
return (self.root == other.root and self.product == other.product)
def __ne__(self, other):
return not self.__eq__(other)
def __lt__(self, other):
return (self.root, self.product) < (other.root, other.product)
def __le__(self, other):
return (self.root, self.product) <= (other.root, other.product)
def __gt__(self, other):
return (self.root, self.product) > (other.root, other.product)
def __ge__(self, other):
return (self.root, self.product) >= (other.root, other.product)
class InnerTree(object):
def __init__(self, root, product):
"""Initialize with the inner tree root (relative to the workspace root)"""
self.root = root
self.product = product
self.domains = {}
# TODO: Base directory on OUT_DIR
self.out = OutDirLayout(os.path.join("out", "trees", root))
def __str__(self):
return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
enquote(self.product),
" ".join([enquote(d) for d in sorted(self.domains.keys())]))
def invoke(self, args):
"""Call the inner tree command for this inner tree. Exits on failure."""
# TODO: Build time tracing
# Validate that there is a .inner_build command to run at the root of the tree
# so we can print a good error message
inner_build_tool = os.path.join(self.root, ".inner_build")
if not os.access(inner_build_tool, os.X_OK):
sys.stderr.write(("Unable to execute %s. Is there an inner tree or lunch combo"
+ " misconfiguration?\n") % inner_build_tool)
sys.exit(1)
# TODO: This is where we should set up the shared trees
# Build the command
cmd = [inner_build_tool, "--out_dir", self.out.root()]
for domain_name in sorted(self.domains.keys()):
cmd.append("--api_domain")
cmd.append(domain_name)
cmd += args
# Run the command
process = subprocess.run(cmd, shell=False)
# TODO: Probably want better handling of inner tree failures
if process.returncode:
sys.stderr.write("Build error in inner tree: %s\nstopping multitree build.\n"
% self.root)
sys.exit(1)
class InnerTrees(object):
def __init__(self, trees, domains):
self.trees = trees
self.domains = domains
def __str__(self):
"Return a debugging dump of this object"
return textwrap.dedent("""\
InnerTrees {
trees: [
%(trees)s
]
domains: [
%(domains)s
]
}""" % {
"trees": "\n ".join(sorted([str(t) for t in self.trees.values()])),
"domains": "\n ".join(sorted([str(d) for d in self.domains.values()])),
})
def for_each_tree(self, func, cookie=None):
"""Call func for each of the inner trees once for each product that will be built in it.
The calls will be in a stable order.
Return a map of the InnerTreeKey to any results returned from func().
"""
result = {}
for key in sorted(self.trees.keys()):
result[key] = func(key, self.trees[key], cookie)
return result
class OutDirLayout(object):
def __init__(self, root):
"Initialize with the root of the OUT_DIR for the inner tree."
self._root = root
def root(self):
return self._root
def tree_info_file(self):
return os.path.join(self._root, "tree_info.json")
def api_contributions_dir(self):
return os.path.join(self._root, "api_contributions")
def enquote(s):
return "None" if s is None else "\"%s\"" % s

29
orchestrator/core/interrogate.py Normal file
View file

@ -0,0 +1,29 @@
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
def interrogate_tree(tree_key, inner_tree, cookie):
inner_tree.invoke(["describe"])
info_json_filename = inner_tree.out.tree_info_file()
# TODO: Error handling
with open(info_json_filename) as f:
info_json = json.load(f)
# TODO: Check orchestrator protocol

177
orchestrator/core/lunch.py
View file

@ -24,8 +24,10 @@ EXIT_STATUS_OK = 0
EXIT_STATUS_ERROR = 1
EXIT_STATUS_NEED_HELP = 2
def FindDirs(path, name, ttl=6):
"""Search at most ttl directories deep inside path for a directory called name."""
def find_dirs(path, name, ttl=6):
"""Search at most ttl directories deep inside path for a directory called name
and yield directories that match."""
# The dance with subdirs is so that we recurse in sorted order.
subdirs = []
with os.scandir(path) as it:
@ -40,10 +42,10 @@ def FindDirs(path, name, ttl=6):
# Consume filesystem errors, e.g. too many links, permission etc.
pass
for subdir in subdirs:
yield from FindDirs(os.path.join(path, subdir), name, ttl-1)
yield from find_dirs(os.path.join(path, subdir), name, ttl-1)
def WalkPaths(path, matcher, ttl=10):
def walk_paths(path, matcher, ttl=10):
"""Do a traversal of all files under path yielding each file that matches
matcher."""
# First look for files, then recurse into directories as needed.
@ -62,22 +64,22 @@ def WalkPaths(path, matcher, ttl=10):
# Consume filesystem errors, e.g. too many links, permission etc.
pass
for subdir in sorted(subdirs):
yield from WalkPaths(os.path.join(path, subdir), matcher, ttl-1)
yield from walk_paths(os.path.join(path, subdir), matcher, ttl-1)
def FindFile(path, filename):
def find_file(path, filename):
"""Return a file called filename inside path, no more than ttl levels deep.
Directories are searched alphabetically.
"""
for f in WalkPaths(path, lambda x: x == filename):
for f in walk_paths(path, lambda x: x == filename):
return f
def FindConfigDirs(workspace_root):
def find_config_dirs(workspace_root):
"""Find the configuration files in the well known locations inside workspace_root
<workspace_root>/build/orchestrator/multitree_combos
<workspace_root>/build/build/orchestrator/multitree_combos
(AOSP devices, such as cuttlefish)
<workspace_root>/vendor/**/multitree_combos
@ -89,29 +91,30 @@ def FindConfigDirs(workspace_root):
Directories are returned specifically in this order, so that aosp can't be
overridden, but vendor overrides device.
"""
# TODO: This is not looking in inner trees correctly.
# TODO: When orchestrator is in its own git project remove the "make/" here
yield os.path.join(workspace_root, "build/make/orchestrator/multitree_combos")
yield os.path.join(workspace_root, "build/build/make/orchestrator/multitree_combos")
dirs = ["vendor", "device"]
for d in dirs:
yield from FindDirs(os.path.join(workspace_root, d), "multitree_combos")
yield from find_dirs(os.path.join(workspace_root, d), "multitree_combos")
def FindNamedConfig(workspace_root, shortname):
def find_named_config(workspace_root, shortname):
"""Find the config with the given shortname inside workspace_root.
Config directories are searched in the order described in FindConfigDirs,
Config directories are searched in the order described in find_config_dirs,
and inside those directories, alphabetically."""
filename = shortname + ".mcombo"
for config_dir in FindConfigDirs(workspace_root):
found = FindFile(config_dir, filename)
for config_dir in find_config_dirs(workspace_root):
found = find_file(config_dir, filename)
if found:
return found
return None
def ParseProductVariant(s):
def parse_product_variant(s):
"""Split a PRODUCT-VARIANT name, or return None if it doesn't match that pattern."""
split = s.split("-")
if len(split) != 2:
@ -119,15 +122,15 @@ def ParseProductVariant(s):
return split
def ChooseConfigFromArgs(workspace_root, args):
def choose_config_from_args(workspace_root, args):
"""Return the config file we should use for the given argument,
or null if there's no file that matches that."""
if len(args) == 1:
# Prefer PRODUCT-VARIANT syntax so if there happens to be a matching
# file we don't match that.
pv = ParseProductVariant(args[0])
pv = parse_product_variant(args[0])
if pv:
config = FindNamedConfig(workspace_root, pv[0])
config = find_named_config(workspace_root, pv[0])
if config:
return (config, pv[1])
return None, None
@ -139,10 +142,12 @@ def ChooseConfigFromArgs(workspace_root, args):
class ConfigException(Exception):
ERROR_IDENTIFY = "identify"
ERROR_PARSE = "parse"
ERROR_CYCLE = "cycle"
ERROR_VALIDATE = "validate"
def __init__(self, kind, message, locations, line=0):
def __init__(self, kind, message, locations=[], line=0):
"""Error thrown when loading and parsing configurations.
Args:
@ -169,13 +174,13 @@ class ConfigException(Exception):
self.line = line
def LoadConfig(filename):
def load_config(filename):
"""Load a config, including processing the inherits fields.
Raises:
ConfigException on errors
"""
def LoadAndMerge(fn, visited):
def load_and_merge(fn, visited):
with open(fn) as f:
try:
contents = json.load(f)
@ -191,34 +196,74 @@ def LoadConfig(filename):
if parent in visited:
raise ConfigException(ConfigException.ERROR_CYCLE, "Cycle detected in inherits",
visited)
DeepMerge(inherited_data, LoadAndMerge(parent, [parent,] + visited))
deep_merge(inherited_data, load_and_merge(parent, [parent,] + visited))
# Then merge inherited_data into contents, but what's already there will win.
DeepMerge(contents, inherited_data)
deep_merge(contents, inherited_data)
contents.pop("inherits", None)
return contents
return LoadAndMerge(filename, [filename,])
return load_and_merge(filename, [filename,])
def DeepMerge(merged, addition):
def deep_merge(merged, addition):
"""Merge all fields of addition into merged. Pre-existing fields win."""
for k, v in addition.items():
if k in merged:
if isinstance(v, dict) and isinstance(merged[k], dict):
DeepMerge(merged[k], v)
deep_merge(merged[k], v)
else:
merged[k] = v
def Lunch(args):
def make_config_header(config_file, config, variant):
def make_table(rows):
maxcols = max([len(row) for row in rows])
widths = [0] * maxcols
for row in rows:
for i in range(len(row)):
widths[i] = max(widths[i], len(row[i]))
text = []
for row in rows:
rowtext = []
for i in range(len(row)):
cell = row[i]
rowtext.append(str(cell))
rowtext.append(" " * (widths[i] - len(cell)))
rowtext.append(" ")
text.append("".join(rowtext))
return "\n".join(text)
trees = [("Component", "Path", "Product"),
("---------", "----", "-------")]
entry = config.get("system", None)
def add_config_tuple(trees, entry, name):
if entry:
trees.append((name, entry.get("tree"), entry.get("product", "")))
add_config_tuple(trees, config.get("system"), "system")
add_config_tuple(trees, config.get("vendor"), "vendor")
for k, v in config.get("modules", {}).items():
add_config_tuple(trees, v, k)
return """========================================
TARGET_BUILD_COMBO=%(TARGET_BUILD_COMBO)s
TARGET_BUILD_VARIANT=%(TARGET_BUILD_VARIANT)s
%(trees)s
========================================\n""" % {
"TARGET_BUILD_COMBO": config_file,
"TARGET_BUILD_VARIANT": variant,
"trees": make_table(trees),
}
def do_lunch(args):
"""Handle the lunch command."""
# Check that we're at the top of a multitree workspace
# TODO: Choose the right sentinel file
if not os.path.exists("build/make/orchestrator"):
# Check that we're at the top of a multitree workspace by seeing if this script exists.
if not os.path.exists("build/build/make/orchestrator/core/lunch.py"):
sys.stderr.write("ERROR: lunch.py must be run from the root of a multi-tree workspace\n")
return EXIT_STATUS_ERROR
# Choose the config file
config_file, variant = ChooseConfigFromArgs(".", args)
config_file, variant = choose_config_from_args(".", args)
if config_file == None:
sys.stderr.write("Can't find lunch combo file for: %s\n" % " ".join(args))
@ -229,7 +274,7 @@ def Lunch(args):
# Parse the config file
try:
config = LoadConfig(config_file)
config = load_config(config_file)
except ConfigException as ex:
sys.stderr.write(str(ex))
return EXIT_STATUS_ERROR
@ -244,47 +289,81 @@ def Lunch(args):
sys.stdout.write("%s\n" % config_file)
sys.stdout.write("%s\n" % variant)
# Write confirmation message to stderr
sys.stderr.write(make_config_header(config_file, config, variant))
return EXIT_STATUS_OK
def FindAllComboFiles(workspace_root):
def find_all_combo_files(workspace_root):
"""Find all .mcombo files in the prescribed locations in the tree."""
for dir in FindConfigDirs(workspace_root):
for file in WalkPaths(dir, lambda x: x.endswith(".mcombo")):
for dir in find_config_dirs(workspace_root):
for file in walk_paths(dir, lambda x: x.endswith(".mcombo")):
yield file
def IsFileLunchable(config_file):
def is_file_lunchable(config_file):
"""Parse config_file, flatten the inheritance, and return whether it can be
used as a lunch target."""
try:
config = LoadConfig(config_file)
config = load_config(config_file)
except ConfigException as ex:
sys.stderr.write("%s" % ex)
return False
return config.get("lunchable", False)
def FindAllLunchable(workspace_root):
def find_all_lunchable(workspace_root):
"""Find all mcombo files in the tree (rooted at workspace_root) that when
parsed (and inheritance is flattened) have lunchable: true."""
for f in [x for x in FindAllComboFiles(workspace_root) if IsFileLunchable(x)]:
for f in [x for x in find_all_combo_files(workspace_root) if is_file_lunchable(x)]:
yield f
def List():
def load_current_config():
"""Load, validate and return the config as specified in TARGET_BUILD_COMBO. Throws
ConfigException if there is a problem."""
# Identify the config file
config_file = os.environ.get("TARGET_BUILD_COMBO")
if not config_file:
raise ConfigException(ConfigException.ERROR_IDENTIFY,
"TARGET_BUILD_COMBO not set. Run lunch or pass a combo file.")
# Parse the config file
config = load_config(config_file)
# Validate the config file
if not config.get("lunchable", False):
raise ConfigException(ConfigException.ERROR_VALIDATE,
"Lunch config file (or inherited files) does not have the 'lunchable'"
+ " flag set, which means it is probably not a complete lunch spec.",
[config_file,])
# TODO: Validate that:
# - there are no modules called system or vendor
# - everything has all the required files
variant = os.environ.get("TARGET_BUILD_VARIANT")
if not variant:
variant = "eng" # TODO: Is this the right default?
# Validate variant is user, userdebug or eng
return config_file, config, variant
def do_list():
"""Handle the --list command."""
for f in sorted(FindAllLunchable(".")):
for f in sorted(find_all_lunchable(".")):
print(f)
def Print(args):
def do_print(args):
"""Handle the --print command."""
# Parse args
if len(args) == 0:
config_file = os.environ.get("TARGET_BUILD_COMBO")
if not config_file:
sys.stderr.write("TARGET_BUILD_COMBO not set. Run lunch or pass a combo file.\n")
sys.stderr.write("TARGET_BUILD_COMBO not set. Run lunch before building.\n")
return EXIT_STATUS_NEED_HELP
elif len(args) == 1:
config_file = args[0]
@ -293,7 +372,7 @@ def Print(args):
# Parse the config file
try:
config = LoadConfig(config_file)
config = load_config(config_file)
except ConfigException as ex:
sys.stderr.write(str(ex))
return EXIT_STATUS_ERROR
@ -309,15 +388,15 @@ def main(argv):
return EXIT_STATUS_NEED_HELP
if len(argv) == 2 and argv[1] == "--list":
List()
do_list()
return EXIT_STATUS_OK
if len(argv) == 2 and argv[1] == "--print":
return Print(argv[2:])
return do_print(argv[2:])
return EXIT_STATUS_OK
if (len(argv) == 2 or len(argv) == 3) and argv[1] == "--lunch":
return Lunch(argv[2:])
if (len(argv) == 3 or len(argv) == 4) and argv[1] == "--lunch":
return do_lunch(argv[2:])
sys.stderr.write("Unknown lunch command: %s\n" % " ".join(argv[1:]))
return EXIT_STATUS_NEED_HELP

123
orchestrator/core/orchestrator.py Executable file
View file

@ -0,0 +1,123 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import sys
sys.dont_write_bytecode = True
import api_assembly
import api_domain
import api_export
import inner_tree
import interrogate
import lunch
EXIT_STATUS_OK = 0
EXIT_STATUS_ERROR = 1
API_DOMAIN_SYSTEM = "system"
API_DOMAIN_VENDOR = "vendor"
API_DOMAIN_MODULE = "module"
def process_config(lunch_config):
"""Returns a InnerTrees object based on the configuration requested in the lunch config."""
def add(domain_name, tree_root, product):
tree_key = inner_tree.InnerTreeKey(tree_root, product)
if tree_key in trees:
tree = trees[tree_key]
else:
tree = inner_tree.InnerTree(tree_root, product)
trees[tree_key] = tree
domain = api_domain.ApiDomain(domain_name, tree, product)
domains[domain_name] = domain
tree.domains[domain_name] = domain
trees = {}
domains = {}
system_entry = lunch_config.get("system")
if system_entry:
add(API_DOMAIN_SYSTEM, system_entry["tree"], system_entry["product"])
vendor_entry = lunch_config.get("vendor")
if vendor_entry:
add(API_DOMAIN_VENDOR, vendor_entry["tree"], vendor_entry["product"])
for module_name, module_entry in lunch_config.get("modules", []).items():
add(module_name, module_entry["tree"], None)
return inner_tree.InnerTrees(trees, domains)
def build():
#
# Load lunch combo
#
# Read the config file
try:
config_file, config, variant = lunch.load_current_config()
except lunch.ConfigException as ex:
sys.stderr.write("%s\n" % ex)
return EXIT_STATUS_ERROR
sys.stdout.write(lunch.make_config_header(config_file, config, variant))
# Construct the trees and domains dicts
inner_trees = process_config(config)
#
# 1. Interrogate the trees
#
inner_trees.for_each_tree(interrogate.interrogate_tree)
# TODO: Detect bazel-only mode
#
# 2a. API Export
#
inner_trees.for_each_tree(api_export.export_apis_from_tree)
#
# 2b. API Surface Assembly
#
api_assembly.assemble_apis(inner_trees)
#
# 3a. API Domain Analysis
#
#
# 3b. Final Packaging Rules
#
#
# 4. Build Execution
#
#
# Success!
#
return EXIT_STATUS_OK
def main(argv):
return build()
if __name__ == "__main__":
sys.exit(main(sys.argv))
# vim: sts=4:ts=4:sw=4

38
orchestrator/core/test_lunch.py
View file

@ -23,73 +23,73 @@ import lunch
class TestStringMethods(unittest.TestCase):
def test_find_dirs(self):
self.assertEqual([x for x in lunch.FindDirs("test/configs", "multitree_combos")], [
self.assertEqual([x for x in lunch.find_dirs("test/configs", "multitree_combos")], [
"test/configs/build/make/orchestrator/multitree_combos",
"test/configs/device/aa/bb/multitree_combos",
"test/configs/vendor/aa/bb/multitree_combos"])
def test_find_file(self):
# Finds the one in device first because this is searching from the root,
# not using FindNamedConfig.
self.assertEqual(lunch.FindFile("test/configs", "v.mcombo"),
# not using find_named_config.
self.assertEqual(lunch.find_file("test/configs", "v.mcombo"),
"test/configs/device/aa/bb/multitree_combos/v.mcombo")
def test_find_config_dirs(self):
self.assertEqual([x for x in lunch.FindConfigDirs("test/configs")], [
self.assertEqual([x for x in lunch.find_config_dirs("test/configs")], [
"test/configs/build/make/orchestrator/multitree_combos",
"test/configs/vendor/aa/bb/multitree_combos",
"test/configs/device/aa/bb/multitree_combos"])
def test_find_named_config(self):
# Inside build/orchestrator, overriding device and vendor
self.assertEqual(lunch.FindNamedConfig("test/configs", "b"),
self.assertEqual(lunch.find_named_config("test/configs", "b"),
"test/configs/build/make/orchestrator/multitree_combos/b.mcombo")
# Nested dir inside a combo dir
self.assertEqual(lunch.FindNamedConfig("test/configs", "nested"),
self.assertEqual(lunch.find_named_config("test/configs", "nested"),
"test/configs/build/make/orchestrator/multitree_combos/nested/nested.mcombo")
# Inside vendor, overriding device
self.assertEqual(lunch.FindNamedConfig("test/configs", "v"),
self.assertEqual(lunch.find_named_config("test/configs", "v"),
"test/configs/vendor/aa/bb/multitree_combos/v.mcombo")
# Inside device
self.assertEqual(lunch.FindNamedConfig("test/configs", "d"),
self.assertEqual(lunch.find_named_config("test/configs", "d"),
"test/configs/device/aa/bb/multitree_combos/d.mcombo")
# Make sure we don't look too deep (for performance)
self.assertIsNone(lunch.FindNamedConfig("test/configs", "too_deep"))
self.assertIsNone(lunch.find_named_config("test/configs", "too_deep"))
def test_choose_config_file(self):
# Empty string argument
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", [""]),
self.assertEqual(lunch.choose_config_from_args("test/configs", [""]),
(None, None))
# A PRODUCT-VARIANT name
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["v-eng"]),
self.assertEqual(lunch.choose_config_from_args("test/configs", ["v-eng"]),
("test/configs/vendor/aa/bb/multitree_combos/v.mcombo", "eng"))
# A PRODUCT-VARIANT name that conflicts with a file
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["b-eng"]),
self.assertEqual(lunch.choose_config_from_args("test/configs", ["b-eng"]),
("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"))
# A PRODUCT-VARIANT that doesn't exist
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["z-user"]),
self.assertEqual(lunch.choose_config_from_args("test/configs", ["z-user"]),
(None, None))
# An explicit file
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs",
self.assertEqual(lunch.choose_config_from_args("test/configs",
["test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"]),
("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"))
# An explicit file that doesn't exist
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs",
self.assertEqual(lunch.choose_config_from_args("test/configs",
["test/configs/doesnt_exist.mcombo", "eng"]),
(None, None))
# An explicit file without a variant should fail
self.assertEqual(lunch.ChooseConfigFromArgs("test/configs",
self.assertEqual(lunch.choose_config_from_args("test/configs",
["test/configs/build/make/orchestrator/multitree_combos/b.mcombo"]),
("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", None))
@ -97,12 +97,12 @@ class TestStringMethods(unittest.TestCase):
def test_config_cycles(self):
# Test that we catch cycles
with self.assertRaises(lunch.ConfigException) as context:
lunch.LoadConfig("test/configs/parsing/cycles/1.mcombo")
lunch.load_config("test/configs/parsing/cycles/1.mcombo")
self.assertEqual(context.exception.kind, lunch.ConfigException.ERROR_CYCLE)
def test_config_merge(self):
# Test the merge logic
self.assertEqual(lunch.LoadConfig("test/configs/parsing/merge/1.mcombo"), {
self.assertEqual(lunch.load_config("test/configs/parsing/merge/1.mcombo"), {
"in_1": "1",
"in_1_2": "1",
"merged": {"merged_1": "1",
@ -119,7 +119,7 @@ class TestStringMethods(unittest.TestCase):
})
def test_list(self):
self.assertEqual(sorted(lunch.FindAllLunchable("test/configs")),
self.assertEqual(sorted(lunch.find_all_lunchable("test/configs")),
["test/configs/build/make/orchestrator/multitree_combos/b.mcombo"])
if __name__ == "__main__":

56
orchestrator/inner_build/common.py Normal file
View file

@ -0,0 +1,56 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import sys
def _parse_arguments(argv):
argv = argv[1:]
"""Return an argparse options object."""
# Top-level parser
parser = argparse.ArgumentParser(prog=".inner_build")
parser.add_argument("--out_dir", action="store", required=True,
help="root of the output directory for this inner tree's API contributions")
parser.add_argument("--api_domain", action="append", required=True,
help="which API domains are to be built in this inner tree")
subparsers = parser.add_subparsers(required=True, dest="command",
help="subcommands")
# inner_build describe command
describe_parser = subparsers.add_parser("describe",
help="describe the capabilities of this inner tree's build system")
# create the parser for the "b" command
export_parser = subparsers.add_parser("export_api_contributions",
help="export the API contributions of this inner tree")
# Parse the arguments
return parser.parse_args(argv)
class Commands(object):
def Run(self, argv):
"""Parse the command arguments and call the corresponding subcommand method on
this object.
Throws AttributeError if the method for the command wasn't found.
"""
args = _parse_arguments(argv)
return getattr(self, args.command)(args)

143
orchestrator/inner_build/inner_build_demo.py Executable file
View file

@ -0,0 +1,143 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import textwrap
sys.dont_write_bytecode = True
import common
def mkdirs(path):
try:
os.makedirs(path)
except FileExistsError:
pass
class InnerBuildSoong(common.Commands):
def describe(self, args):
mkdirs(args.out_dir)
with open(os.path.join(args.out_dir, "tree_info.json"), "w") as f:
f.write(textwrap.dedent("""\
{
"requires_ninja": true,
"orchestrator_protocol_version": 1
}"""))
def export_api_contributions(self, args):
contributions_dir = os.path.join(args.out_dir, "api_contributions")
mkdirs(contributions_dir)
if "system" in args.api_domain:
with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f:
# 'name: android' is android.jar
f.write(textwrap.dedent("""\
{
"name": "public_api",
"version": 1,
"api_domain": "system",
"cc_libraries": [
{
"name": "libhwui",
"headers": [
{
"root": "frameworks/base/libs/hwui/apex/include",
"files": [
"android/graphics/jni_runtime.h",
"android/graphics/paint.h",
"android/graphics/matrix.h",
"android/graphics/canvas.h",
"android/graphics/renderthread.h",
"android/graphics/bitmap.h",
"android/graphics/region.h"
]
}
],
"api": [
"frameworks/base/libs/hwui/libhwui.map.txt"
]
}
],
"java_libraries": [
{
"name": "android",
"api": [
"frameworks/base/core/api/current.txt"
]
}
],
"resource_libraries": [
{
"name": "android",
"api": "frameworks/base/core/res/res/values/public.xml"
}
],
"host_executables": [
{
"name": "aapt2",
"binary": "out/host/bin/aapt2",
"runfiles": [
"../lib/todo.so"
]
}
]
}"""))
elif "com.android.bionic" in args.api_domain:
with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f:
# 'name: android' is android.jar
f.write(textwrap.dedent("""\
{
"name": "public_api",
"version": 1,
"api_domain": "system",
"cc_libraries": [
{
"name": "libc",
"headers": [
{
"root": "bionic/libc/include",
"files": [
"stdio.h",
"sys/klog.h"
]
}
],
"api": "bionic/libc/libc.map.txt"
}
],
"java_libraries": [
{
"name": "android",
"api": [
"frameworks/base/libs/hwui/api/current.txt"
]
}
]
}"""))
def main(argv):
return InnerBuildSoong().Run(argv)
if __name__ == "__main__":
sys.exit(main(sys.argv))
# vim: sts=4:ts=4:sw=4

37
orchestrator/inner_build/inner_build_soong.py Executable file
View file

@ -0,0 +1,37 @@
#!/usr/bin/python3
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import sys
sys.dont_write_bytecode = True
import common
class InnerBuildSoong(common.Commands):
def describe(self, args):
pass
def export_api_contributions(self, args):
pass
def main(argv):
return InnerBuildSoong().Run(argv)
if __name__ == "__main__":
sys.exit(main(sys.argv))

16
orchestrator/multitree_combos/aosp_cf_arm64_phone.mcombo Normal file
View file

@ -0,0 +1,16 @@
{
"lunchable": true,
"system": {
"tree": "master",
"product": "aosp_cf_arm64_phone"
},
"vendor": {
"tree": "master",
"product": "aosp_cf_arm64_phone"
},
"modules": {
"com.android.bionic": {
"tree": "sc-mainline-prod"
}
}
}