Merge "Checkpoint new build orchestrator"

envsetup.sh

@@ -456,7 +456,7 @@ function multitree_lunch()
     if $(echo "$1" | grep -q '^-') ; then
         # Calls starting with a -- argument are passed directly and the function
         # returns with the lunch.py exit code.
-        build/make/orchestrator/core/lunch.py "$@"
+        build/build/make/orchestrator/core/lunch.py "$@"
         code=$?
         if [[ $code -eq 2 ]] ; then
           echo 1>&2
@@ -467,7 +467,7 @@ function multitree_lunch()
         fi
     else
         # All other calls go through the --lunch variant of lunch.py
-        results=($(build/make/orchestrator/core/lunch.py --lunch "$@"))
+        results=($(build/build/make/orchestrator/core/lunch.py --lunch "$@"))
         code=$?
         if [[ $code -eq 2 ]] ; then
           echo 1>&2
@@ -944,6 +944,34 @@ function gettop
     fi
 }
 
+# TODO: Merge into gettop as part of launching multitree
+function multitree_gettop
+{
+    local TOPFILE=build/build/make/core/envsetup.mk
+    if [ -n "$TOP" -a -f "$TOP/$TOPFILE" ] ; then
+        # The following circumlocution ensures we remove symlinks from TOP.
+        (cd "$TOP"; PWD= /bin/pwd)
+    else
+        if [ -f $TOPFILE ] ; then
+            # The following circumlocution (repeated below as well) ensures
+            # that we record the true directory name and not one that is
+            # faked up with symlink names.
+            PWD= /bin/pwd
+        else
+            local HERE=$PWD
+            local T=
+            while [ \( ! \( -f $TOPFILE \) \) -a \( "$PWD" != "/" \) ]; do
+                \cd ..
+                T=`PWD= /bin/pwd -P`
+            done
+            \cd "$HERE"
+            if [ -f "$T/$TOPFILE" ]; then
+                echo "$T"
+            fi
+        fi
+    fi
+}
+
 function croot()
 {
     local T=$(gettop)
@@ -1826,6 +1854,21 @@ function make()
     _wrap_build $(get_make_command "$@") "$@"
 }
 
+function _multitree_lunch_error()
+{
+      >&2 echo "Couldn't locate the top of the tree. Please run \'source build/envsetup.sh\' and multitree_lunch from the root of your workspace."
+}
+
+function multitree_build()
+{
+    if T="$(multitree_gettop)"; then
+      "$T/build/build/orchestrator/core/orchestrator.py" "$@"
+    else
+      _multitree_lunch_error
+      return 1
+    fi
+}
+
 function provision()
 {
     if [ ! "$ANDROID_PRODUCT_OUT" ]; then

orchestrator/README

@@ -0,0 +1,7 @@
+DEMO
+
+from the root of the workspace
+
+ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build
+ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build
+

orchestrator/core/api_assembly.py

@@ -0,0 +1,151 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+def assemble_apis(inner_trees):
+
+    # Find all of the contributions from the inner tree
+    contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)
+
+    # Load and validate the contribution files
+    # TODO: Check timestamps and skip unnecessary work
+    contributions = []
+    for tree_key, filenames in contribution_files_dict.items():
+        for filename in filenames:
+            contribution_data = load_contribution_file(filename)
+            if not contribution_data:
+                continue
+            # TODO: Validate the configs, especially that the domains match what we asked for
+            # from the lunch config.
+            contributions.append(contribution_data)
+
+    # Group contributions by language and API surface
+    stub_libraries = collate_contributions(contributions)
+
+    # Iterate through all of the stub libraries and generate rules to assemble them
+    # and Android.bp/BUILD files to make those available to inner trees.
+    # TODO: Parallelize? Skip unnecessary work?
+    ninja_file = NinjaFile() # TODO: parameters?
+    build_file = BuildFile() # TODO: parameters?
+    for stub_library in stub_libraries:
+        STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)
+
+    # TODO: Handle host_executables separately or as a StubLibrary language?
+
+
+def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
+    "Scan an inner_tree's out dir for the api contribution files."
+    directory = inner_tree.out.api_contributions_dir()
+    result = []
+    with os.scandir(directory) as it:
+        for dirent in it:
+            if not dirent.is_file():
+                break
+            if dirent.name.endswith(".json"):
+                result.append(os.path.join(directory, dirent.name))
+    return result
+
+
+def load_contribution_file(filename):
+    "Load and return the API contribution at filename. On error report error and return None."
+    with open(filename) as f:
+        try:
+            return json.load(f)
+        except json.decoder.JSONDecodeError as ex:
+            # TODO: Error reporting
+            raise ex
+
+
+class StubLibraryContribution(object):
+    def __init__(self, api_domain, library_contribution):
+        self.api_domain = api_domain
+        self.library_contribution = library_contribution
+
+
+class StubLibrary(object):
+    def __init__(self, language, api_surface, api_surface_version, name):
+        self.language = language
+        self.api_surface = api_surface
+        self.api_surface_version = api_surface_version
+        self.name = name
+        self.contributions = []
+
+    def add_contribution(self, contrib):
+        self.contributions.append(contrib)
+
+
+def collate_contributions(contributions):
+    """Take the list of parsed API contribution files, and group targets by API Surface, version,
+    language and library name, and return a StubLibrary object for each of those.
+    """
+    grouped = {}
+    for contribution in contributions:
+        for language in STUB_LANGUAGE_HANDLERS.keys():
+            for library in contribution.get(language, []):
+                key = (language, contribution["name"], contribution["version"], library["name"])
+                stub_library = grouped.get(key)
+                if not stub_library:
+                    stub_library = StubLibrary(language, contribution["name"],
+                            contribution["version"], library["name"])
+                    grouped[key] = stub_library
+                stub_library.add_contribution(StubLibraryContribution(
+                        contribution["api_domain"], library))
+    return list(grouped.values())
+
+
+def assemble_cc_api_library(ninja_file, build_file, stub_library):
+    print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
+            stub_library.name))
+    for contrib in stub_library.contributions:
+        print("  %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
+    # TODO: Implement me
+
+
+def assemble_java_api_library(ninja_file, build_file, stub_library):
+    print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
+            stub_library.name))
+    for contrib in stub_library.contributions:
+        print("  %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
+    # TODO: Implement me
+
+
+def assemble_resource_api_library(ninja_file, build_file, stub_library):
+    print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
+            stub_library.name))
+    for contrib in stub_library.contributions:
+        print("  %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
+    # TODO: Implement me
+
+
+STUB_LANGUAGE_HANDLERS = {
+    "cc_libraries": assemble_cc_api_library,
+    "java_libraries": assemble_java_api_library,
+    "resource_libraries": assemble_resource_api_library,
+}
+
+
+class NinjaFile(object):
+    "Generator for build actions and dependencies."
+    pass
+
+
+class BuildFile(object):
+    "Abstract generator for Android.bp files and BUILD files."
+    pass
+
+

orchestrator/core/api_domain.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class ApiDomain(object):
+    def __init__(self, name, tree, product):
+        # Product will be null for modules
+        self.name = name
+        self.tree = tree
+        self.product = product
+
+    def __str__(self):
+        return "ApiDomain(name=\"%s\" tree.root=\"%s\" product=%s)" % (
+                self.name, self.tree.root,
+                "None" if self.product is None else "\"%s\"" % self.product)
+

orchestrator/core/api_export.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def export_apis_from_tree(tree_key, inner_tree, cookie):
+    inner_tree.invoke(["export_api_contributions"])
+
+

orchestrator/core/inner_tree.py

@@ -0,0 +1,155 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+import textwrap
+
+class InnerTreeKey(object):
+    """Trees are identified uniquely by their root and the TARGET_PRODUCT they will use to build.
+    If a single tree uses two different prdoucts, then we won't make assumptions about
+    them sharing _anything_.
+    TODO: This is true for soong. It's more likely that bazel could do analysis for two
+    products at the same time in a single tree, so there's an optimization there to do
+    eventually."""
+    def __init__(self, root, product):
+        self.root = root
+        self.product = product
+
+    def __str__(self):
+        return "TreeKey(root=%s product=%s)" % (enquote(self.root), enquote(self.product))
+
+    def __hash__(self):
+        return hash((self.root, self.product))
+
+    def __eq__(self, other):
+        return (self.root == other.root and self.product == other.product)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __lt__(self, other):
+        return (self.root, self.product) < (other.root, other.product)
+
+    def __le__(self, other):
+        return (self.root, self.product) <= (other.root, other.product)
+
+    def __gt__(self, other):
+        return (self.root, self.product) > (other.root, other.product)
+
+    def __ge__(self, other):
+        return (self.root, self.product) >= (other.root, other.product)
+
+
+class InnerTree(object):
+    def __init__(self, root, product):
+        """Initialize with the inner tree root (relative to the workspace root)"""
+        self.root = root
+        self.product = product
+        self.domains = {}
+        # TODO: Base directory on OUT_DIR
+        self.out = OutDirLayout(os.path.join("out", "trees", root))
+
+    def __str__(self):
+        return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
+                enquote(self.product),
+                " ".join([enquote(d) for d in sorted(self.domains.keys())]))
+
+    def invoke(self, args):
+        """Call the inner tree command for this inner tree. Exits on failure."""
+        # TODO: Build time tracing
+
+        # Validate that there is a .inner_build command to run at the root of the tree
+        # so we can print a good error message
+        inner_build_tool = os.path.join(self.root, ".inner_build")
+        if not os.access(inner_build_tool, os.X_OK):
+            sys.stderr.write(("Unable to execute %s. Is there an inner tree or lunch combo"
+                    + " misconfiguration?\n") % inner_build_tool)
+            sys.exit(1)
+
+        # TODO: This is where we should set up the shared trees
+
+        # Build the command
+        cmd = [inner_build_tool, "--out_dir", self.out.root()]
+        for domain_name in sorted(self.domains.keys()):
+            cmd.append("--api_domain")
+            cmd.append(domain_name)
+        cmd += args
+
+        # Run the command
+        process = subprocess.run(cmd, shell=False)
+
+        # TODO: Probably want better handling of inner tree failures
+        if process.returncode:
+            sys.stderr.write("Build error in inner tree: %s\nstopping multitree build.\n"
+                    % self.root)
+            sys.exit(1)
+
+
+class InnerTrees(object):
+    def __init__(self, trees, domains):
+        self.trees = trees
+        self.domains = domains
+
+    def __str__(self):
+        "Return a debugging dump of this object"
+        return textwrap.dedent("""\
+        InnerTrees {
+            trees: [
+                %(trees)s
+            ]
+            domains: [
+                %(domains)s
+            ]
+        }""" % {
+            "trees": "\n        ".join(sorted([str(t) for t in self.trees.values()])),
+            "domains": "\n        ".join(sorted([str(d) for d in self.domains.values()])),
+        })
+
+
+    def for_each_tree(self, func, cookie=None):
+        """Call func for each of the inner trees once for each product that will be built in it.
+
+        The calls will be in a stable order.
+
+        Return a map of the InnerTreeKey to any results returned from func().
+        """
+        result = {}
+        for key in sorted(self.trees.keys()):
+            result[key] = func(key, self.trees[key], cookie)
+        return result
+
+
+class OutDirLayout(object):
+    def __init__(self, root):
+        "Initialize with the root of the OUT_DIR for the inner tree."
+        self._root = root
+
+    def root(self):
+        return self._root
+
+    def tree_info_file(self):
+        return os.path.join(self._root, "tree_info.json")
+
+    def api_contributions_dir(self):
+        return os.path.join(self._root, "api_contributions")
+
+
+def enquote(s):
+    return "None" if s is None else "\"%s\"" % s
+
+

orchestrator/core/interrogate.py

@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+def interrogate_tree(tree_key, inner_tree, cookie):
+    inner_tree.invoke(["describe"])
+
+    info_json_filename = inner_tree.out.tree_info_file()
+
+    # TODO: Error handling
+    with open(info_json_filename) as f:
+        info_json = json.load(f)
+
+    # TODO: Check orchestrator protocol
+

orchestrator/core/lunch.py

@@ -24,8 +24,10 @@ EXIT_STATUS_OK = 0
 EXIT_STATUS_ERROR = 1
 EXIT_STATUS_NEED_HELP = 2
 
-def FindDirs(path, name, ttl=6):
+
-    """Search at most ttl directories deep inside path for a directory called name."""
+def find_dirs(path, name, ttl=6):
+    """Search at most ttl directories deep inside path for a directory called name
+    and yield directories that match."""
     # The dance with subdirs is so that we recurse in sorted order.
     subdirs = []
     with os.scandir(path) as it:
@@ -40,10 +42,10 @@ def FindDirs(path, name, ttl=6):
                 # Consume filesystem errors, e.g. too many links, permission etc.
                 pass
     for subdir in subdirs:
-        yield from FindDirs(os.path.join(path, subdir), name, ttl-1)
+        yield from find_dirs(os.path.join(path, subdir), name, ttl-1)
 
 
-def WalkPaths(path, matcher, ttl=10):
+def walk_paths(path, matcher, ttl=10):
     """Do a traversal of all files under path yielding each file that matches
     matcher."""
     # First look for files, then recurse into directories as needed.
@@ -62,22 +64,22 @@ def WalkPaths(path, matcher, ttl=10):
                 # Consume filesystem errors, e.g. too many links, permission etc.
                 pass
     for subdir in sorted(subdirs):
-        yield from WalkPaths(os.path.join(path, subdir), matcher, ttl-1)
+        yield from walk_paths(os.path.join(path, subdir), matcher, ttl-1)
 
 
-def FindFile(path, filename):
+def find_file(path, filename):
     """Return a file called filename inside path, no more than ttl levels deep.
 
     Directories are searched alphabetically.
     """
-    for f in WalkPaths(path, lambda x: x == filename):
+    for f in walk_paths(path, lambda x: x == filename):
         return f
 
 
-def FindConfigDirs(workspace_root):
+def find_config_dirs(workspace_root):
     """Find the configuration files in the well known locations inside workspace_root
 
-        <workspace_root>/build/orchestrator/multitree_combos
+        <workspace_root>/build/build/orchestrator/multitree_combos
            (AOSP devices, such as cuttlefish)
 
         <workspace_root>/vendor/**/multitree_combos
@@ -89,29 +91,30 @@ def FindConfigDirs(workspace_root):
     Directories are returned specifically in this order, so that aosp can't be
     overridden, but vendor overrides device.
     """
+    # TODO: This is not looking in inner trees correctly.
 
     # TODO: When orchestrator is in its own git project remove the "make/" here
-    yield os.path.join(workspace_root, "build/make/orchestrator/multitree_combos")
+    yield os.path.join(workspace_root, "build/build/make/orchestrator/multitree_combos")
 
     dirs = ["vendor", "device"]
     for d in dirs:
-        yield from FindDirs(os.path.join(workspace_root, d), "multitree_combos")
+        yield from find_dirs(os.path.join(workspace_root, d), "multitree_combos")
 
 
-def FindNamedConfig(workspace_root, shortname):
+def find_named_config(workspace_root, shortname):
     """Find the config with the given shortname inside workspace_root.
 
-    Config directories are searched in the order described in FindConfigDirs,
+    Config directories are searched in the order described in find_config_dirs,
     and inside those directories, alphabetically."""
     filename = shortname + ".mcombo"
-    for config_dir in FindConfigDirs(workspace_root):
+    for config_dir in find_config_dirs(workspace_root):
-        found = FindFile(config_dir, filename)
+        found = find_file(config_dir, filename)
         if found:
             return found
     return None
 
 
-def ParseProductVariant(s):
+def parse_product_variant(s):
     """Split a PRODUCT-VARIANT name, or return None if it doesn't match that pattern."""
     split = s.split("-")
     if len(split) != 2:
@@ -119,15 +122,15 @@ def ParseProductVariant(s):
     return split
 
 
-def ChooseConfigFromArgs(workspace_root, args):
+def choose_config_from_args(workspace_root, args):
     """Return the config file we should use for the given argument,
     or null if there's no file that matches that."""
     if len(args) == 1:
         # Prefer PRODUCT-VARIANT syntax so if there happens to be a matching
         # file we don't match that.
-        pv = ParseProductVariant(args[0])
+        pv = parse_product_variant(args[0])
         if pv:
-            config = FindNamedConfig(workspace_root, pv[0])
+            config = find_named_config(workspace_root, pv[0])
             if config:
                 return (config, pv[1])
             return None, None
@@ -139,10 +142,12 @@ def ChooseConfigFromArgs(workspace_root, args):
 
 
 class ConfigException(Exception):
+    ERROR_IDENTIFY = "identify"
     ERROR_PARSE = "parse"
     ERROR_CYCLE = "cycle"
+    ERROR_VALIDATE = "validate"
 
-    def __init__(self, kind, message, locations, line=0):
+    def __init__(self, kind, message, locations=[], line=0):
         """Error thrown when loading and parsing configurations.
 
         Args:
@@ -169,13 +174,13 @@ class ConfigException(Exception):
         self.line = line
 
 
-def LoadConfig(filename):
+def load_config(filename):
     """Load a config, including processing the inherits fields.
 
     Raises:
         ConfigException on errors
     """
-    def LoadAndMerge(fn, visited):
+    def load_and_merge(fn, visited):
         with open(fn) as f:
             try:
                 contents = json.load(f)
@@ -191,34 +196,74 @@ def LoadConfig(filename):
                 if parent in visited:
                     raise ConfigException(ConfigException.ERROR_CYCLE, "Cycle detected in inherits",
                             visited)
-                DeepMerge(inherited_data, LoadAndMerge(parent, [parent,] + visited))
+                deep_merge(inherited_data, load_and_merge(parent, [parent,] + visited))
             # Then merge inherited_data into contents, but what's already there will win.
-            DeepMerge(contents, inherited_data)
+            deep_merge(contents, inherited_data)
             contents.pop("inherits", None)
         return contents
-    return LoadAndMerge(filename, [filename,])
+    return load_and_merge(filename, [filename,])
 
 
-def DeepMerge(merged, addition):
+def deep_merge(merged, addition):
     """Merge all fields of addition into merged. Pre-existing fields win."""
     for k, v in addition.items():
         if k in merged:
             if isinstance(v, dict) and isinstance(merged[k], dict):
-                DeepMerge(merged[k], v)
+                deep_merge(merged[k], v)
         else:
             merged[k] = v
 
 
-def Lunch(args):
+def make_config_header(config_file, config, variant):
+    def make_table(rows):
+        maxcols = max([len(row) for row in rows])
+        widths = [0] * maxcols
+        for row in rows:
+            for i in range(len(row)):
+                widths[i] = max(widths[i], len(row[i]))
+        text = []
+        for row in rows:
+            rowtext = []
+            for i in range(len(row)):
+                cell = row[i]
+                rowtext.append(str(cell))
+                rowtext.append(" " * (widths[i] - len(cell)))
+                rowtext.append("  ")
+            text.append("".join(rowtext))
+        return "\n".join(text)
+
+    trees = [("Component", "Path", "Product"),
+             ("---------", "----", "-------")]
+    entry = config.get("system", None)
+    def add_config_tuple(trees, entry, name):
+        if entry:
+            trees.append((name, entry.get("tree"), entry.get("product", "")))
+    add_config_tuple(trees, config.get("system"), "system")
+    add_config_tuple(trees, config.get("vendor"), "vendor")
+    for k, v in config.get("modules", {}).items():
+        add_config_tuple(trees, v, k)
+
+    return """========================================
+TARGET_BUILD_COMBO=%(TARGET_BUILD_COMBO)s
+TARGET_BUILD_VARIANT=%(TARGET_BUILD_VARIANT)s
+
+%(trees)s
+========================================\n""" % {
+        "TARGET_BUILD_COMBO": config_file,
+        "TARGET_BUILD_VARIANT": variant,
+        "trees": make_table(trees),
+    }
+
+
+def do_lunch(args):
     """Handle the lunch command."""
-    # Check that we're at the top of a multitree workspace
+    # Check that we're at the top of a multitree workspace by seeing if this script exists.
-    # TODO: Choose the right sentinel file
+    if not os.path.exists("build/build/make/orchestrator/core/lunch.py"):
-    if not os.path.exists("build/make/orchestrator"):
         sys.stderr.write("ERROR: lunch.py must be run from the root of a multi-tree workspace\n")
         return EXIT_STATUS_ERROR
 
     # Choose the config file
-    config_file, variant = ChooseConfigFromArgs(".", args)
+    config_file, variant = choose_config_from_args(".", args)
 
     if config_file == None:
         sys.stderr.write("Can't find lunch combo file for: %s\n" % " ".join(args))
@@ -229,7 +274,7 @@ def Lunch(args):
 
     # Parse the config file
     try:
-        config = LoadConfig(config_file)
+        config = load_config(config_file)
     except ConfigException as ex:
         sys.stderr.write(str(ex))
         return EXIT_STATUS_ERROR
@@ -244,47 +289,81 @@ def Lunch(args):
     sys.stdout.write("%s\n" % config_file)
     sys.stdout.write("%s\n" % variant)
 
+    # Write confirmation message to stderr
+    sys.stderr.write(make_config_header(config_file, config, variant))
+
     return EXIT_STATUS_OK
 
 
-def FindAllComboFiles(workspace_root):
+def find_all_combo_files(workspace_root):
     """Find all .mcombo files in the prescribed locations in the tree."""
-    for dir in FindConfigDirs(workspace_root):
+    for dir in find_config_dirs(workspace_root):
-        for file in WalkPaths(dir, lambda x: x.endswith(".mcombo")):
+        for file in walk_paths(dir, lambda x: x.endswith(".mcombo")):
             yield file
 
 
-def IsFileLunchable(config_file):
+def is_file_lunchable(config_file):
     """Parse config_file, flatten the inheritance, and return whether it can be
     used as a lunch target."""
     try:
-        config = LoadConfig(config_file)
+        config = load_config(config_file)
     except ConfigException as ex:
         sys.stderr.write("%s" % ex)
         return False
     return config.get("lunchable", False)
 
 
-def FindAllLunchable(workspace_root):
+def find_all_lunchable(workspace_root):
     """Find all mcombo files in the tree (rooted at workspace_root) that when
     parsed (and inheritance is flattened) have lunchable: true."""
-    for f in [x for x in FindAllComboFiles(workspace_root) if IsFileLunchable(x)]:
+    for f in [x for x in find_all_combo_files(workspace_root) if is_file_lunchable(x)]:
         yield f
 
 
-def List():
+def load_current_config():
+    """Load, validate and return the config as specified in TARGET_BUILD_COMBO.  Throws
+    ConfigException if there is a problem."""
+
+    # Identify the config file
+    config_file = os.environ.get("TARGET_BUILD_COMBO")
+    if not config_file:
+        raise ConfigException(ConfigException.ERROR_IDENTIFY,
+                "TARGET_BUILD_COMBO not set. Run lunch or pass a combo file.")
+
+    # Parse the config file
+    config = load_config(config_file)
+
+    # Validate the config file
+    if not config.get("lunchable", False):
+        raise ConfigException(ConfigException.ERROR_VALIDATE,
+                "Lunch config file (or inherited files) does not have the 'lunchable'"
+                    + " flag set, which means it is probably not a complete lunch spec.",
+                [config_file,])
+
+    # TODO: Validate that:
+    #   - there are no modules called system or vendor
+    #   - everything has all the required files
+
+    variant = os.environ.get("TARGET_BUILD_VARIANT")
+    if not variant:
+        variant = "eng" # TODO: Is this the right default?
+    # Validate variant is user, userdebug or eng
+
+    return config_file, config, variant
+
+def do_list():
     """Handle the --list command."""
-    for f in sorted(FindAllLunchable(".")):
+    for f in sorted(find_all_lunchable(".")):
         print(f)
 
 
-def Print(args):
+def do_print(args):
     """Handle the --print command."""
     # Parse args
     if len(args) == 0:
         config_file = os.environ.get("TARGET_BUILD_COMBO")
         if not config_file:
-            sys.stderr.write("TARGET_BUILD_COMBO not set. Run lunch or pass a combo file.\n")
+            sys.stderr.write("TARGET_BUILD_COMBO not set. Run lunch before building.\n")
             return EXIT_STATUS_NEED_HELP
     elif len(args) == 1:
         config_file = args[0]
@@ -293,7 +372,7 @@ def Print(args):
 
     # Parse the config file
     try:
-        config = LoadConfig(config_file)
+        config = load_config(config_file)
     except ConfigException as ex:
         sys.stderr.write(str(ex))
         return EXIT_STATUS_ERROR
@@ -309,15 +388,15 @@ def main(argv):
         return EXIT_STATUS_NEED_HELP
 
     if len(argv) == 2 and argv[1] == "--list":
-        List()
+        do_list()
         return EXIT_STATUS_OK
 
     if len(argv) == 2 and argv[1] == "--print":
-        return Print(argv[2:])
+        return do_print(argv[2:])
         return EXIT_STATUS_OK
 
-    if (len(argv) == 2 or len(argv) == 3) and argv[1] == "--lunch":
+    if (len(argv) == 3 or len(argv) == 4) and argv[1] == "--lunch":
-        return Lunch(argv[2:])
+        return do_lunch(argv[2:])
 
     sys.stderr.write("Unknown lunch command: %s\n" % " ".join(argv[1:]))
     return EXIT_STATUS_NEED_HELP

orchestrator/core/orchestrator.py

@@ -0,0 +1,123 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+sys.dont_write_bytecode = True
+import api_assembly
+import api_domain
+import api_export
+import inner_tree
+import interrogate
+import lunch
+
+EXIT_STATUS_OK = 0
+EXIT_STATUS_ERROR = 1
+
+API_DOMAIN_SYSTEM = "system"
+API_DOMAIN_VENDOR = "vendor"
+API_DOMAIN_MODULE = "module"
+
+def process_config(lunch_config):
+    """Returns a InnerTrees object based on the configuration requested in the lunch config."""
+    def add(domain_name, tree_root, product):
+        tree_key = inner_tree.InnerTreeKey(tree_root, product)
+        if tree_key in trees:
+            tree = trees[tree_key]
+        else:
+            tree = inner_tree.InnerTree(tree_root, product)
+            trees[tree_key] = tree
+        domain = api_domain.ApiDomain(domain_name, tree, product)
+        domains[domain_name] = domain
+        tree.domains[domain_name] = domain
+
+    trees = {}
+    domains = {}
+
+    system_entry = lunch_config.get("system")
+    if system_entry:
+        add(API_DOMAIN_SYSTEM, system_entry["tree"], system_entry["product"])
+
+    vendor_entry = lunch_config.get("vendor")
+    if vendor_entry:
+        add(API_DOMAIN_VENDOR, vendor_entry["tree"], vendor_entry["product"])
+
+    for module_name, module_entry in lunch_config.get("modules", []).items():
+        add(module_name, module_entry["tree"], None)
+
+    return inner_tree.InnerTrees(trees, domains)
+
+
+def build():
+    #
+    # Load lunch combo
+    #
+
+    # Read the config file
+    try:
+        config_file, config, variant = lunch.load_current_config()
+    except lunch.ConfigException as ex:
+        sys.stderr.write("%s\n" % ex)
+        return EXIT_STATUS_ERROR
+    sys.stdout.write(lunch.make_config_header(config_file, config, variant))
+
+    # Construct the trees and domains dicts
+    inner_trees = process_config(config)
+
+    #
+    # 1. Interrogate the trees
+    #
+    inner_trees.for_each_tree(interrogate.interrogate_tree)
+    # TODO: Detect bazel-only mode
+
+    #
+    # 2a. API Export
+    #
+    inner_trees.for_each_tree(api_export.export_apis_from_tree)
+
+    #
+    # 2b. API Surface Assembly
+    #
+    api_assembly.assemble_apis(inner_trees)
+
+    #
+    # 3a. API Domain Analysis
+    #
+
+    #
+    # 3b. Final Packaging Rules
+    #
+
+    #
+    # 4. Build Execution
+    #
+
+
+    #
+    # Success!
+    #
+    return EXIT_STATUS_OK
+
+def main(argv):
+    return build()
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
+
+
+# vim: sts=4:ts=4:sw=4

orchestrator/core/test_lunch.py

@@ -23,73 +23,73 @@ import lunch
 class TestStringMethods(unittest.TestCase):
 
     def test_find_dirs(self):
-        self.assertEqual([x for x in lunch.FindDirs("test/configs", "multitree_combos")], [
+        self.assertEqual([x for x in lunch.find_dirs("test/configs", "multitree_combos")], [
                     "test/configs/build/make/orchestrator/multitree_combos",
                     "test/configs/device/aa/bb/multitree_combos",
                     "test/configs/vendor/aa/bb/multitree_combos"])
 
     def test_find_file(self):
         # Finds the one in device first because this is searching from the root,
-        # not using FindNamedConfig.
+        # not using find_named_config.
-        self.assertEqual(lunch.FindFile("test/configs", "v.mcombo"),
+        self.assertEqual(lunch.find_file("test/configs", "v.mcombo"),
                    "test/configs/device/aa/bb/multitree_combos/v.mcombo")
 
     def test_find_config_dirs(self):
-        self.assertEqual([x for x in lunch.FindConfigDirs("test/configs")], [
+        self.assertEqual([x for x in lunch.find_config_dirs("test/configs")], [
                     "test/configs/build/make/orchestrator/multitree_combos",
                     "test/configs/vendor/aa/bb/multitree_combos",
                     "test/configs/device/aa/bb/multitree_combos"])
 
     def test_find_named_config(self):
         # Inside build/orchestrator, overriding device and vendor
-        self.assertEqual(lunch.FindNamedConfig("test/configs", "b"),
+        self.assertEqual(lunch.find_named_config("test/configs", "b"),
                     "test/configs/build/make/orchestrator/multitree_combos/b.mcombo")
 
         # Nested dir inside a combo dir
-        self.assertEqual(lunch.FindNamedConfig("test/configs", "nested"),
+        self.assertEqual(lunch.find_named_config("test/configs", "nested"),
                     "test/configs/build/make/orchestrator/multitree_combos/nested/nested.mcombo")
 
         # Inside vendor, overriding device
-        self.assertEqual(lunch.FindNamedConfig("test/configs", "v"),
+        self.assertEqual(lunch.find_named_config("test/configs", "v"),
                     "test/configs/vendor/aa/bb/multitree_combos/v.mcombo")
 
         # Inside device
-        self.assertEqual(lunch.FindNamedConfig("test/configs", "d"),
+        self.assertEqual(lunch.find_named_config("test/configs", "d"),
                     "test/configs/device/aa/bb/multitree_combos/d.mcombo")
 
         # Make sure we don't look too deep (for performance)
-        self.assertIsNone(lunch.FindNamedConfig("test/configs", "too_deep"))
+        self.assertIsNone(lunch.find_named_config("test/configs", "too_deep"))
 
 
     def test_choose_config_file(self):
         # Empty string argument
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", [""]),
+        self.assertEqual(lunch.choose_config_from_args("test/configs", [""]),
                     (None, None))
 
         # A PRODUCT-VARIANT name
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["v-eng"]),
+        self.assertEqual(lunch.choose_config_from_args("test/configs", ["v-eng"]),
                     ("test/configs/vendor/aa/bb/multitree_combos/v.mcombo", "eng"))
 
         # A PRODUCT-VARIANT name that conflicts with a file
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["b-eng"]),
+        self.assertEqual(lunch.choose_config_from_args("test/configs", ["b-eng"]),
                     ("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"))
 
         # A PRODUCT-VARIANT that doesn't exist
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs", ["z-user"]),
+        self.assertEqual(lunch.choose_config_from_args("test/configs", ["z-user"]),
                     (None, None))
 
         # An explicit file
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs",
+        self.assertEqual(lunch.choose_config_from_args("test/configs",
                         ["test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"]),
                     ("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", "eng"))
 
         # An explicit file that doesn't exist
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs",
+        self.assertEqual(lunch.choose_config_from_args("test/configs",
                         ["test/configs/doesnt_exist.mcombo", "eng"]),
                     (None, None))
 
         # An explicit file without a variant should fail
-        self.assertEqual(lunch.ChooseConfigFromArgs("test/configs",
+        self.assertEqual(lunch.choose_config_from_args("test/configs",
                         ["test/configs/build/make/orchestrator/multitree_combos/b.mcombo"]),
                     ("test/configs/build/make/orchestrator/multitree_combos/b.mcombo", None))
 
@@ -97,12 +97,12 @@ class TestStringMethods(unittest.TestCase):
     def test_config_cycles(self):
         # Test that we catch cycles
         with self.assertRaises(lunch.ConfigException) as context:
-            lunch.LoadConfig("test/configs/parsing/cycles/1.mcombo")
+            lunch.load_config("test/configs/parsing/cycles/1.mcombo")
         self.assertEqual(context.exception.kind, lunch.ConfigException.ERROR_CYCLE)
 
     def test_config_merge(self):
         # Test the merge logic
-        self.assertEqual(lunch.LoadConfig("test/configs/parsing/merge/1.mcombo"), {
+        self.assertEqual(lunch.load_config("test/configs/parsing/merge/1.mcombo"), {
                             "in_1": "1",
                             "in_1_2": "1",
                             "merged": {"merged_1": "1",
@@ -119,7 +119,7 @@ class TestStringMethods(unittest.TestCase):
                         })
 
     def test_list(self):
-        self.assertEqual(sorted(lunch.FindAllLunchable("test/configs")),
+        self.assertEqual(sorted(lunch.find_all_lunchable("test/configs")),
                 ["test/configs/build/make/orchestrator/multitree_combos/b.mcombo"])
 
 if __name__ == "__main__":

orchestrator/inner_build/common.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+def _parse_arguments(argv):
+    argv = argv[1:]
+    """Return an argparse options object."""
+    # Top-level parser
+    parser = argparse.ArgumentParser(prog=".inner_build")
+
+    parser.add_argument("--out_dir", action="store", required=True,
+            help="root of the output directory for this inner tree's API contributions")
+
+    parser.add_argument("--api_domain", action="append", required=True,
+            help="which API domains are to be built in this inner tree")
+
+    subparsers = parser.add_subparsers(required=True, dest="command",
+            help="subcommands")
+
+    # inner_build describe command
+    describe_parser = subparsers.add_parser("describe",
+            help="describe the capabilities of this inner tree's build system")
+
+    # create the parser for the "b" command
+    export_parser = subparsers.add_parser("export_api_contributions",
+            help="export the API contributions of this inner tree")
+
+    # Parse the arguments
+    return parser.parse_args(argv)
+
+
+class Commands(object):
+    def Run(self, argv):
+        """Parse the command arguments and call the corresponding subcommand method on
+        this object.
+
+        Throws AttributeError if the method for the command wasn't found.
+        """
+        args = _parse_arguments(argv)
+        return getattr(self, args.command)(args)
+

orchestrator/inner_build/inner_build_demo.py

@@ -0,0 +1,143 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import textwrap
+
+sys.dont_write_bytecode = True
+import common
+
+def mkdirs(path):
+    try:
+        os.makedirs(path)
+    except FileExistsError:
+        pass
+
+
+class InnerBuildSoong(common.Commands):
+    def describe(self, args):
+        mkdirs(args.out_dir)
+
+        with open(os.path.join(args.out_dir, "tree_info.json"), "w") as f:
+            f.write(textwrap.dedent("""\
+            {
+                "requires_ninja": true,
+                "orchestrator_protocol_version": 1
+            }"""))
+
+    def export_api_contributions(self, args):
+        contributions_dir = os.path.join(args.out_dir, "api_contributions")
+        mkdirs(contributions_dir)
+
+        if "system" in args.api_domain:
+            with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f:
+                # 'name: android' is android.jar
+                f.write(textwrap.dedent("""\
+                {
+                    "name": "public_api",
+                    "version": 1,
+                    "api_domain": "system",
+                    "cc_libraries": [
+                        {
+                            "name": "libhwui",
+                            "headers": [
+                                {
+                                    "root": "frameworks/base/libs/hwui/apex/include",
+                                    "files": [
+                                        "android/graphics/jni_runtime.h",
+                                        "android/graphics/paint.h",
+                                        "android/graphics/matrix.h",
+                                        "android/graphics/canvas.h",
+                                        "android/graphics/renderthread.h",
+                                        "android/graphics/bitmap.h",
+                                        "android/graphics/region.h"
+                                    ]
+                                }
+                            ],
+                            "api": [
+                                "frameworks/base/libs/hwui/libhwui.map.txt"
+                            ]
+                        }
+                    ],
+                    "java_libraries": [
+                        {
+                            "name": "android",
+                            "api": [
+                                "frameworks/base/core/api/current.txt"
+                            ]
+                        }
+                    ],
+                    "resource_libraries": [
+                        {
+                            "name": "android",
+                            "api": "frameworks/base/core/res/res/values/public.xml"
+                        }
+                    ],
+                    "host_executables": [
+                        {
+                            "name": "aapt2",
+                            "binary": "out/host/bin/aapt2",
+                            "runfiles": [
+                                "../lib/todo.so"
+                            ]
+                        }
+                    ]
+                }"""))
+        elif "com.android.bionic" in args.api_domain:
+            with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f:
+                # 'name: android' is android.jar
+                f.write(textwrap.dedent("""\
+                {
+                    "name": "public_api",
+                    "version": 1,
+                    "api_domain": "system",
+                    "cc_libraries": [
+                        {
+                            "name": "libc",
+                            "headers": [
+                                {
+                                    "root": "bionic/libc/include",
+                                    "files": [
+                                        "stdio.h",
+                                        "sys/klog.h"
+                                    ]
+                                }
+                            ],
+                            "api": "bionic/libc/libc.map.txt"
+                        }
+                    ],
+                    "java_libraries": [
+                        {
+                            "name": "android",
+                            "api": [
+                                "frameworks/base/libs/hwui/api/current.txt"
+                            ]
+                        }
+                    ]
+                }"""))
+
+
+
+def main(argv):
+    return InnerBuildSoong().Run(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
+
+
+# vim: sts=4:ts=4:sw=4

orchestrator/inner_build/inner_build_soong.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+sys.dont_write_bytecode = True
+import common
+
+class InnerBuildSoong(common.Commands):
+    def describe(self, args):
+        pass
+
+
+    def export_api_contributions(self, args):
+        pass
+
+
+def main(argv):
+    return InnerBuildSoong().Run(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

orchestrator/multitree_combos/aosp_cf_arm64_phone.mcombo

@@ -0,0 +1,16 @@
+{
+    "lunchable": true,
+    "system": {
+        "tree": "master",
+        "product": "aosp_cf_arm64_phone"
+    },
+    "vendor": {
+        "tree": "master",
+        "product": "aosp_cf_arm64_phone"
+    },
+    "modules": {
+        "com.android.bionic": {
+            "tree": "sc-mainline-prod"
+        }
+    }
+}