tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Orchestrator can build end to end."

Browse source
This commit is contained in:
Treehugger Robot 2022-05-27 03:52:34 +00:00 committed by Gerrit Code Review
commit d0035d5d88
16 changed files with 277 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
orchestrator/README
View file

@ -2,6 +2,7 @@ DEMO
from the root of the workspace
ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build
ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build
multitree_lunch build/build/make/orchestrator/test_workspace/combo.mcombo eng
rm -rf out && multitree_build && echo "==== Files ====" && find out -type f

5
orchestrator/core/api_assembly.py
View file

@ -34,7 +34,7 @@ def assemble_apis(context, inner_trees):
contributions = []
for tree_key, filenames in contribution_files_dict.items():
for filename in filenames:
json_data = load_contribution_file(filename)
json_data = load_contribution_file(context, filename)
if not json_data:
continue
# TODO: Validate the configs, especially that the domains match what we asked for
@ -76,13 +76,14 @@ def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
return result
def load_contribution_file(filename):
def load_contribution_file(context, filename):
"Load and return the API contribution at filename. On error report error and return None."
with open(filename) as f:
try:
return json.load(f)
except json.decoder.JSONDecodeError as ex:
# TODO: Error reporting
context.errors.error(ex.msg, filename, ex.lineno, ex.colno)
raise ex

7
orchestrator/core/api_assembly_cc.py
View file

@ -17,17 +17,10 @@
import os
def assemble_cc_api_library(context, ninja, build_file, stub_library):
print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,
stub_library.api_surface_version, stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution))
staging_dir = context.out.api_library_dir(stub_library.api_surface,
stub_library.api_surface_version, stub_library.name)
work_dir = context.out.api_library_work_dir(stub_library.api_surface,
stub_library.api_surface_version, stub_library.name)
print("staging_dir=%s" % (staging_dir))
print("work_dir=%s" % (work_dir))
# Generate rules to copy headers
includes = []

90
orchestrator/core/final_packaging.py
View file

@ -13,10 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import sys
import ninja_tools
import ninja_syntax # Has to be after ninja_tools because of the path hack
def final_packaging(context):
def final_packaging(context, inner_trees):
"""Pull together all of the previously defined rules into the final build stems."""
with open(context.out.outer_ninja_file(), "w") as ninja_file:
@ -25,5 +29,89 @@ def final_packaging(context):
# Add the api surfaces file
ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))
# For each inner tree
for tree in inner_trees.keys():
# TODO: Verify that inner_tree.ninja was generated
# Read and verify file
build_targets = read_build_targets_json(context, tree)
if not build_targets:
continue
# Generate the ninja and build files for this inner tree
generate_cross_domain_build_rules(context, ninja, tree, build_targets)
# Finish writing the ninja file
ninja.write()
def read_build_targets_json(context, tree):
"""Read and validate the build_targets.json file for the given tree."""
try:
f = open(tree.out.build_targets_file())
except FileNotFoundError:
# It's allowed not to have any artifacts (e.g. if a tree is a light tree with only APIs)
return None
data = None
with f:
try:
data = json.load(f)
except json.decoder.JSONDecodeError as ex:
sys.stderr.write("Error parsing file: %s\n" % tree.out.build_targets_file())
# TODO: Error reporting
raise ex
# TODO: Better error handling
# TODO: Validate json schema
return data
def generate_cross_domain_build_rules(context, ninja, tree, build_targets):
"Generate the ninja and build files for the inner tree."
# Include the inner tree's inner_tree.ninja
ninja.add_subninja(ninja_syntax.Subninja(tree.out.main_ninja_file(), chDir=tree.root))
# Generate module rules and files
for module in build_targets.get("modules", []):
generate_shared_module(context, ninja, tree, module)
# Generate staging rules
staging_dir = context.out.staging_dir()
for staged in build_targets.get("staging", []):
# TODO: Enforce that dest isn't in disallowed subdir of out or absolute
dest = staged["dest"]
dest = os.path.join(staging_dir, dest)
if "src" in staged and "obj" in staged:
context.errors.error("Can't have both \"src\" and \"obj\" tags in \"staging\" entry."
) # TODO: Filename and line if possible
if "src" in staged:
ninja.add_copy_file(dest, os.path.join(tree.root, staged["src"]))
elif "obj" in staged:
ninja.add_copy_file(dest, os.path.join(tree.out.root(), staged["obj"]))
ninja.add_global_phony("staging", [dest])
# Generate dist rules
dist_dir = context.out.dist_dir()
for disted in build_targets.get("dist", []):
# TODO: Enforce that dest absolute
dest = disted["dest"]
dest = os.path.join(dist_dir, dest)
ninja.add_copy_file(dest, os.path.join(tree.root, disted["src"]))
ninja.add_global_phony("dist", [dest])
def generate_shared_module(context, ninja, tree, module):
"""Generate ninja rules for the given build_targets.json defined module."""
module_name = module["name"]
module_type = module["type"]
share_dir = context.out.module_share_dir(module_type, module_name)
src_file = os.path.join(tree.root, module["file"])
if module_type == "apex":
ninja.add_copy_file(os.path.join(share_dir, module_name + ".apex"), src_file)
# TODO: Generate build file
else:
# TODO: Better error handling
raise Exception("Invalid module type: %s" % module)

45
orchestrator/core/inner_tree.py
View file

@ -36,23 +36,38 @@ class InnerTreeKey(object):
def __hash__(self):
return hash((self.root, self.product))
def _cmp(self, other):
if self.root < other.root:
return -1
if self.root > other.root:
return 1
if self.product == other.product:
return 0
if self.product is None:
return -1
if other.product is None:
return 1
if self.product < other.product:
return -1
return 1
def __eq__(self, other):
return (self.root == other.root and self.product == other.product)
return self._cmp(other) == 0
def __ne__(self, other):
return not self.__eq__(other)
return self._cmp(other) != 0
def __lt__(self, other):
return (self.root, self.product) < (other.root, other.product)
return self._cmp(other) < 0
def __le__(self, other):
return (self.root, self.product) <= (other.root, other.product)
return self._cmp(other) <= 0
def __gt__(self, other):
return (self.root, self.product) > (other.root, other.product)
return self._cmp(other) > 0
def __ge__(self, other):
return (self.root, self.product) >= (other.root, other.product)
return self._cmp(other) >= 0
class InnerTree(object):
@ -62,7 +77,12 @@ class InnerTree(object):
self.product = product
self.domains = {}
# TODO: Base directory on OUT_DIR
self.out = OutDirLayout(context.out.inner_tree_dir(root))
out_root = context.out.inner_tree_dir(root)
if product:
out_root += "_" + product
else:
out_root += "_unbundled"
self.out = OutDirLayout(out_root)
def __str__(self):
return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
@ -138,6 +158,11 @@ class InnerTrees(object):
"""Get an inner tree for tree_key"""
return self.trees.get(tree_key)
def keys(self):
"Get the keys for the inner trees in name order."
return [self.trees[k] for k in sorted(self.trees.keys())]
class OutDirLayout(object):
"""Encapsulates the logic about the layout of the inner tree out directories.
See also context.OutDir for outer tree out dir contents."""
@ -155,6 +180,12 @@ class OutDirLayout(object):
def api_contributions_dir(self):
return os.path.join(self._root, "api_contributions")
def build_targets_file(self):
return os.path.join(self._root, "build_targets.json")
def main_ninja_file(self):
return os.path.join(self._root, "inner_tree.ninja")
def enquote(s):
return "None" if s is None else "\"%s\"" % s

1
orchestrator/core/ninja_runner.py
View file

@ -14,6 +14,7 @@
# limitations under the License.
import subprocess
import sys
def run_ninja(context, targets):
"""Run ninja.

13
orchestrator/core/ninja_tools.py
View file

@ -30,6 +30,7 @@ class Ninja(ninja_writer.Writer):
super(Ninja, self).__init__(file)
self._context = context
self._did_copy_file = False
self._phonies = {}
def add_copy_file(self, copy_to, copy_from):
if not self._did_copy_file:
@ -43,4 +44,16 @@ class Ninja(ninja_writer.Writer):
build_action.add_variable("out_dir", os.path.dirname(copy_to))
self.add_build_action(build_action)
def add_global_phony(self, name, deps):
"""Add a phony target where there are multiple places that will want to add to
the same phony. If you can, to save memory, use add_phony instead of this function."""
if type(deps) not in (list, tuple):
raise Exception("Assertion failed: bad type of deps: %s" % type(deps))
self._phonies.setdefault(name, []).extend(deps)
def write(self):
for phony, deps in self._phonies.items():
self.add_phony(phony, deps)
super(Ninja, self).write()

28
orchestrator/core/orchestrator.py
View file

@ -24,6 +24,7 @@ import api_domain
import api_export
import final_packaging
import inner_tree
import tree_analysis
import interrogate
import lunch
import ninja_runner
@ -67,14 +68,10 @@ def process_config(context, lunch_config):
def build():
#
# Load lunch combo
#
# Choose the out directory, set up error handling, etc.
context = utils.Context(utils.choose_out_dir(), utils.Errors(sys.stderr))
# Read the config file
# Read the lunch config file
try:
config_file, config, variant = lunch.load_current_config()
except lunch.ConfigException as ex:
@ -85,44 +82,31 @@ def build():
# Construct the trees and domains dicts
inner_trees = process_config(context, config)
#
# 1. Interrogate the trees
#
inner_trees.for_each_tree(interrogate.interrogate_tree)
# TODO: Detect bazel-only mode
#
# 2a. API Export
#
inner_trees.for_each_tree(api_export.export_apis_from_tree)
#
# 2b. API Surface Assembly
#
api_assembly.assemble_apis(context, inner_trees)
#
# 3a. API Domain Analysis
#
# 3a. Inner tree analysis
tree_analysis.analyze_trees(context, inner_trees)
#
# 3b. Final Packaging Rules
#
final_packaging.final_packaging(context)
final_packaging.final_packaging(context, inner_trees)
#
# 4. Build Execution
#
# TODO: Decide what we want the UX for selecting targets to be across
# branches... since there are very likely to be conflicting soong short
# names.
print("Running ninja...")
targets = ["public_api-1-libhwui", "public_api-1-libc"]
targets = ["staging", "system"]
ninja_runner.run_ninja(context, targets)
#
# Success!
#
return EXIT_STATUS_OK
def main(argv):

24
orchestrator/core/tree_analysis.py Normal file
View file

@ -0,0 +1,24 @@
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def analyze_trees(context, inner_trees):
inner_trees.for_each_tree(run_analysis)
def run_analysis(tree_key, inner_tree, cookie):
inner_tree.invoke(["analyze"])

41
orchestrator/core/utils.py
View file

@ -38,33 +38,42 @@ class OutDir(object):
def __init__(self, root):
"Initialize with the root of the OUT_DIR for the outer tree."
self._root = root
self._out_root = root
self._intermediates = "intermediates"
def root(self):
return self._root
return self._out_root
def inner_tree_dir(self, tree_root):
"""Root directory for inner tree inside the out dir."""
return os.path.join(self._root, "trees", tree_root)
return os.path.join(self._out_root, "trees", tree_root)
def api_ninja_file(self):
"""The ninja file that assembles API surfaces."""
return os.path.join(self._root, "api_surfaces.ninja")
return os.path.join(self._out_root, "api_surfaces.ninja")
def api_library_dir(self, surface, version, library):
"""Directory for all the contents of a library inside an API surface, including
the build files. Any intermediates should go in api_library_work_dir."""
return os.path.join(self._root, "api_surfaces", surface, str(version), library)
return os.path.join(self._out_root, "api_surfaces", surface, str(version), library)
def api_library_work_dir(self, surface, version, library):
"""Intermediates / scratch directory for library inside an API surface."""
return os.path.join(self._root, self._intermediates, "api_surfaces", surface, str(version),
library)
return os.path.join(self._out_root, self._intermediates, "api_surfaces", surface,
str(version), library)
def outer_ninja_file(self):
return os.path.join(self._root, "multitree.ninja")
return os.path.join(self._out_root, "multitree.ninja")
def module_share_dir(self, module_type, module_name):
return os.path.join(self._out_root, "shared", module_type, module_name)
def staging_dir(self):
return os.path.join(self._out_root, "staging")
def dist_dir(self):
"The DIST_DIR provided or out/dist" # TODO: Look at DIST_DIR
return os.path.join(self._out_root, "dist")
class Errors(object):
"""Class for reporting and tracking errors."""
@ -73,9 +82,21 @@ class Errors(object):
self._stream = stream
self._all = []
def error(self, message):
def error(self, message, file=None, line=None, col=None):
"""Record the error message."""
s = str(s)
s = ""
if file:
s += str(file)
s += ":"
if line:
s += str(line)
s += ":"
if col:
s += str(col)
s += ":"
if s:
s += " "
s += str(message)
if s[-1] != "\n":
s += "\n"
self._all.append(s)

4
orchestrator/inner_build/common.py
View file

@ -40,6 +40,10 @@ def _parse_arguments(argv):
export_parser = subparsers.add_parser("export_api_contributions",
help="export the API contributions of this inner tree")
# create the parser for the "b" command
export_parser = subparsers.add_parser("analyze",
help="main build analysis for this inner tree")
# Parse the arguments
return parser.parse_args(argv)

103
orchestrator/inner_build/inner_build_demo.py
View file

@ -44,93 +44,60 @@ class InnerBuildSoong(common.Commands):
mkdirs(contributions_dir)
if "system" in args.api_domain:
with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f:
with open(os.path.join(contributions_dir, "api_a-1.json"), "w") as f:
# 'name: android' is android.jar
f.write(textwrap.dedent("""\
{
"name": "public_api",
"name": "api_a",
"version": 1,
"api_domain": "system",
"cc_libraries": [
{
"name": "libhwui",
"name": "libhello1",
"headers": [
{
"root": "frameworks/base/libs/hwui/apex/include",
"root": "build/build/make/orchestrator/test_workspace/inner_tree_1",
"files": [
"android/graphics/jni_runtime.h",
"android/graphics/paint.h",
"android/graphics/matrix.h",
"android/graphics/canvas.h",
"android/graphics/renderthread.h",
"android/graphics/bitmap.h",
"android/graphics/region.h"
"hello1.h"
]
}
],
"api": [
"frameworks/base/libs/hwui/libhwui.map.txt"
]
}
],
"java_libraries": [
{
"name": "android",
"api": [
"frameworks/base/core/api/current.txt"
]
}
],
"resource_libraries": [
{
"name": "android",
"api": "frameworks/base/core/res/res/values/public.xml"
}
],
"host_executables": [
{
"name": "aapt2",
"binary": "out/host/bin/aapt2",
"runfiles": [
"../lib/todo.so"
]
}
]
}"""))
elif "com.android.bionic" in args.api_domain:
with open(os.path.join(contributions_dir, "public_api-1.json"), "w") as f:
# 'name: android' is android.jar
f.write(textwrap.dedent("""\
{
"name": "public_api",
"version": 1,
"api_domain": "system",
"cc_libraries": [
{
"name": "libc",
"headers": [
{
"root": "bionic/libc/include",
"files": [
"stdio.h",
"sys/klog.h"
]
}
],
"api": "bionic/libc/libc.map.txt"
}
],
"java_libraries": [
{
"name": "android",
"api": [
"frameworks/base/libs/hwui/api/current.txt"
"build/build/make/orchestrator/test_workspace/inner_tree_1/libhello1"
]
}
]
}"""))
def analyze(self, args):
if "system" in args.api_domain:
# Nothing to export in this demo
# Write a fake inner_tree.ninja; what the inner tree would have generated
with open(os.path.join(args.out_dir, "inner_tree.ninja"), "w") as f:
# TODO: Note that this uses paths relative to the workspace not the iner tree
# for demo purposes until we get the ninja chdir change in.
f.write(textwrap.dedent("""\
rule compile_c
command = mkdir -p ${out_dir} && g++ -c ${cflags} -o ${out} ${in}
rule link_so
command = mkdir -p ${out_dir} && gcc -shared -o ${out} ${in}
build %(OUT_DIR)s/libhello1/hello1.o: compile_c build/build/make/orchestrator/test_workspace/inner_tree_1/libhello1/hello1.c
out_dir = %(OUT_DIR)s/libhello1
cflags = -Ibuild/build/make/orchestrator/test_workspace/inner_tree_1/libhello1/include
build %(OUT_DIR)s/libhello1/libhello1.so: link_so %(OUT_DIR)s/libhello1/hello1.o
out_dir = %(OUT_DIR)s/libhello1
build system: phony %(OUT_DIR)s/libhello1/libhello1.so
""" % { "OUT_DIR": args.out_dir }))
with open(os.path.join(args.out_dir, "build_targets.json"), "w") as f:
f.write(textwrap.dedent("""\
{
"staging": [
{
"dest": "staging/system/lib/libhello1.so",
"obj": "libhello1/libhello1.so"
}
]
}""" % { "OUT_DIR": args.out_dir }))
def main(argv):
return InnerBuildSoong().Run(argv)

17
orchestrator/test_workspace/combo.mcombo Normal file
View file

@ -0,0 +1,17 @@
{
"lunchable": true,
"system": {
"tree": "build/build/make/orchestrator/test_workspace/inner_tree_1",
"product": "test_product1"
},
"vendor": {
"tree": "build/build/make/orchestrator/test_workspace/inner_tree_1",
"product": "test_product2"
},
"modules": {
"module_1": {
"tree": "build/build/make/orchestrator/test_workspace/inner_tree_1"
}
}
}

1
orchestrator/test_workspace/inner_tree_1/.inner_build Symbolic link
View file

@ -0,0 +1 @@
../../inner_build/inner_build_demo.py

8
orchestrator/test_workspace/inner_tree_1/libhello1/hello1.c Normal file
View file

@ -0,0 +1,8 @@
#include <stdio.h>
#include "hello1.h"
void hello1(void) {
printf("hello1");
}

4
orchestrator/test_workspace/inner_tree_1/libhello1/include/hello1.h Normal file
View file

@ -0,0 +1,4 @@
#pragma once
extern "C" void hello1(void);