From bbe2cc61918c07656980391e432e5274ed17629c Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 12 Jul 2023 17:55:28 -0700 Subject: [PATCH] Cleanup genrule_sandbox_test.py In preparation for adding the ability to check all genrules in the tree. Bug: 290816499 Test: ./build/soong/tests/genrule_sandbox_test.py extensions_db.pb Change-Id: I6454be7e88a876a2ab1105c04f5ecda03ecd7771 --- tests/genrule_sandbox_test.py | 140 +++++++++++++--------------------- 1 file changed, 55 insertions(+), 85 deletions(-) diff --git a/tests/genrule_sandbox_test.py b/tests/genrule_sandbox_test.py index a9f0c9b50..0cebc2aa0 100755 --- a/tests/genrule_sandbox_test.py +++ b/tests/genrule_sandbox_test.py @@ -17,34 +17,37 @@ import argparse import collections import json -import os.path +import os import subprocess +import sys import tempfile -SRC_ROOT_DIR = os.path.abspath(__file__ + "/../../../..") +def get_top() -> str: + path = '.' + while not os.path.isfile(os.path.join(path, 'build/soong/tests/genrule_sandbox_test.py')): + if os.path.abspath(path) == '/': + sys.exit('Could not find android source tree root.') + path = os.path.join(path, '..') + return os.path.abspath(path) - -def _module_graph_path(out_dir): - return os.path.join(SRC_ROOT_DIR, out_dir, "soong", "module-actions.json") - - -def _build_with_soong(targets, target_product, out_dir, extra_env={}): +def _build_with_soong(targets, target_product, *, keep_going = False, extra_env={}): env = { + **os.environ, "TARGET_PRODUCT": target_product, "TARGET_BUILD_VARIANT": "userdebug", } - env.update(os.environ) env.update(extra_env) args = [ "build/soong/soong_ui.bash", "--make-mode", "--skip-soong-tests", ] + if keep_going: + args.append("-k") args.extend(targets) try: - out = subprocess.check_output( + subprocess.check_output( args, - cwd=SRC_ROOT_DIR, env=env, ) except subprocess.CalledProcessError as e: @@ -55,14 +58,13 @@ def _build_with_soong(targets, target_product, out_dir, extra_env={}): def _find_outputs_for_modules(modules, out_dir, target_product): - module_path = os.path.join( - SRC_ROOT_DIR, out_dir, "soong", "module-actions.json" - ) + module_path = os.path.join(out_dir, "soong", "module-actions.json") if not os.path.exists(module_path): - _build_with_soong(["json-module-graph"], target_product, out_dir) + _build_with_soong(["json-module-graph"], target_product) - action_graph = json.load(open(_module_graph_path(out_dir))) + with open(module_path) as f: + action_graph = json.load(f) module_to_outs = collections.defaultdict(set) for mod in action_graph: @@ -74,50 +76,15 @@ def _find_outputs_for_modules(modules, out_dir, target_product): return module_to_outs -def _store_outputs_to_tmp(output_files): - try: - tempdir = tempfile.TemporaryDirectory() - for f in output_files: - out = subprocess.check_output( - ["cp", "--parents", f, tempdir.name], - cwd=SRC_ROOT_DIR, - ) - return tempdir - except subprocess.CalledProcessError as e: - print(e) - print(e.stdout) - print(e.stderr) - - -def _diff_outs(file1, file2, show_diff): - output = None - base_args = ["diff"] - if not show_diff: - base_args.append("--brief") - try: - args = base_args + [file1, file2] - output = subprocess.check_output( - args, - cwd=SRC_ROOT_DIR, - ) - except subprocess.CalledProcessError as e: - if e.returncode == 1: - if show_diff: - return output - return True - return None - - -def _compare_outputs(module_to_outs, tempdir, show_diff): +def _compare_outputs(module_to_outs, tempdir) -> dict[str, list[str]]: different_modules = collections.defaultdict(list) for module, outs in module_to_outs.items(): for out in outs: - output = None - diff = _diff_outs(os.path.join(tempdir.name, out), out, show_diff) - if diff: - different_modules[module].append(diff) + try: + subprocess.check_output(["diff", os.path.join(tempdir, out), out]) + except subprocess.CalledProcessError as e: + different_modules[module].append(e.stdout) - tempdir.cleanup() return different_modules @@ -138,53 +105,56 @@ def main(): "--show-diff", "-d", action="store_true", - required=False, help="whether to display differing files", ) parser.add_argument( "--output-paths-only", "-o", action="store_true", - required=False, help="Whether to only return the output paths per module", ) args = parser.parse_args() + os.chdir(get_top()) out_dir = os.environ.get("OUT_DIR", "out") - target_product = args.target_product - modules = set(args.modules) - module_to_outs = _find_outputs_for_modules(modules, out_dir, target_product) + print("finding output files for the modules...") + module_to_outs = _find_outputs_for_modules(set(args.modules), out_dir, args.target_product) if not module_to_outs: - print("No outputs found") - exit(1) + sys.exit("No outputs found") if args.output_paths_only: for m, o in module_to_outs.items(): print(f"{m} outputs: {o}") - exit(0) + sys.exit(0) - all_outs = set() - for outs in module_to_outs.values(): - all_outs.update(outs) - print("build without sandboxing") - _build_with_soong(list(all_outs), target_product, out_dir) - tempdir = _store_outputs_to_tmp(all_outs) - print("build with sandboxing") - _build_with_soong( - list(all_outs), - target_product, - out_dir, - extra_env={"GENRULE_SANDBOXING": "true"}, - ) - diffs = _compare_outputs(module_to_outs, tempdir, args.show_diff) - if len(diffs) == 0: - print("All modules are correct") - elif args.show_diff: - for m, d in diffs.items(): - print(f"Module {m} has diffs {d}") - else: - print(f"Modules {list(diffs.keys())} have diffs") + all_outs = list(set.union(*module_to_outs.values())) + + print("building without sandboxing...") + _build_with_soong(all_outs, args.target_product) + with tempfile.TemporaryDirectory() as tempdir: + for f in all_outs: + subprocess.check_call(["cp", "--parents", f, tempdir]) + + print("building with sandboxing...") + _build_with_soong( + all_outs, + args.target_product, + # We've verified these build without sandboxing already, so do the sandboxing build + # with keep_going = True so that we can find all the genrules that fail to build with + # sandboxing. + keep_going = True, + extra_env={"GENRULE_SANDBOXING": "true"}, + ) + + diffs = _compare_outputs(module_to_outs, tempdir) + if len(diffs) == 0: + print("All modules are correct") + elif args.show_diff: + for m, d in diffs.items(): + print(f"Module {m} has diffs {d}") + else: + print(f"Modules {list(diffs.keys())} have diffs") if __name__ == "__main__":