diff --git a/ci/build_test_suites b/ci/build_test_suites new file mode 100755 index 0000000000..861065a73a --- /dev/null +++ b/ci/build_test_suites @@ -0,0 +1,22 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import build_test_suites + +if __name__ == '__main__': + sys.dont_write_bytecode = True + + build_test_suites.main(sys.argv) diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py new file mode 100644 index 0000000000..e88b420d3f --- /dev/null +++ b/ci/build_test_suites.py @@ -0,0 +1,282 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to build only the necessary modules for general-tests along + +with whatever other targets are passed in. +""" + +import argparse +from collections.abc import Sequence +import json +import os +import pathlib +import re +import subprocess +import sys +from typing import Any, Dict, Set, Text + +import test_mapping_module_retriever + + +# List of modules that are always required to be in general-tests.zip +REQUIRED_MODULES = frozenset( + ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip'] +) + + +def build_test_suites(argv): + args = parse_args(argv) + + if not args.change_info: + build_everything(args) + return + + # Call the class to map changed files to modules to build. + # TODO(lucafarsi): Move this into a replaceable class. + build_affected_modules(args) + + +def parse_args(argv): + argparser = argparse.ArgumentParser() + argparser.add_argument( + 'extra_targets', nargs='*', help='Extra test suites to build.' + ) + argparser.add_argument('--target_product') + argparser.add_argument('--target_release') + argparser.add_argument( + '--with_dexpreopt_boot_img_and_system_server_only', action='store_true' + ) + argparser.add_argument('--dist_dir') + argparser.add_argument('--change_info', nargs='?') + argparser.add_argument('--extra_required_modules', nargs='*') + + return argparser.parse_args() + + +def build_everything(args: argparse.Namespace): + build_command = base_build_command(args) + build_command.append('general-tests') + + run_command(build_command) + + +def build_affected_modules(args: argparse.Namespace): + modules_to_build = find_modules_to_build( + pathlib.Path(args.change_info), args.extra_required_modules + ) + + # Call the build command with everything. + build_command = base_build_command(args) + build_command.extend(modules_to_build) + + run_command(build_command) + + zip_build_outputs(modules_to_build, args.dist_dir) + + +def base_build_command(args: argparse.Namespace) -> list: + build_command = [] + build_command.append('time') + build_command.append('./build/soong/soong_ui.bash') + build_command.append('--make-mode') + build_command.append('dist') + build_command.append('DIST_DIR=' + args.dist_dir) + build_command.append('TARGET_PRODUCT=' + args.target_product) + build_command.append('TARGET_RELEASE=' + args.target_release) + build_command.extend(args.extra_targets) + + return build_command + + +def run_command(args: list[str]) -> str: + result = subprocess.run( + args=args, + text=True, + capture_output=True, + check=False, + ) + # If the process failed, print its stdout and propagate the exception. + if not result.returncode == 0: + print('Build command failed! output:') + print('stdout: ' + result.stdout) + print('stderr: ' + result.stderr) + + result.check_returncode() + return result.stdout + + +def find_modules_to_build( + change_info: pathlib.Path, extra_required_modules: list[Text] +) -> Set[Text]: + changed_files = find_changed_files(change_info) + + test_mappings = test_mapping_module_retriever.GetTestMappings( + changed_files, set() + ) + + # Soong_zip is required to generate the output zip so always build it. + modules_to_build = set(REQUIRED_MODULES) + if extra_required_modules: + modules_to_build.update(extra_required_modules) + + modules_to_build.update(find_affected_modules(test_mappings, changed_files)) + + return modules_to_build + + +def find_changed_files(change_info: pathlib.Path) -> Set[Text]: + with open(change_info) as change_info_file: + change_info_contents = json.load(change_info_file) + + changed_files = set() + + for change in change_info_contents['changes']: + project_path = change.get('projectPath') + '/' + + for revision in change.get('revisions'): + for file_info in revision.get('fileInfos'): + changed_files.add(project_path + file_info.get('path')) + + return changed_files + + +def find_affected_modules( + test_mappings: Dict[str, Any], changed_files: Set[Text] +) -> Set[Text]: + modules = set() + + # The test_mappings object returned by GetTestMappings is organized as + # follows: + # { + # 'test_mapping_file_path': { + # 'group_name' : [ + # 'name': 'module_name', + # ], + # } + # } + for test_mapping in test_mappings.values(): + for group in test_mapping.values(): + for entry in group: + module_name = entry.get('name', None) + + if not module_name: + continue + + file_patterns = entry.get('file_patterns') + if not file_patterns: + modules.add(module_name) + continue + + if matches_file_patterns(file_patterns, changed_files): + modules.add(module_name) + continue + + return modules + + +# TODO(lucafarsi): Share this logic with the original logic in +# test_mapping_test_retriever.py +def matches_file_patterns( + file_patterns: list[Text], changed_files: Set[Text] +) -> bool: + for changed_file in changed_files: + for pattern in file_patterns: + if re.search(pattern, changed_file): + return True + + return False + + +def zip_build_outputs(modules_to_build: Set[Text], dist_dir: Text): + src_top = os.environ.get('TOP', os.getcwd()) + + # Call dumpvars to get the necessary things. + # TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can + # do it but it requires parsing. + host_out_testcases = get_soong_var('HOST_OUT_TESTCASES') + target_out_testcases = get_soong_var('TARGET_OUT_TESTCASES') + product_out = get_soong_var('PRODUCT_OUT') + soong_host_out = get_soong_var('SOONG_HOST_OUT') + host_out = get_soong_var('HOST_OUT') + + # Call the class to package the outputs. + # TODO(lucafarsi): Move this code into a replaceable class. + host_paths = [] + target_paths = [] + for module in modules_to_build: + host_path = os.path.join(host_out_testcases, module) + if os.path.exists(host_path): + host_paths.append(host_path) + + target_path = os.path.join(target_out_testcases, module) + if os.path.exists(target_path): + target_paths.append(target_path) + + zip_command = ['time', os.path.join(host_out, 'bin', 'soong_zip')] + + # Add host testcases. + zip_command.append('-C') + zip_command.append(os.path.join(src_top, soong_host_out)) + zip_command.append('-P') + zip_command.append('host/') + for path in host_paths: + zip_command.append('-D') + zip_command.append(path) + + # Add target testcases. + zip_command.append('-C') + zip_command.append(os.path.join(src_top, product_out)) + zip_command.append('-P') + zip_command.append('target') + for path in target_paths: + zip_command.append('-D') + zip_command.append(path) + + # TODO(lucafarsi): Push this logic into a general-tests-minimal build command + # Add necessary tools. These are also hardcoded in general-tests.mk. + framework_path = os.path.join(soong_host_out, 'framework') + + zip_command.append('-C') + zip_command.append(framework_path) + zip_command.append('-P') + zip_command.append('host/tools') + zip_command.append('-f') + zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar')) + zip_command.append('-f') + zip_command.append( + os.path.join(framework_path, 'compatibility-host-util.jar') + ) + zip_command.append('-f') + zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar')) + + # Zip to the DIST dir. + zip_command.append('-o') + zip_command.append(os.path.join(dist_dir, 'general-tests.zip')) + + run_command(zip_command) + + +def get_soong_var(var: str) -> str: + value = run_command( + ['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var] + ).strip() + if not value: + raise RuntimeError('Necessary soong variable ' + var + ' not found.') + + return value + + +def main(argv): + build_test_suites(sys.argv) diff --git a/ci/test_mapping_module_retriever.py b/ci/test_mapping_module_retriever.py new file mode 100644 index 0000000000..d2c13c0e7d --- /dev/null +++ b/ci/test_mapping_module_retriever.py @@ -0,0 +1,125 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Simple parsing code to scan test_mapping files and determine which +modules are needed to build for the given list of changed files. +TODO(lucafarsi): Deduplicate from artifact_helper.py +""" + +from typing import Any, Dict, Set, Text +import json +import os +import re + +# Regex to extra test name from the path of test config file. +TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config' + +# Key name for TEST_MAPPING imports +KEY_IMPORTS = 'imports' +KEY_IMPORT_PATH = 'path' + +# Name of TEST_MAPPING file. +TEST_MAPPING = 'TEST_MAPPING' + +# Pattern used to identify double-quoted strings and '//'-format comments in +# TEST_MAPPING file, but only double-quoted strings are included within the +# matching group. +_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?') + + +def FilterComments(test_mapping_file: Text) -> Text: + """Remove comments in TEST_MAPPING file to valid format. + + Only '//' is regarded as comments. + + Args: + test_mapping_file: Path to a TEST_MAPPING file. + + Returns: + Valid json string without comments. + """ + return re.sub(_COMMENTS_RE, r'\1', test_mapping_file) + +def GetTestMappings(paths: Set[Text], + checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]: + """Get the affected TEST_MAPPING files. + + TEST_MAPPING files in source code are packaged into a build artifact + `test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file + is preserved. From all TEST_MAPPING files in the source code, this method + locates the affected TEST_MAPPING files based on the given paths list. + + A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files + from a different location, e.g., + "imports": [ + { + "path": "../folder2" + } + ] + In that example, TEST_MAPPING files inside ../folder2 (relative to the + TEST_MAPPING file containing that imports section) and its parent directories + will also be included. + + Args: + paths: A set of paths with related TEST_MAPPING files for given changes. + checked_paths: A set of paths that have been checked for TEST_MAPPING file + already. The set is updated after processing each TEST_MAPPING file. It's + used to prevent infinite loop when the method is called recursively. + + Returns: + A dictionary of Test Mapping containing the content of the affected + TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file. + """ + test_mappings = {} + + # Search for TEST_MAPPING files in each modified path and its parent + # directories. + all_paths = set() + for path in paths: + dir_names = path.split(os.path.sep) + all_paths |= set( + [os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))]) + # Add root directory to the paths to search for TEST_MAPPING file. + all_paths.add('') + + all_paths.difference_update(checked_paths) + checked_paths |= all_paths + # Try to load TEST_MAPPING file in each possible path. + for path in all_paths: + try: + test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING') + # Read content of TEST_MAPPING file. + content = FilterComments(open(test_mapping_file, "r").read()) + test_mapping = json.loads(content) + test_mappings[path] = test_mapping + + import_paths = set() + for import_detail in test_mapping.get(KEY_IMPORTS, []): + import_path = import_detail[KEY_IMPORT_PATH] + # Try the import path as absolute path. + import_paths.add(import_path) + # Try the import path as relative path based on the test mapping file + # containing the import. + norm_import_path = os.path.normpath(os.path.join(path, import_path)) + import_paths.add(norm_import_path) + import_paths.difference_update(checked_paths) + if import_paths: + import_test_mappings = GetTestMappings(import_paths, checked_paths) + test_mappings.update(import_test_mappings) + except (KeyError, FileNotFoundError, NotADirectoryError): + # TEST_MAPPING file doesn't exist in path + pass + + return test_mappings