Merge "Add a script to optimize Test Mapping builds" into main
This commit is contained in:
Luca Farsi
Gerrit Code Review
commit
f88ce5a619
3 changed files with 429 additions and 0 deletions
22
ci/build_test_suites
Executable file
22
ci/build_test_suites
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
# Copyright 2024, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
|
||||
import build_test_suites
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
build_test_suites.main(sys.argv)
|
||||
282
ci/build_test_suites.py
Normal file
282
ci/build_test_suites.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
# Copyright 2024, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Script to build only the necessary modules for general-tests along
|
||||
|
||||
with whatever other targets are passed in.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from collections.abc import Sequence
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Dict, Set, Text
|
||||
|
||||
import test_mapping_module_retriever
|
||||
|
||||
|
||||
# List of modules that are always required to be in general-tests.zip
|
||||
REQUIRED_MODULES = frozenset(
|
||||
['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip']
|
||||
)
|
||||
|
||||
|
||||
def build_test_suites(argv):
|
||||
args = parse_args(argv)
|
||||
|
||||
if not args.change_info:
|
||||
build_everything(args)
|
||||
return
|
||||
|
||||
# Call the class to map changed files to modules to build.
|
||||
# TODO(lucafarsi): Move this into a replaceable class.
|
||||
build_affected_modules(args)
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument(
|
||||
'extra_targets', nargs='*', help='Extra test suites to build.'
|
||||
)
|
||||
argparser.add_argument('--target_product')
|
||||
argparser.add_argument('--target_release')
|
||||
argparser.add_argument(
|
||||
'--with_dexpreopt_boot_img_and_system_server_only', action='store_true'
|
||||
)
|
||||
argparser.add_argument('--dist_dir')
|
||||
argparser.add_argument('--change_info', nargs='?')
|
||||
argparser.add_argument('--extra_required_modules', nargs='*')
|
||||
|
||||
return argparser.parse_args()
|
||||
|
||||
|
||||
def build_everything(args: argparse.Namespace):
|
||||
build_command = base_build_command(args)
|
||||
build_command.append('general-tests')
|
||||
|
||||
run_command(build_command)
|
||||
|
||||
|
||||
def build_affected_modules(args: argparse.Namespace):
|
||||
modules_to_build = find_modules_to_build(
|
||||
pathlib.Path(args.change_info), args.extra_required_modules
|
||||
)
|
||||
|
||||
# Call the build command with everything.
|
||||
build_command = base_build_command(args)
|
||||
build_command.extend(modules_to_build)
|
||||
|
||||
run_command(build_command)
|
||||
|
||||
zip_build_outputs(modules_to_build, args.dist_dir)
|
||||
|
||||
|
||||
def base_build_command(args: argparse.Namespace) -> list:
|
||||
build_command = []
|
||||
build_command.append('time')
|
||||
build_command.append('./build/soong/soong_ui.bash')
|
||||
build_command.append('--make-mode')
|
||||
build_command.append('dist')
|
||||
build_command.append('DIST_DIR=' + args.dist_dir)
|
||||
build_command.append('TARGET_PRODUCT=' + args.target_product)
|
||||
build_command.append('TARGET_RELEASE=' + args.target_release)
|
||||
build_command.extend(args.extra_targets)
|
||||
|
||||
return build_command
|
||||
|
||||
|
||||
def run_command(args: list[str]) -> str:
|
||||
result = subprocess.run(
|
||||
args=args,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
# If the process failed, print its stdout and propagate the exception.
|
||||
if not result.returncode == 0:
|
||||
print('Build command failed! output:')
|
||||
print('stdout: ' + result.stdout)
|
||||
print('stderr: ' + result.stderr)
|
||||
|
||||
result.check_returncode()
|
||||
return result.stdout
|
||||
|
||||
|
||||
def find_modules_to_build(
|
||||
change_info: pathlib.Path, extra_required_modules: list[Text]
|
||||
) -> Set[Text]:
|
||||
changed_files = find_changed_files(change_info)
|
||||
|
||||
test_mappings = test_mapping_module_retriever.GetTestMappings(
|
||||
changed_files, set()
|
||||
)
|
||||
|
||||
# Soong_zip is required to generate the output zip so always build it.
|
||||
modules_to_build = set(REQUIRED_MODULES)
|
||||
if extra_required_modules:
|
||||
modules_to_build.update(extra_required_modules)
|
||||
|
||||
modules_to_build.update(find_affected_modules(test_mappings, changed_files))
|
||||
|
||||
return modules_to_build
|
||||
|
||||
|
||||
def find_changed_files(change_info: pathlib.Path) -> Set[Text]:
|
||||
with open(change_info) as change_info_file:
|
||||
change_info_contents = json.load(change_info_file)
|
||||
|
||||
changed_files = set()
|
||||
|
||||
for change in change_info_contents['changes']:
|
||||
project_path = change.get('projectPath') + '/'
|
||||
|
||||
for revision in change.get('revisions'):
|
||||
for file_info in revision.get('fileInfos'):
|
||||
changed_files.add(project_path + file_info.get('path'))
|
||||
|
||||
return changed_files
|
||||
|
||||
|
||||
def find_affected_modules(
|
||||
test_mappings: Dict[str, Any], changed_files: Set[Text]
|
||||
) -> Set[Text]:
|
||||
modules = set()
|
||||
|
||||
# The test_mappings object returned by GetTestMappings is organized as
|
||||
# follows:
|
||||
# {
|
||||
# 'test_mapping_file_path': {
|
||||
# 'group_name' : [
|
||||
# 'name': 'module_name',
|
||||
# ],
|
||||
# }
|
||||
# }
|
||||
for test_mapping in test_mappings.values():
|
||||
for group in test_mapping.values():
|
||||
for entry in group:
|
||||
module_name = entry.get('name', None)
|
||||
|
||||
if not module_name:
|
||||
continue
|
||||
|
||||
file_patterns = entry.get('file_patterns')
|
||||
if not file_patterns:
|
||||
modules.add(module_name)
|
||||
continue
|
||||
|
||||
if matches_file_patterns(file_patterns, changed_files):
|
||||
modules.add(module_name)
|
||||
continue
|
||||
|
||||
return modules
|
||||
|
||||
|
||||
# TODO(lucafarsi): Share this logic with the original logic in
|
||||
# test_mapping_test_retriever.py
|
||||
def matches_file_patterns(
|
||||
file_patterns: list[Text], changed_files: Set[Text]
|
||||
) -> bool:
|
||||
for changed_file in changed_files:
|
||||
for pattern in file_patterns:
|
||||
if re.search(pattern, changed_file):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def zip_build_outputs(modules_to_build: Set[Text], dist_dir: Text):
|
||||
src_top = os.environ.get('TOP', os.getcwd())
|
||||
|
||||
# Call dumpvars to get the necessary things.
|
||||
# TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can
|
||||
# do it but it requires parsing.
|
||||
host_out_testcases = get_soong_var('HOST_OUT_TESTCASES')
|
||||
target_out_testcases = get_soong_var('TARGET_OUT_TESTCASES')
|
||||
product_out = get_soong_var('PRODUCT_OUT')
|
||||
soong_host_out = get_soong_var('SOONG_HOST_OUT')
|
||||
host_out = get_soong_var('HOST_OUT')
|
||||
|
||||
# Call the class to package the outputs.
|
||||
# TODO(lucafarsi): Move this code into a replaceable class.
|
||||
host_paths = []
|
||||
target_paths = []
|
||||
for module in modules_to_build:
|
||||
host_path = os.path.join(host_out_testcases, module)
|
||||
if os.path.exists(host_path):
|
||||
host_paths.append(host_path)
|
||||
|
||||
target_path = os.path.join(target_out_testcases, module)
|
||||
if os.path.exists(target_path):
|
||||
target_paths.append(target_path)
|
||||
|
||||
zip_command = ['time', os.path.join(host_out, 'bin', 'soong_zip')]
|
||||
|
||||
# Add host testcases.
|
||||
zip_command.append('-C')
|
||||
zip_command.append(os.path.join(src_top, soong_host_out))
|
||||
zip_command.append('-P')
|
||||
zip_command.append('host/')
|
||||
for path in host_paths:
|
||||
zip_command.append('-D')
|
||||
zip_command.append(path)
|
||||
|
||||
# Add target testcases.
|
||||
zip_command.append('-C')
|
||||
zip_command.append(os.path.join(src_top, product_out))
|
||||
zip_command.append('-P')
|
||||
zip_command.append('target')
|
||||
for path in target_paths:
|
||||
zip_command.append('-D')
|
||||
zip_command.append(path)
|
||||
|
||||
# TODO(lucafarsi): Push this logic into a general-tests-minimal build command
|
||||
# Add necessary tools. These are also hardcoded in general-tests.mk.
|
||||
framework_path = os.path.join(soong_host_out, 'framework')
|
||||
|
||||
zip_command.append('-C')
|
||||
zip_command.append(framework_path)
|
||||
zip_command.append('-P')
|
||||
zip_command.append('host/tools')
|
||||
zip_command.append('-f')
|
||||
zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar'))
|
||||
zip_command.append('-f')
|
||||
zip_command.append(
|
||||
os.path.join(framework_path, 'compatibility-host-util.jar')
|
||||
)
|
||||
zip_command.append('-f')
|
||||
zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar'))
|
||||
|
||||
# Zip to the DIST dir.
|
||||
zip_command.append('-o')
|
||||
zip_command.append(os.path.join(dist_dir, 'general-tests.zip'))
|
||||
|
||||
run_command(zip_command)
|
||||
|
||||
|
||||
def get_soong_var(var: str) -> str:
|
||||
value = run_command(
|
||||
['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var]
|
||||
).strip()
|
||||
if not value:
|
||||
raise RuntimeError('Necessary soong variable ' + var + ' not found.')
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def main(argv):
|
||||
build_test_suites(sys.argv)
|
||||
125
ci/test_mapping_module_retriever.py
Normal file
125
ci/test_mapping_module_retriever.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
# Copyright 2024, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Simple parsing code to scan test_mapping files and determine which
|
||||
modules are needed to build for the given list of changed files.
|
||||
TODO(lucafarsi): Deduplicate from artifact_helper.py
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Set, Text
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
# Regex to extra test name from the path of test config file.
|
||||
TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config'
|
||||
|
||||
# Key name for TEST_MAPPING imports
|
||||
KEY_IMPORTS = 'imports'
|
||||
KEY_IMPORT_PATH = 'path'
|
||||
|
||||
# Name of TEST_MAPPING file.
|
||||
TEST_MAPPING = 'TEST_MAPPING'
|
||||
|
||||
# Pattern used to identify double-quoted strings and '//'-format comments in
|
||||
# TEST_MAPPING file, but only double-quoted strings are included within the
|
||||
# matching group.
|
||||
_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?')
|
||||
|
||||
|
||||
def FilterComments(test_mapping_file: Text) -> Text:
|
||||
"""Remove comments in TEST_MAPPING file to valid format.
|
||||
|
||||
Only '//' is regarded as comments.
|
||||
|
||||
Args:
|
||||
test_mapping_file: Path to a TEST_MAPPING file.
|
||||
|
||||
Returns:
|
||||
Valid json string without comments.
|
||||
"""
|
||||
return re.sub(_COMMENTS_RE, r'\1', test_mapping_file)
|
||||
|
||||
def GetTestMappings(paths: Set[Text],
|
||||
checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]:
|
||||
"""Get the affected TEST_MAPPING files.
|
||||
|
||||
TEST_MAPPING files in source code are packaged into a build artifact
|
||||
`test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file
|
||||
is preserved. From all TEST_MAPPING files in the source code, this method
|
||||
locates the affected TEST_MAPPING files based on the given paths list.
|
||||
|
||||
A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files
|
||||
from a different location, e.g.,
|
||||
"imports": [
|
||||
{
|
||||
"path": "../folder2"
|
||||
}
|
||||
]
|
||||
In that example, TEST_MAPPING files inside ../folder2 (relative to the
|
||||
TEST_MAPPING file containing that imports section) and its parent directories
|
||||
will also be included.
|
||||
|
||||
Args:
|
||||
paths: A set of paths with related TEST_MAPPING files for given changes.
|
||||
checked_paths: A set of paths that have been checked for TEST_MAPPING file
|
||||
already. The set is updated after processing each TEST_MAPPING file. It's
|
||||
used to prevent infinite loop when the method is called recursively.
|
||||
|
||||
Returns:
|
||||
A dictionary of Test Mapping containing the content of the affected
|
||||
TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file.
|
||||
"""
|
||||
test_mappings = {}
|
||||
|
||||
# Search for TEST_MAPPING files in each modified path and its parent
|
||||
# directories.
|
||||
all_paths = set()
|
||||
for path in paths:
|
||||
dir_names = path.split(os.path.sep)
|
||||
all_paths |= set(
|
||||
[os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))])
|
||||
# Add root directory to the paths to search for TEST_MAPPING file.
|
||||
all_paths.add('')
|
||||
|
||||
all_paths.difference_update(checked_paths)
|
||||
checked_paths |= all_paths
|
||||
# Try to load TEST_MAPPING file in each possible path.
|
||||
for path in all_paths:
|
||||
try:
|
||||
test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING')
|
||||
# Read content of TEST_MAPPING file.
|
||||
content = FilterComments(open(test_mapping_file, "r").read())
|
||||
test_mapping = json.loads(content)
|
||||
test_mappings[path] = test_mapping
|
||||
|
||||
import_paths = set()
|
||||
for import_detail in test_mapping.get(KEY_IMPORTS, []):
|
||||
import_path = import_detail[KEY_IMPORT_PATH]
|
||||
# Try the import path as absolute path.
|
||||
import_paths.add(import_path)
|
||||
# Try the import path as relative path based on the test mapping file
|
||||
# containing the import.
|
||||
norm_import_path = os.path.normpath(os.path.join(path, import_path))
|
||||
import_paths.add(norm_import_path)
|
||||
import_paths.difference_update(checked_paths)
|
||||
if import_paths:
|
||||
import_test_mappings = GetTestMappings(import_paths, checked_paths)
|
||||
test_mappings.update(import_test_mappings)
|
||||
except (KeyError, FileNotFoundError, NotADirectoryError):
|
||||
# TEST_MAPPING file doesn't exist in path
|
||||
pass
|
||||
|
||||
return test_mappings
|
||||
Loading…
Reference in a new issue