From a61cde074737c8bb301b6100d083aa13f08f8446 Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Fri, 3 May 2019 14:19:13 -0700 Subject: [PATCH] Begins merging META/dynamic_partitions_info.txt. This file is used by OTA generation so it needs to appear in mixed builds with the combined content from the system and other versions of the file. Test: python -m unittest test_merge_target_files Test: Running merge_target_files on a dynamic-partition-enabled build and observing the resulting target files. Bug: 131889742 Change-Id: I4ddbebc087e430f6307d0bd5461121a374e58ea4 --- tools/releasetools/merge_target_files.py | 149 +++++++++++++++--- tools/releasetools/test_merge_target_files.py | 34 +++- 2 files changed, 161 insertions(+), 22 deletions(-) diff --git a/tools/releasetools/merge_target_files.py b/tools/releasetools/merge_target_files.py index f0d55d71b5..20a9c64ddd 100755 --- a/tools/releasetools/merge_target_files.py +++ b/tools/releasetools/merge_target_files.py @@ -372,6 +372,63 @@ def append_recovery_to_filesystem_config(output_target_files_temp_dir): 'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n') +def merge_dynamic_partition_info_dicts(system_dict, + other_dict, + include_dynamic_partition_list=True, + size_prefix='', + size_suffix='', + list_prefix='', + list_suffix=''): + """Merges dynamic partition info variables. + + Args: + system_dict: The dictionary of dynamic partition info variables from the + partial system target files. + other_dict: The dictionary of dynamic partition info variables from the + partial other target files. + include_dynamic_partition_list: If true, merges the dynamic_partition_list + variable. Not all use cases need this variable merged. + size_prefix: The prefix in partition group size variables that precedes the + name of the partition group. For example, partition group 'group_a' with + corresponding size variable 'super_group_a_group_size' would have the + size_prefix 'super_'. + size_suffix: Similar to size_prefix but for the variable's suffix. For + example, 'super_group_a_group_size' would have size_suffix '_group_size'. + list_prefix: Similar to size_prefix but for the partition group's + partition_list variable. + list_suffix: Similar to size_suffix but for the partition group's + partition_list variable. + + Returns: + The merged dynamic partition info dictionary. + """ + merged_dict = {} + # Partition groups and group sizes are defined by the other (non-system) + # dict because these values may vary for each board that uses a shared system + # image. + merged_dict['super_partition_groups'] = other_dict['super_partition_groups'] + if include_dynamic_partition_list: + system_dynamic_partition_list = system_dict.get('dynamic_partition_list', + '') + other_dynamic_partition_list = other_dict.get('dynamic_partition_list', '') + merged_dict['dynamic_partition_list'] = ( + '%s %s' % + (system_dynamic_partition_list, other_dynamic_partition_list)).strip() + for partition_group in merged_dict['super_partition_groups'].split(' '): + # Set the partition group's size using the value from the other dict. + key = '%s%s%s' % (size_prefix, partition_group, size_suffix) + if key not in other_dict: + raise ValueError('Other dict does not contain required key %s.' % key) + merged_dict[key] = other_dict[key] + + # Set the partition group's partition list using a concatenation of the + # system and other partition lists. + key = '%s%s%s' % (list_prefix, partition_group, list_suffix) + merged_dict[key] = ( + '%s %s' % (system_dict.get(key, ''), other_dict.get(key, ''))).strip() + return merged_dict + + def process_misc_info_txt(system_target_files_temp_dir, other_target_files_temp_dir, output_target_files_temp_dir, system_misc_info_keys): @@ -417,32 +474,76 @@ def process_misc_info_txt(system_target_files_temp_dir, # Merge misc info keys used for Dynamic Partitions. if (merged_info_dict.get('use_dynamic_partitions') == 'true') and ( system_info_dict.get('use_dynamic_partitions') == 'true'): - merged_info_dict['dynamic_partition_list'] = '%s %s' % ( - system_info_dict.get('dynamic_partition_list', ''), - merged_info_dict.get('dynamic_partition_list', '')) - # Partition groups and group sizes are defined by the other (non-system) - # misc info file because these values may vary for each board that uses - # a shared system image. - for partition_group in merged_info_dict['super_partition_groups'].split( - ' '): - if ('super_%s_group_size' % partition_group) not in merged_info_dict: - raise ValueError( - 'Other META/misc_info.txt does not contain required key ' - 'super_%s_group_size.' % partition_group) - key = 'super_%s_partition_list' % partition_group - merged_info_dict[key] = '%s %s' % (system_info_dict.get( - key, ''), merged_info_dict.get(key, '')) + merged_dynamic_partitions_dict = merge_dynamic_partition_info_dicts( + system_dict=system_info_dict, + other_dict=merged_info_dict, + size_prefix='super_', + size_suffix='_group_size', + list_prefix='super_', + list_suffix='_partition_list') + merged_info_dict.update(merged_dynamic_partitions_dict) output_misc_info_txt = os.path.join(output_target_files_temp_dir, 'META', 'misc_info.txt') - - sorted_keys = sorted(merged_info_dict.keys()) - with open(output_misc_info_txt, 'w') as output: + sorted_keys = sorted(merged_info_dict.keys()) for key in sorted_keys: output.write('{}={}\n'.format(key, merged_info_dict[key])) +def process_dynamic_partitions_info_txt(system_target_files_dir, + other_target_files_dir, + output_target_files_dir): + """Perform special processing for META/dynamic_partitions_info.txt. + + This function merges the contents of the META/dynamic_partitions_info.txt + files from the system directory and the other directory, placing the merged + result in the output directory. + + This function does nothing if META/dynamic_partitions_info.txt from the other + directory does not exist. + + Args: + system_target_files_dir: The name of a directory containing the special + items extracted from the system target files package. + other_target_files_dir: The name of a directory containing the special items + extracted from the other target files package. + output_target_files_dir: The name of a directory that will be used to create + the output target files package after all the special cases are processed. + """ + + if not os.path.exists( + os.path.join(other_target_files_dir, 'META', + 'dynamic_partitions_info.txt')): + return + + def read_helper(d): + dynamic_partitions_info_txt = os.path.join(d, 'META', + 'dynamic_partitions_info.txt') + with open(dynamic_partitions_info_txt) as f: + return list(f.read().splitlines()) + + system_dynamic_partitions_dict = common.LoadDictionaryFromLines( + read_helper(system_target_files_dir)) + other_dynamic_partitions_dict = common.LoadDictionaryFromLines( + read_helper(other_target_files_dir)) + + merged_dynamic_partitions_dict = merge_dynamic_partition_info_dicts( + system_dict=system_dynamic_partitions_dict, + other_dict=other_dynamic_partitions_dict, + # META/dynamic_partitions_info.txt does not use dynamic_partition_list. + include_dynamic_partition_list=False, + size_suffix='_size', + list_suffix='_partition_list') + + output_dynamic_partitions_info_txt = os.path.join( + output_target_files_dir, 'META', 'dynamic_partitions_info.txt') + with open(output_dynamic_partitions_info_txt, 'w') as output: + sorted_keys = sorted(merged_dynamic_partitions_dict.keys()) + for key in sorted_keys: + output.write('{}={}\n'.format(key, merged_dynamic_partitions_dict[key])) + + def process_special_cases(system_target_files_temp_dir, other_target_files_temp_dir, output_target_files_temp_dir, system_misc_info_keys, @@ -482,6 +583,11 @@ def process_special_cases(system_target_files_temp_dir, output_target_files_temp_dir=output_target_files_temp_dir, system_misc_info_keys=system_misc_info_keys) + process_dynamic_partitions_info_txt( + system_target_files_temp_dir=system_target_files_temp_dir, + other_target_files_temp_dir=other_target_files_temp_dir, + output_target_files_temp_dir=output_target_files_temp_dir) + def merge_target_files(temp_dir, system_target_files, system_item_list, system_misc_info_keys, other_target_files, @@ -648,7 +754,8 @@ def merge_target_files(temp_dir, system_target_files, system_item_list, output_target_files_meta_dir, ] find_process = common.Run(find_command, stdout=subprocess.PIPE, verbose=False) - meta_content = common.RunAndCheckOutput(['sort'], stdin=find_process.stdout, + meta_content = common.RunAndCheckOutput(['sort'], + stdin=find_process.stdout, verbose=False) find_command = [ @@ -656,7 +763,8 @@ def merge_target_files(temp_dir, system_target_files, system_item_list, output_target_files_meta_dir, '-prune', '-o', '-print' ] find_process = common.Run(find_command, stdout=subprocess.PIPE, verbose=False) - other_content = common.RunAndCheckOutput(['sort'], stdin=find_process.stdout, + other_content = common.RunAndCheckOutput(['sort'], + stdin=find_process.stdout, verbose=False) with open(output_target_files_list, 'wb') as f: @@ -686,7 +794,6 @@ def merge_target_files(temp_dir, system_target_files, system_item_list, ota_from_target_files.main(ota_from_target_files_args) - def call_func_with_temp_dir(func, keep_tmp): """Manage the creation and cleanup of the temporary directory. diff --git a/tools/releasetools/test_merge_target_files.py b/tools/releasetools/test_merge_target_files.py index 7e18a3425d..3f15d8f61f 100644 --- a/tools/releasetools/test_merge_target_files.py +++ b/tools/releasetools/test_merge_target_files.py @@ -21,7 +21,8 @@ import test_utils from merge_target_files import (read_config_list, validate_config_lists, default_system_item_list, default_other_item_list, - default_system_misc_info_keys, copy_items) + default_system_misc_info_keys, copy_items, + merge_dynamic_partition_info_dicts) class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase): @@ -128,3 +129,34 @@ class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase): self.assertFalse( validate_config_lists(default_system_item_list, system_misc_info_keys, default_other_item_list)) + + def test_merge_dynamic_partition_info_dicts_ReturnsMergedDict(self): + system_dict = { + 'super_partition_groups': 'group_a', + 'dynamic_partition_list': 'system', + 'super_group_a_list': 'system', + } + other_dict = { + 'super_partition_groups': 'group_a group_b', + 'dynamic_partition_list': 'vendor product', + 'super_group_a_list': 'vendor', + 'super_group_a_size': '1000', + 'super_group_b_list': 'product', + 'super_group_b_size': '2000', + } + merged_dict = merge_dynamic_partition_info_dicts( + system_dict=system_dict, + other_dict=other_dict, + size_prefix='super_', + size_suffix='_size', + list_prefix='super_', + list_suffix='_list') + expected_merged_dict = { + 'super_partition_groups': 'group_a group_b', + 'dynamic_partition_list': 'system vendor product', + 'super_group_a_list': 'system vendor', + 'super_group_a_size': '1000', + 'super_group_b_list': 'product', + 'super_group_b_size': '2000', + } + self.assertEqual(merged_dict, expected_merged_dict)