Merge "Generate OTA for non-A/B devices with dynamic partitions" am: 885787f4d3

am: 4b0f34da5d

Change-Id: I76ad575bf821f693b2879af5926ffd8abe73fc7e
This commit is contained in:
Yifan Hong 2019-01-16 10:58:14 -08:00 committed by android-build-merger
commit 7f7e2635cb
2 changed files with 324 additions and 41 deletions

View file

@ -14,6 +14,7 @@
from __future__ import print_function
import collections
import copy
import errno
import getopt
@ -1523,6 +1524,13 @@ class DeviceSpecificParams(object):
"""Called at the start of full OTA installation."""
return self._DoCall("FullOTA_InstallBegin")
def FullOTA_GetBlockDifferences(self):
"""Called during full OTA installation and verification.
Implementation should return a list of BlockDifference objects describing
the update on each additional partitions.
"""
return self._DoCall("FullOTA_GetBlockDifferences")
def FullOTA_InstallEnd(self):
"""Called at the end of full OTA installation; typically this is
used to install the image for the device's baseband processor."""
@ -1551,6 +1559,13 @@ class DeviceSpecificParams(object):
verification is complete)."""
return self._DoCall("IncrementalOTA_InstallBegin")
def IncrementalOTA_GetBlockDifferences(self):
"""Called during incremental OTA installation and verification.
Implementation should return a list of BlockDifference objects describing
the update on each additional partitions.
"""
return self._DoCall("IncrementalOTA_GetBlockDifferences")
def IncrementalOTA_InstallEnd(self):
"""Called at the end of incremental OTA installation; typically
this is used to install the image for the device's baseband
@ -1745,11 +1760,29 @@ class BlockDifference(object):
self.touched_src_ranges = b.touched_src_ranges
self.touched_src_sha1 = b.touched_src_sha1
if src is None:
_, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict)
# On devices with dynamic partitions, for new partitions,
# src is None but OPTIONS.source_info_dict is not.
if OPTIONS.source_info_dict is None:
is_dynamic_build = OPTIONS.info_dict.get(
"use_dynamic_partitions") == "true"
else:
_, self.device = GetTypeAndDevice("/" + partition,
OPTIONS.source_info_dict)
is_dynamic_build = OPTIONS.source_info_dict.get(
"use_dynamic_partitions") == "true"
# For dynamic partitions builds, always check partition list in target build
# because new partitions may be added.
is_dynamic = is_dynamic_build and partition in shlex.split(
OPTIONS.info_dict.get("dynamic_partition_list", "").strip())
if is_dynamic:
self.device = 'map_partition("%s")' % partition
else:
if OPTIONS.source_info_dict is None:
_, device_path = GetTypeAndDevice("/" + partition, OPTIONS.info_dict)
else:
_, device_path = GetTypeAndDevice("/" + partition,
OPTIONS.source_info_dict)
self.device = '"%s"' % device_path
@property
def required_cache(self):
@ -1768,7 +1801,7 @@ class BlockDifference(object):
self._WriteUpdate(script, output_zip)
if write_verify_script:
self._WritePostInstallVerifyScript(script)
self.WritePostInstallVerifyScript(script)
def WriteStrictVerifyScript(self, script):
"""Verify all the blocks in the care_map, including clobbered blocks.
@ -1782,11 +1815,11 @@ class BlockDifference(object):
ranges = self.tgt.care_map
ranges_str = ranges.to_string_raw()
script.AppendExtra(
'range_sha1("%s", "%s") == "%s" && ui_print(" Verified.") || '
'ui_print("\\"%s\\" has unexpected contents.");' % (
'range_sha1(%s, "%s") == "%s" && ui_print(" Verified.") || '
'ui_print("%s has unexpected contents.");' % (
self.device, ranges_str,
self.tgt.TotalSha1(include_clobbered_blocks=True),
self.device))
self.partition))
script.AppendExtra("")
def WriteVerifyScript(self, script, touched_blocks_only=False):
@ -1811,7 +1844,7 @@ class BlockDifference(object):
ranges_str = ranges.to_string_raw()
script.AppendExtra(
'if (range_sha1("%s", "%s") == "%s" || block_image_verify("%s", '
'if (range_sha1(%s, "%s") == "%s" || block_image_verify(%s, '
'package_extract_file("%s.transfer.list"), "%s.new.dat", '
'"%s.patch.dat")) then' % (
self.device, ranges_str, expected_sha1,
@ -1828,7 +1861,7 @@ class BlockDifference(object):
# this check fails, give an explicit log message about the partition
# having been remounted R/W (the most likely explanation).
if self.check_first_block:
script.AppendExtra('check_first_block("%s");' % (self.device,))
script.AppendExtra('check_first_block(%s);' % (self.device,))
# If version >= 4, try block recovery before abort update
if partition == "system":
@ -1836,8 +1869,8 @@ class BlockDifference(object):
else:
code = ErrorCode.VENDOR_RECOVER_FAILURE
script.AppendExtra((
'ifelse (block_image_recover("{device}", "{ranges}") && '
'block_image_verify("{device}", '
'ifelse (block_image_recover({device}, "{ranges}") && '
'block_image_verify({device}, '
'package_extract_file("{partition}.transfer.list"), '
'"{partition}.new.dat", "{partition}.patch.dat"), '
'ui_print("{partition} recovered successfully."), '
@ -1859,14 +1892,14 @@ class BlockDifference(object):
'abort("E%d: %s partition has unexpected contents");\n'
'endif;') % (code, partition))
def _WritePostInstallVerifyScript(self, script):
def WritePostInstallVerifyScript(self, script):
partition = self.partition
script.Print('Verifying the updated %s image...' % (partition,))
# Unlike pre-install verification, clobbered_blocks should not be ignored.
ranges = self.tgt.care_map
ranges_str = ranges.to_string_raw()
script.AppendExtra(
'if range_sha1("%s", "%s") == "%s" then' % (
'if range_sha1(%s, "%s") == "%s" then' % (
self.device, ranges_str,
self.tgt.TotalSha1(include_clobbered_blocks=True)))
@ -1875,7 +1908,7 @@ class BlockDifference(object):
if self.tgt.extended:
ranges_str = self.tgt.extended.to_string_raw()
script.AppendExtra(
'if range_sha1("%s", "%s") == "%s" then' % (
'if range_sha1(%s, "%s") == "%s" then' % (
self.device, ranges_str,
self._HashZeroBlocks(self.tgt.extended.size())))
script.Print('Verified the updated %s image.' % (partition,))
@ -1941,7 +1974,7 @@ class BlockDifference(object):
else:
code = ErrorCode.VENDOR_UPDATE_FAILURE
call = ('block_image_update("{device}", '
call = ('block_image_update({device}, '
'package_extract_file("{partition}.transfer.list"), '
'"{new_data_name}", "{partition}.patch.dat") ||\n'
' abort("E{code}: Failed to update {partition} image.");'.format(
@ -2134,3 +2167,209 @@ fi
logger.info("putting script in %s", sh_location)
output_sink(sh_location, sh)
class DynamicPartitionUpdate(object):
def __init__(self, src_group=None, tgt_group=None, progress=None,
block_difference=None):
self.src_group = src_group
self.tgt_group = tgt_group
self.progress = progress
self.block_difference = block_difference
@property
def src_size(self):
if not self.block_difference:
return 0
return DynamicPartitionUpdate._GetSparseImageSize(self.block_difference.src)
@property
def tgt_size(self):
if not self.block_difference:
return 0
return DynamicPartitionUpdate._GetSparseImageSize(self.block_difference.tgt)
@staticmethod
def _GetSparseImageSize(img):
if not img:
return 0
return img.blocksize * img.total_blocks
class DynamicGroupUpdate(object):
def __init__(self, src_size=None, tgt_size=None):
# None: group does not exist. 0: no size limits.
self.src_size = src_size
self.tgt_size = tgt_size
class DynamicPartitionsDifference(object):
def __init__(self, info_dict, block_diffs, progress_dict=None,
source_info_dict=None):
if progress_dict is None:
progress_dict = dict()
self._remove_all_before_apply = False
if source_info_dict is None:
self._remove_all_before_apply = True
source_info_dict = dict()
block_diff_dict = {e.partition:e for e in block_diffs}
assert len(block_diff_dict) == len(block_diffs), \
"Duplicated BlockDifference object for {}".format(
[partition for partition, count in
collections.Counter(e.partition for e in block_diffs).items()
if count > 1])
dynamic_partitions = set(shlex.split(info_dict.get(
"dynamic_partition_list", "").strip()))
assert set(block_diff_dict.keys()) == dynamic_partitions, \
"Dynamic partitions: {}, BlockDifference objects: {}".format(
list(dynamic_partitions), list(block_diff_dict.keys()))
self._partition_updates = dict()
for p, block_diff in block_diff_dict.items():
self._partition_updates[p] = DynamicPartitionUpdate()
self._partition_updates[p].block_difference = block_diff
for p, progress in progress_dict.items():
if p in self._partition_updates:
self._partition_updates[p].progress = progress
tgt_groups = shlex.split(info_dict.get(
"super_partition_groups", "").strip())
src_groups = shlex.split(source_info_dict.get(
"super_partition_groups", "").strip())
for g in tgt_groups:
for p in shlex.split(info_dict.get(
"super_%s_partition_list" % g, "").strip()):
assert p in self._partition_updates, \
"{} is in target super_{}_partition_list but no BlockDifference " \
"object is provided.".format(p, g)
self._partition_updates[p].tgt_group = g
for g in src_groups:
for p in shlex.split(source_info_dict.get(
"super_%s_partition_list" % g, "").strip()):
assert p in self._partition_updates, \
"{} is in source super_{}_partition_list but no BlockDifference " \
"object is provided.".format(p, g)
self._partition_updates[p].src_group = g
if self._partition_updates:
logger.info("Updating dynamic partitions %s",
self._partition_updates.keys())
self._group_updates = dict()
for g in tgt_groups:
self._group_updates[g] = DynamicGroupUpdate()
self._group_updates[g].tgt_size = int(info_dict.get(
"super_%s_group_size" % g, "0").strip())
for g in src_groups:
if g not in self._group_updates:
self._group_updates[g] = DynamicGroupUpdate()
self._group_updates[g].src_size = int(source_info_dict.get(
"super_%s_group_size" % g, "0").strip())
self._Compute()
def WriteScript(self, script, output_zip, write_verify_script=False):
script.Comment('--- Start patching dynamic partitions ---')
for p, u in self._partition_updates.items():
if u.src_size and u.tgt_size and u.src_size > u.tgt_size:
script.Comment('Patch partition %s' % p)
u.block_difference.WriteScript(script, output_zip, progress=u.progress,
write_verify_script=False)
op_list_path = MakeTempFile()
with open(op_list_path, 'w') as f:
for line in self._op_list:
f.write('{}\n'.format(line))
ZipWrite(output_zip, op_list_path, "dynamic_partitions_op_list")
script.Comment('Update dynamic partition metadata')
script.AppendExtra('assert(update_dynamic_partitions('
'package_extract_file("dynamic_partitions_op_list")));')
if write_verify_script:
for p, u in self._partition_updates.items():
if u.src_size and u.tgt_size and u.src_size > u.tgt_size:
u.block_difference.WritePostInstallVerifyScript(script)
script.AppendExtra('unmap_partition("%s");' % p) # ignore errors
for p, u in self._partition_updates.items():
if u.tgt_size and u.src_size <= u.tgt_size:
script.Comment('Patch partition %s' % p)
u.block_difference.WriteScript(script, output_zip, progress=u.progress,
write_verify_script=write_verify_script)
if write_verify_script:
script.AppendExtra('unmap_partition("%s");' % p) # ignore errors
script.Comment('--- End patching dynamic partitions ---')
def _Compute(self):
self._op_list = list()
def append(line):
self._op_list.append(line)
def comment(line):
self._op_list.append("# %s" % line)
if self._remove_all_before_apply:
comment('Remove all existing dynamic partitions and groups before '
'applying full OTA')
append('remove_all_groups')
for p, u in self._partition_updates.items():
if u.src_group and not u.tgt_group:
append('remove %s' % p)
for p, u in self._partition_updates.items():
if u.src_group and u.tgt_group and u.src_group != u.tgt_group:
comment('Move partition %s from %s to default' % (p, u.src_group))
append('move %s default' % p)
for p, u in self._partition_updates.items():
if u.src_size and u.tgt_size and u.src_size > u.tgt_size:
comment('Shrink partition %s from %d to %d' %
(p, u.src_size, u.tgt_size))
append('resize %s %s' % (p, u.tgt_size))
for g, u in self._group_updates.items():
if u.src_size is not None and u.tgt_size is None:
append('remove_group %s' % g)
if (u.src_size is not None and u.tgt_size is not None and
u.src_size > u.tgt_size):
comment('Shrink group %s from %d to %d' % (g, u.src_size, u.tgt_size))
append('resize_group %s %d' % (g, u.tgt_size))
for g, u in self._group_updates.items():
if u.src_size is None and u.tgt_size is not None:
comment('Add group %s with maximum size %d' % (g, u.tgt_size))
append('add_group %s %d' % (g, u.tgt_size))
if (u.src_size is not None and u.tgt_size is not None and
u.src_size < u.tgt_size):
comment('Grow group %s from %d to %d' % (g, u.src_size, u.tgt_size))
append('resize_group %s %d' % (g, u.tgt_size))
for p, u in self._partition_updates.items():
if u.tgt_group and not u.src_group:
comment('Add partition %s to group %s' % (p, u.tgt_group))
append('add %s %s' % (p, u.tgt_group))
for p, u in self._partition_updates.items():
if u.tgt_size and u.src_size < u.tgt_size:
comment('Grow partition %s from %d to %d' % (p, u.src_size, u.tgt_size))
append('resize %s %d' % (p, u.tgt_size))
for p, u in self._partition_updates.items():
if u.src_group and u.tgt_group and u.src_group != u.tgt_group:
comment('Move partition %s from default to %s' %
(p, u.tgt_group))
append('move %s %s' % (p, u.tgt_group))

View file

@ -826,32 +826,51 @@ else if get_stage("%(bcb_dev)s") == "3/3" then
# See the notes in WriteBlockIncrementalOTAPackage().
allow_shared_blocks = target_info.get('ext4_share_dup_blocks') == "true"
# Full OTA is done as an "incremental" against an empty source image. This
# has the effect of writing new data from the package to the entire
# partition, but lets us reuse the updater code that writes incrementals to
# do it.
system_tgt = common.GetSparseImage("system", OPTIONS.input_tmp, input_zip,
allow_shared_blocks)
system_tgt.ResetFileMap()
system_diff = common.BlockDifference("system", system_tgt, src=None)
system_diff.WriteScript(script, output_zip,
write_verify_script=OPTIONS.verify)
def GetBlockDifference(partition):
# Full OTA is done as an "incremental" against an empty source image. This
# has the effect of writing new data from the package to the entire
# partition, but lets us reuse the updater code that writes incrementals to
# do it.
tgt = common.GetSparseImage(partition, OPTIONS.input_tmp, input_zip,
allow_shared_blocks)
tgt.ResetFileMap()
diff = common.BlockDifference(partition, tgt, src=None)
return diff
boot_img = common.GetBootableImage(
"boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
device_specific_diffs = device_specific.FullOTA_GetBlockDifferences()
if device_specific_diffs:
assert all(isinstance(diff, common.BlockDifference)
for diff in device_specific_diffs), \
"FullOTA_GetBlockDifferences is not returning a list of " \
"BlockDifference objects"
progress_dict = dict()
block_diffs = [GetBlockDifference("system")]
if HasVendorPartition(input_zip):
script.ShowProgress(0.1, 0)
block_diffs.append(GetBlockDifference("vendor"))
progress_dict["vendor"] = 0.1
if device_specific_diffs:
block_diffs += device_specific_diffs
vendor_tgt = common.GetSparseImage("vendor", OPTIONS.input_tmp, input_zip,
allow_shared_blocks)
vendor_tgt.ResetFileMap()
vendor_diff = common.BlockDifference("vendor", vendor_tgt)
vendor_diff.WriteScript(script, output_zip,
write_verify_script=OPTIONS.verify)
if target_info.get('use_dynamic_partitions') == "true":
# Use empty source_info_dict to indicate that all partitions / groups must
# be re-added.
dynamic_partitions_diff = common.DynamicPartitionsDifference(
info_dict=OPTIONS.info_dict,
block_diffs=block_diffs,
progress_dict=progress_dict)
dynamic_partitions_diff.WriteScript(script, output_zip,
write_verify_script=OPTIONS.verify)
else:
for block_diff in block_diffs:
block_diff.WriteScript(script, output_zip,
progress=progress_dict.get(block_diff.partition),
write_verify_script=OPTIONS.verify)
AddCompatibilityArchiveIfTrebleEnabled(input_zip, output_zip, target_info)
boot_img = common.GetBootableImage(
"boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
common.CheckSize(boot_img.data, "boot.img", target_info)
common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
@ -1571,18 +1590,43 @@ else
system_diff.WriteVerifyScript(script, touched_blocks_only=True)
if vendor_diff:
vendor_diff.WriteVerifyScript(script, touched_blocks_only=True)
device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences()
if device_specific_diffs:
assert all(isinstance(diff, common.BlockDifference)
for diff in device_specific_diffs), \
"IncrementalOTA_GetBlockDifferences is not returning a list of " \
"BlockDifference objects"
for diff in device_specific_diffs:
diff.WriteVerifyScript(script, touched_blocks_only=True)
script.Comment("---- start making changes here ----")
device_specific.IncrementalOTA_InstallBegin()
system_diff.WriteScript(script, output_zip,
progress=0.8 if vendor_diff else 0.9,
write_verify_script=OPTIONS.verify)
block_diffs = [system_diff]
progress_dict = {"system": 0.8 if vendor_diff else 0.9}
if vendor_diff:
vendor_diff.WriteScript(script, output_zip, progress=0.1,
write_verify_script=OPTIONS.verify)
block_diffs.append(vendor_diff)
progress_dict["vendor"] = 0.1
if device_specific_diffs:
block_diffs += device_specific_diffs
if OPTIONS.source_info_dict.get("use_dynamic_partitions") == "true":
if OPTIONS.target_info_dict.get("use_dynamic_partitions") != "true":
raise RuntimeError(
"can't generate incremental that disables dynamic partitions")
dynamic_partitions_diff = common.DynamicPartitionsDifference(
info_dict=OPTIONS.target_info_dict,
source_info_dict=OPTIONS.source_info_dict,
block_diffs=block_diffs,
progress_dict=progress_dict)
dynamic_partitions_diff.WriteScript(
script, output_zip, write_verify_script=OPTIONS.verify)
else:
for block_diff in block_diffs:
block_diff.WriteScript(script, output_zip,
progress=progress_dict.get(block_diff.partition),
write_verify_script=OPTIONS.verify)
if OPTIONS.two_step:
common.ZipWriteStr(output_zip, "boot.img", target_boot.data)