tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Don't sparse right-sized ext4 and erofs images.

Browse source 
When we introduced Dynamic Partitions, we stopped giving readonly
partitions fixed sizes. In addition we introduced deduplication for
ext4. These two factors greatly reduce the impact of sparse images,
since there aren't many fill blocks to optimize.

This patch disables sparsing for images that are rightsized and do not
explicitly specify extra reserved space. This makes the images a little
easier to work with from an engineering perspective. They no longer have
to be unsparsed to interact with any tooling. It also eases a potential
source of bugs, as b/184225422 is not reproducible with sparsing off.

On Pixel, the difference between the sparsed partitions and unsparsed is
12M (out of roughly 4G).

Bug: 198001223
Test: make, treehugger, make target-files-package
      dynamic partitions are no longer sparse images
Change-Id: I74459f8abe74a15a24ba5a40cf701e6af2db8179
This commit is contained in:
David Anderson 2021-08-31 21:32:45 -07:00
parent bdf9be4708
commit 9e95a02a87
3 changed files with 44 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
core/Makefile
View file

@ -1601,6 +1601,18 @@ endif
endif # PRODUCT_USE_DYNAMIC_PARTITIONS
# $(1) the partition variable component (eg SYSTEM)
# $(2) the partition prefix in properties (eg system)
# $(3) the image prop file
# $(4) optional "other" size
define add-sparse-flags-to-image-props
$(eval _size := $(BOARD_$(1)IMAGE_PARTITION_SIZE))
$(eval _reserved := $(BOARD_$(1)IMAGE_PARTITION_RESERVED_SIZE))
$(eval _headroom := $(PRODUCT_$(1)_HEADROOM))
$(if $(or $(_size), $(_reserved), $(_headroom), $(4)),,
$(hide) echo "$(2)_disable_sparse=true" >> $(3))
endef
# $(1): the path of the output dictionary file
# $(2): a subset of "system vendor cache userdata product system_ext oem odm vendor_dlkm odm_dlkm"
# $(3): additional "key=value" pairs to append to the dictionary file.
@ -1621,11 +1633,14 @@ $(if $(filter $(2),system),\
$(if $(PRODUCT_SYSTEM_BASE_FS_PATH),$(hide) echo "system_base_fs_file=$(PRODUCT_SYSTEM_BASE_FS_PATH)" >> $(1))
$(if $(PRODUCT_SYSTEM_HEADROOM),$(hide) echo "system_headroom=$(PRODUCT_SYSTEM_HEADROOM)" >> $(1))
$(if $(BOARD_SYSTEMIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "system_reserved_size=$(BOARD_SYSTEMIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,SYSTEM,system,$(1))
$(hide) echo "system_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_system_image=$(BUILDING_SYSTEM_IMAGE)" >> $(1)
)
$(if $(filter $(2),system_other),\
$(hide) echo "building_system_other_image=$(BUILDING_SYSTEM_OTHER_IMAGE)" >> $(1)
$(if $(INTERNAL_SYSTEM_OTHER_PARTITION_SIZE),,
$(hide) echo "system_other_disable_sparse=true" >> $(1))
)
$(if $(filter $(2),userdata),\
$(if $(BOARD_USERDATAIMAGE_FILE_SYSTEM_TYPE),$(hide) echo "userdata_fs_type=$(BOARD_USERDATAIMAGE_FILE_SYSTEM_TYPE)" >> $(1))
@ -1656,6 +1671,7 @@ $(if $(filter $(2),vendor),\
$(if $(BOARD_VENDORIMAGE_SQUASHFS_DISABLE_4K_ALIGN),$(hide) echo "vendor_squashfs_disable_4k_align=$(BOARD_VENDORIMAGE_SQUASHFS_DISABLE_4K_ALIGN)" >> $(1))
$(if $(PRODUCT_VENDOR_BASE_FS_PATH),$(hide) echo "vendor_base_fs_file=$(PRODUCT_VENDOR_BASE_FS_PATH)" >> $(1))
$(if $(BOARD_VENDORIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "vendor_reserved_size=$(BOARD_VENDORIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,VENDOR,vendor,$(1))
$(hide) echo "vendor_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_vendor_image=$(BUILDING_VENDOR_IMAGE)" >> $(1)
)
@ -1673,6 +1689,7 @@ $(if $(filter $(2),product),\
$(if $(BOARD_PRODUCTIMAGE_SQUASHFS_DISABLE_4K_ALIGN),$(hide) echo "product_squashfs_disable_4k_align=$(BOARD_PRODUCTIMAGE_SQUASHFS_DISABLE_4K_ALIGN)" >> $(1))
$(if $(PRODUCT_PRODUCT_BASE_FS_PATH),$(hide) echo "product_base_fs_file=$(PRODUCT_PRODUCT_BASE_FS_PATH)" >> $(1))
$(if $(BOARD_PRODUCTIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "product_reserved_size=$(BOARD_PRODUCTIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,PRODUCT,product,$(1))
$(hide) echo "product_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_product_image=$(BUILDING_PRODUCT_IMAGE)" >> $(1)
)
@ -1689,6 +1706,7 @@ $(if $(filter $(2),system_ext),\
$(if $(BOARD_SYSTEM_EXTIMAGE_SQUASHFS_BLOCK_SIZE),$(hide) echo "system_ext_squashfs_block_size=$(BOARD_SYSTEM_EXTIMAGE_SQUASHFS_BLOCK_SIZE)" >> $(1))
$(if $(BOARD_SYSTEM_EXTIMAGE_SQUASHFS_DISABLE_4K_ALIGN),$(hide) echo "system_ext_squashfs_disable_4k_align=$(BOARD_SYSTEM_EXTIMAGE_SQUASHFS_DISABLE_4K_ALIGN)" >> $(1))
$(if $(BOARD_SYSTEM_EXTIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "system_ext_reserved_size=$(BOARD_SYSTEM_EXTIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,SYSTEM_EXT,system_ext,$(1))
$(hide) echo "system_ext_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_system_ext_image=$(BUILDING_SYSTEM_EXT_IMAGE)" >> $(1)
)
@ -1704,6 +1722,7 @@ $(if $(filter $(2),odm),\
$(if $(BOARD_ODMIMAGE_SQUASHFS_DISABLE_4K_ALIGN),$(hide) echo "odm_squashfs_disable_4k_align=$(BOARD_ODMIMAGE_SQUASHFS_DISABLE_4K_ALIGN)" >> $(1))
$(if $(PRODUCT_ODM_BASE_FS_PATH),$(hide) echo "odm_base_fs_file=$(PRODUCT_ODM_BASE_FS_PATH)" >> $(1))
$(if $(BOARD_ODMIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "odm_reserved_size=$(BOARD_ODMIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,ODM,odm,$(1))
$(hide) echo "odm_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_odm_image=$(BUILDING_ODM_IMAGE)" >> $(1)
)
@ -1720,6 +1739,7 @@ $(if $(filter $(2),vendor_dlkm),\
$(if $(BOARD_VENDOR_DLKMIMAGE_SQUASHFS_BLOCK_SIZE),$(hide) echo "vendor_dlkm_squashfs_block_size=$(BOARD_VENDOR_DLKMIMAGE_SQUASHFS_BLOCK_SIZE)" >> $(1))
$(if $(BOARD_VENDOR_DLKMIMAGE_SQUASHFS_DISABLE_4K_ALIGN),$(hide) echo "vendor_dlkm_squashfs_disable_4k_align=$(BOARD_VENDOR_DLKMIMAGE_SQUASHFS_DISABLE_4K_ALIGN)" >> $(1))
$(if $(BOARD_VENDOR_DLKMIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "vendor_dlkm_reserved_size=$(BOARD_VENDOR_DLKMIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,VENDOR_DLKM,vendor_dlkm,$(1))
$(hide) echo "vendor_dlkm_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_vendor_dlkm_image=$(BUILDING_VENDOR_DLKM_IMAGE)" >> $(1)
)
@ -1734,6 +1754,7 @@ $(if $(filter $(2),odm_dlkm),\
$(if $(BOARD_ODM_DLKMIMAGE_SQUASHFS_BLOCK_SIZE),$(hide) echo "odm_dlkm_squashfs_block_size=$(BOARD_ODM_DLKMIMAGE_SQUASHFS_BLOCK_SIZE)" >> $(1))
$(if $(BOARD_ODM_DLKMIMAGE_SQUASHFS_DISABLE_4K_ALIGN),$(hide) echo "odm_dlkm_squashfs_disable_4k_align=$(BOARD_ODM_DLKMIMAGE_SQUASHFS_DISABLE_4K_ALIGN)" >> $(1))
$(if $(BOARD_ODM_DLKMIMAGE_PARTITION_RESERVED_SIZE),$(hide) echo "odm_dlkm_reserved_size=$(BOARD_ODM_DLKMIMAGE_PARTITION_RESERVED_SIZE)" >> $(1))
$(call add-sparse-flags-to-image-props,ODM_DLKM,odm_dlkm,$(1))
$(hide) echo "odm_dlkm_selinux_fc=$(SELINUX_FC)" >> $(1)
$(hide) echo "building_odm_dlkm_image=$(BUILDING_ODM_DLKM_IMAGE)" >> $(1)
)

24
tools/releasetools/build_image.py
View file

@ -256,9 +256,11 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config):
needs_casefold = prop_dict.get("needs_casefold", 0)
needs_compress = prop_dict.get("needs_compress", 0)
disable_sparse = "disable_sparse" in prop_dict
if fs_type.startswith("ext"):
build_command = [prop_dict["ext_mkuserimg"]]
if "extfs_sparse_flag" in prop_dict:
if "extfs_sparse_flag" in prop_dict and not disable_sparse:
build_command.append(prop_dict["extfs_sparse_flag"])
run_e2fsck = True
build_command.extend([in_dir, out_file, fs_type,
@ -303,7 +305,7 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config):
elif fs_type.startswith("erofs"):
build_command = ["mkerofsimage.sh"]
build_command.extend([in_dir, out_file])
if "erofs_sparse_flag" in prop_dict:
if "erofs_sparse_flag" in prop_dict and not disable_sparse:
build_command.extend([prop_dict["erofs_sparse_flag"]])
build_command.extend(["-m", prop_dict["mount_point"]])
if target_out:
@ -319,7 +321,7 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config):
elif fs_type.startswith("squash"):
build_command = ["mksquashfsimage.sh"]
build_command.extend([in_dir, out_file])
if "squashfs_sparse_flag" in prop_dict:
if "squashfs_sparse_flag" in prop_dict and not disable_sparse:
build_command.extend([prop_dict["squashfs_sparse_flag"]])
build_command.extend(["-m", prop_dict["mount_point"]])
if target_out:
@ -341,7 +343,7 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config):
elif fs_type.startswith("f2fs"):
build_command = ["mkf2fsuserimg.sh"]
build_command.extend([out_file, prop_dict["image_size"]])
if "f2fs_sparse_flag" in prop_dict:
if "f2fs_sparse_flag" in prop_dict and not disable_sparse:
build_command.extend([prop_dict["f2fs_sparse_flag"]])
if fs_config:
build_command.extend(["-C", fs_config])
@ -448,6 +450,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
# or None if not applicable.
verity_image_builder = verity_utils.CreateVerityImageBuilder(prop_dict)
disable_sparse = "disable_sparse" in prop_dict
if (prop_dict.get("use_dynamic_partition_size") == "true" and
"partition_size" not in prop_dict):
# If partition_size is not defined, use output of `du' + reserved_size.
@ -481,7 +485,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
size // BYTES_IN_MB, prop_dict["extfs_inode_count"])
BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config)
sparse_image = False
if "extfs_sparse_flag" in prop_dict:
if "extfs_sparse_flag" in prop_dict and not disable_sparse:
sparse_image = True
fs_dict = GetFilesystemCharacteristics(fs_type, out_file, sparse_image)
os.remove(out_file)
@ -525,7 +529,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
prop_dict["image_size"] = str(size)
BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config)
sparse_image = False
if "f2fs_sparse_flag" in prop_dict:
if "f2fs_sparse_flag" in prop_dict and not disable_sparse:
sparse_image = True
fs_dict = GetFilesystemCharacteristics(fs_type, out_file, sparse_image)
os.remove(out_file)
@ -642,6 +646,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("system_reserved_size", "partition_reserved_size")
copy_prop("system_selinux_fc", "selinux_fc")
copy_prop("system_disable_sparse", "disable_sparse")
elif mount_point == "system_other":
# We inherit the selinux policies of /system since we contain some of its
# files.
@ -668,6 +673,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("system_reserved_size", "partition_reserved_size")
copy_prop("system_selinux_fc", "selinux_fc")
copy_prop("system_disable_sparse", "disable_sparse")
elif mount_point == "data":
# Copy the generic fs type first, override with specific one if available.
copy_prop("fs_type", "fs_type")
@ -708,6 +714,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("vendor_reserved_size", "partition_reserved_size")
copy_prop("vendor_selinux_fc", "selinux_fc")
copy_prop("vendor_disable_sparse", "disable_sparse")
elif mount_point == "product":
copy_prop("avb_product_hashtree_enable", "avb_hashtree_enable")
copy_prop("avb_product_add_hashtree_footer_args",
@ -733,6 +740,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("product_reserved_size", "partition_reserved_size")
copy_prop("product_selinux_fc", "selinux_fc")
copy_prop("product_disable_sparse", "disable_sparse")
elif mount_point == "system_ext":
copy_prop("avb_system_ext_hashtree_enable", "avb_hashtree_enable")
copy_prop("avb_system_ext_add_hashtree_footer_args",
@ -760,6 +768,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("system_ext_reserved_size", "partition_reserved_size")
copy_prop("system_ext_selinux_fc", "selinux_fc")
copy_prop("system_ext_disable_sparse", "disable_sparse")
elif mount_point == "odm":
copy_prop("avb_odm_hashtree_enable", "avb_hashtree_enable")
copy_prop("avb_odm_add_hashtree_footer_args",
@ -783,6 +792,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("odm_reserved_size", "partition_reserved_size")
copy_prop("odm_selinux_fc", "selinux_fc")
copy_prop("odm_disable_sparse", "disable_sparse")
elif mount_point == "vendor_dlkm":
copy_prop("avb_vendor_dlkm_hashtree_enable", "avb_hashtree_enable")
copy_prop("avb_vendor_dlkm_add_hashtree_footer_args",
@ -808,6 +818,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("vendor_dlkm_reserved_size", "partition_reserved_size")
copy_prop("vendor_dlkm_selinux_fc", "selinux_fc")
copy_prop("vendor_dlkm_disable_sparse", "disable_sparse")
elif mount_point == "odm_dlkm":
copy_prop("avb_odm_dlkm_hashtree_enable", "avb_hashtree_enable")
copy_prop("avb_odm_dlkm_add_hashtree_footer_args",
@ -831,6 +842,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
d["extfs_rsv_pct"] = "0"
copy_prop("odm_dlkm_reserved_size", "partition_reserved_size")
copy_prop("odm_dlkm_selinux_fc", "selinux_fc")
copy_prop("odm_dlkm_disable_sparse", "disable_sparse")
elif mount_point == "oem":
copy_prop("fs_type", "fs_type")
copy_prop("oem_size", "partition_size")

6
tools/releasetools/common.py
View file

@ -1983,6 +1983,8 @@ def GetUserImage(which, tmpdir, input_zip,
info_dict = LoadInfoDict(input_zip)
is_sparse = info_dict.get("extfs_sparse_flag")
if info_dict.get(which + "_disable_sparse"):
is_sparse = False
# When target uses 'BOARD_EXT4_SHARE_DUP_BLOCKS := true', images may contain
# shared blocks (i.e. some blocks will show up in multiple files' block
@ -3851,12 +3853,14 @@ def GetCareMap(which, imgname):
if not image_size:
return None
disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")
image_blocks = int(image_size) // 4096 - 1
assert image_blocks > 0, "blocks for {} must be positive".format(which)
# For sparse images, we will only check the blocks that are listed in the care
# map, i.e. the ones with meaningful data.
if "extfs_sparse_flag" in OPTIONS.info_dict:
if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:
simg = sparse_img.SparseImage(imgname)
care_map_ranges = simg.care_map.intersect(
rangelib.RangeSet("0-{}".format(image_blocks)))