Merge changes Ia5ce6870,I155654cd

am: c619f99e2f

Change-Id: I5006c4e87607a55f914b90f7218802bc29fc76c5

core/Makefile

@@ -1905,6 +1905,13 @@ else
 $(BUILT_TARGET_FILES_PACKAGE): PRIVATE_RECOVERY_OUT := RECOVERY
 endif
 
+# Run fs_config while creating the target files package
+# $1: root directory
+# $2: add prefix
+define fs_config
+(cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC)
+endef
+
 # Depending on the various images guarantees that the underlying
 # directories are up-to-date.
 $(BUILT_TARGET_FILES_PACKAGE): \
@@ -2128,28 +2135,29 @@ ifdef BOARD_PREBUILT_VENDORIMAGE
 	$(hide) mkdir -p $(zip_root)/IMAGES
 	$(hide) cp $(INSTALLED_VENDORIMAGE_TARGET) $(zip_root)/IMAGES/
 endif
+	@# Run fs_config on all the system, vendor, boot ramdisk,
+	@# and recovery ramdisk files in the zip, and save the output
+	$(hide) $(call fs_config,$(zip_root)/SYSTEM,system/) > $(zip_root)/META/filesystem_config.txt
+ifdef BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE
+	$(hide) $(call fs_config,$(zip_root)/VENDOR,vendor/) > $(zip_root)/META/vendor_filesystem_config.txt
+endif
+ifeq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true)
+	$(hide) $(call fs_config,$(zip_root)/ROOT,) > $(zip_root)/META/root_filesystem_config.txt
+endif
+	$(hide) $(call fs_config,$(zip_root)/BOOT/RAMDISK,) > $(zip_root)/META/boot_filesystem_config.txt
+ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),)
+	$(hide) $(call fs_config,$(zip_root)/RECOVERY/RAMDISK,) > $(zip_root)/META/recovery_filesystem_config.txt
+endif
+ifdef INSTALLED_SYSTEMOTHERIMAGE_TARGET
+	$(hide) $(call fs_config,$(zip_root)/SYSTEM_OTHER,system/) > $(zip_root)/META/system_other_filesystem_config.txt
+endif
+	$(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
+	    ./build/tools/releasetools/add_img_to_target_files -a -v -p $(HOST_OUT) $(zip_root)
 	@# Zip everything up, preserving symlinks and placing META/ files first to
 	@# help early validation of the .zip file while uploading it.
 	$(hide) find $(zip_root)/META | sort >$@.list
 	$(hide) find $(zip_root) | grep -v "^$(zip_root)/META/" | sort >>$@.list
 	$(hide) $(SOONG_ZIP) -d -o $@ -C $(zip_root) -l $@.list
-	@# Run fs_config on all the system, vendor, boot ramdisk,
-	@# and recovery ramdisk files in the zip, and save the output
-	$(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM/" } /^SYSTEM\// {print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/filesystem_config.txt
-	$(hide) zipinfo -1 $@ | awk 'BEGIN { FS="VENDOR/" } /^VENDOR\// {print "vendor/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/vendor_filesystem_config.txt
-ifeq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true)
-	$(hide) zipinfo -1 $@ | awk 'BEGIN { FS="ROOT/" } /^ROOT\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/root_filesystem_config.txt
-endif
-	$(hide) zipinfo -1 $@ | awk 'BEGIN { FS="BOOT/RAMDISK/" } /^BOOT\/RAMDISK\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/boot_filesystem_config.txt
-ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),)
-	$(hide) zipinfo -1 $@ | awk 'BEGIN { FS="RECOVERY/RAMDISK/" } /^RECOVERY\/RAMDISK\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/recovery_filesystem_config.txt
-endif
-ifdef INSTALLED_SYSTEMOTHERIMAGE_TARGET
-	$(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM_OTHER/" } /^SYSTEM_OTHER\// { print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/system_other_filesystem_config.txt
-endif
-	$(hide) (cd $(zip_root) && zip -qX ../$(notdir $@) META/*filesystem_config.txt)
-	$(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
-	    ./build/tools/releasetools/add_img_to_target_files -a -v -p $(HOST_OUT) $@
 
 .PHONY: target-files-package
 target-files-package: $(BUILT_TARGET_FILES_PACKAGE)

tools/releasetools/add_img_to_target_files.py

@@ -73,6 +73,25 @@ OPTIONS.replace_verity_public_key = False
 OPTIONS.replace_verity_private_key = False
 OPTIONS.is_signing = False
 
+
+class OutputFile(object):
+  def __init__(self, output_zip, input_dir, prefix, name):
+    self._output_zip = output_zip
+    self.input_name = os.path.join(input_dir, prefix, name)
+
+    if self._output_zip:
+      self._zip_name = os.path.join(prefix, name)
+
+      root, suffix = os.path.splitext(name)
+      self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix)
+    else:
+      self.name = self.input_name
+
+  def Write(self):
+    if self._output_zip:
+      common.ZipWrite(self._output_zip, self.name, self._zip_name)
+
+
 def GetCareMap(which, imgname):
   """Generate care_map of system (or vendor) partition"""
 
@@ -98,10 +117,10 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
   """Turn the contents of SYSTEM into a system image and store it in
   output_zip. Returns the name of the system image file."""
 
-  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img")
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img")
-  if os.path.exists(prebuilt_path):
+  if os.path.exists(img.input_name):
     print("system.img already exists in %s, no need to rebuild..." % (prefix,))
-    return prebuilt_path
+    return img.input_name
 
   def output_sink(fn, data):
     ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
@@ -113,74 +132,52 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
     common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
                              boot_img, info_dict=OPTIONS.info_dict)
 
-  block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")
+  block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map")
-  imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,
+  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img,
               block_list=block_list)
 
-  common.ZipWrite(output_zip, imgname, prefix + "system.img")
+  return img.name
-  common.ZipWrite(output_zip, block_list, prefix + "system.map")
-  return imgname
-
-
-def BuildSystem(input_dir, info_dict, block_list=None):
-  """Build the (sparse) system image and return the name of a temp
-  file containing it."""
-  return CreateImage(input_dir, info_dict, "system", block_list=block_list)
 
 
 def AddSystemOther(output_zip, prefix="IMAGES/"):
   """Turn the contents of SYSTEM_OTHER into a system_other image
   and store it in output_zip."""
 
-  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img")
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img")
-  if os.path.exists(prebuilt_path):
+  if os.path.exists(img.input_name):
     print("system_other.img already exists in %s, no need to rebuild..." % (
         prefix,))
     return
 
-  imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict)
+  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
-  common.ZipWrite(output_zip, imgname, prefix + "system_other.img")
-
-def BuildSystemOther(input_dir, info_dict):
-  """Build the (sparse) system_other image and return the name of a temp
-  file containing it."""
-  return CreateImage(input_dir, info_dict, "system_other", block_list=None)
 
 
 def AddVendor(output_zip, prefix="IMAGES/"):
   """Turn the contents of VENDOR into a vendor image and store in it
   output_zip."""
 
-  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img")
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img")
-  if os.path.exists(prebuilt_path):
+  if os.path.exists(img.input_name):
     print("vendor.img already exists in %s, no need to rebuild..." % (prefix,))
-    return prebuilt_path
+    return img.input_name
 
-  block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")
+  block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map")
-  imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,
+  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img,
               block_list=block_list)
-  common.ZipWrite(output_zip, imgname, prefix + "vendor.img")
+  return img.name
-  common.ZipWrite(output_zip, block_list, prefix + "vendor.map")
-  return imgname
 
 
-def BuildVendor(input_dir, info_dict, block_list=None):
+def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
-  """Build the (sparse) vendor image and return the name of a temp
-  file containing it."""
-  return CreateImage(input_dir, info_dict, "vendor", block_list=block_list)
-
-
-def CreateImage(input_dir, info_dict, what, block_list=None):
   print("creating " + what + ".img...")
 
-  img = common.MakeTempFile(prefix=what + "-", suffix=".img")
-
   # The name of the directory it is making an image out of matters to
   # mkyaffs2image.  It wants "system" but we have a directory named
   # "SYSTEM", so create a symlink.
+  temp_dir = tempfile.mkdtemp()
+  OPTIONS.tempfiles.append(temp_dir)
   try:
     os.symlink(os.path.join(input_dir, what.upper()),
-               os.path.join(input_dir, what))
+               os.path.join(temp_dir, what))
   except OSError as e:
     # bogus error on my mac version?
     #   File "./build/tools/releasetools/img_from_target_files"
@@ -215,12 +212,16 @@ def CreateImage(input_dir, info_dict, what, block_list=None):
   if fs_config:
     image_props["fs_config"] = fs_config
   if block_list:
-    image_props["block_list"] = block_list
+    image_props["block_list"] = block_list.name
 
-  succ = build_image.BuildImage(os.path.join(input_dir, what),
+  succ = build_image.BuildImage(os.path.join(temp_dir, what),
-                                image_props, img)
+                                image_props, output_file.name)
   assert succ, "build " + what + ".img image failed"
 
+  output_file.Write()
+  if block_list:
+    block_list.Write()
+
   is_verity_partition = "verity_block_device" in image_props
   verity_supported = image_props.get("verity") == "true"
   if is_verity_partition and verity_supported:
@@ -229,8 +230,6 @@ def CreateImage(input_dir, info_dict, what, block_list=None):
       adjusted_blocks_key = what + "_adjusted_partition_size"
       info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1
 
-  return img
-
 
 def AddUserdata(output_zip, prefix="IMAGES/"):
   """Create a userdata image and store it in output_zip.
@@ -241,8 +240,8 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
   in OPTIONS.info_dict.
   """
 
-  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img")
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img")
-  if os.path.exists(prebuilt_path):
+  if os.path.exists(img.input_name):
     print("userdata.img already exists in %s, no need to rebuild..." % (
         prefix,))
     return
@@ -265,6 +264,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
   # empty dir named "data", or a symlink to the DATA dir,
   # and build the image from that.
   temp_dir = tempfile.mkdtemp()
+  OPTIONS.tempfiles.append(temp_dir)
   user_dir = os.path.join(temp_dir, "data")
   empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")
   if empty:
@@ -275,8 +275,6 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
     os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),
                user_dir)
 
-  img = tempfile.NamedTemporaryFile()
-
   fstab = OPTIONS.info_dict["fstab"]
   if fstab:
     image_props["fs_type"] = fstab["/data"].fs_type
@@ -284,17 +282,15 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
   assert succ, "build userdata.img image failed"
 
   common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
-  common.ZipWrite(output_zip, img.name, prefix + "userdata.img")
+  img.Write()
-  img.close()
-  shutil.rmtree(temp_dir)
 
 
 def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
   """Create a VBMeta image and store it in output_zip."""
-  _, img_file_name = tempfile.mkstemp()
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img")
   avbtool = os.getenv('AVBTOOL') or "avbtool"
   cmd = [avbtool, "make_vbmeta_image",
-         "--output", img_file_name,
+         "--output", img.name,
          "--include_descriptors_from_image", boot_img_path,
          "--include_descriptors_from_image", system_img_path,
          "--generate_dm_verity_cmdline_from_hashtree", system_img_path]
@@ -305,19 +301,19 @@ def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
   p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   p.communicate()
   assert p.returncode == 0, "avbtool make_vbmeta_image failed"
-  common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img")
+  img.Write()
 
 
 def AddPartitionTable(output_zip, prefix="IMAGES/"):
   """Create a partition table image and store it in output_zip."""
 
-  _, img_file_name = tempfile.mkstemp()
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img")
-  _, bpt_file_name = tempfile.mkstemp()
+  bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt")
 
   # use BPTTOOL from environ, or "bpttool" if empty or not set.
   bpttool = os.getenv("BPTTOOL") or "bpttool"
-  cmd = [bpttool, "make_table", "--output_json", bpt_file_name,
+  cmd = [bpttool, "make_table", "--output_json", bpt.name,
-         "--output_gpt", img_file_name]
+         "--output_gpt", img.name]
   input_files_str = OPTIONS.info_dict["board_bpt_input_files"]
   input_files = input_files_str.split(" ")
   for i in input_files:
@@ -333,15 +329,15 @@ def AddPartitionTable(output_zip, prefix="IMAGES/"):
   p.communicate()
   assert p.returncode == 0, "bpttool make_table failed"
 
-  common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img")
+  img.Write()
-  common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt")
+  bpt.Write()
 
 
 def AddCache(output_zip, prefix="IMAGES/"):
   """Create an empty cache image and store it in output_zip."""
 
-  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img")
+  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img")
-  if os.path.exists(prebuilt_path):
+  if os.path.exists(img.input_name):
     print("cache.img already exists in %s, no need to rebuild..." % (prefix,))
     return
 
@@ -362,9 +358,9 @@ def AddCache(output_zip, prefix="IMAGES/"):
   # mkyaffs2image.  So we create a temp dir, and within it we create an
   # empty dir named "cache", and build the image from that.
   temp_dir = tempfile.mkdtemp()
+  OPTIONS.tempfiles.append(temp_dir)
   user_dir = os.path.join(temp_dir, "cache")
   os.mkdir(user_dir)
-  img = tempfile.NamedTemporaryFile()
 
   fstab = OPTIONS.info_dict["fstab"]
   if fstab:
@@ -373,35 +369,39 @@ def AddCache(output_zip, prefix="IMAGES/"):
   assert succ, "build cache.img image failed"
 
   common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
-  common.ZipWrite(output_zip, img.name, prefix + "cache.img")
+  img.Write()
-  img.close()
-  os.rmdir(user_dir)
-  os.rmdir(temp_dir)
 
 
 def AddImagesToTargetFiles(filename):
+  if os.path.isdir(filename):
+    OPTIONS.input_tmp = os.path.abspath(filename)
+    input_zip = None
+  else:
     OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
 
   if not OPTIONS.add_missing:
-    for n in input_zip.namelist():
+    if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):
-      if n.startswith("IMAGES/"):
       print("target_files appears to already contain images.")
       sys.exit(1)
 
-  try:
+  has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR"))
-    input_zip.getinfo("VENDOR/")
+  has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp,
-    has_vendor = True
+                                                "SYSTEM_OTHER"))
-  except KeyError:
-    has_vendor = False
-
-  has_system_other = "SYSTEM_OTHER/" in input_zip.namelist()
 
+  if input_zip:
     OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
 
     common.ZipClose(input_zip)
     output_zip = zipfile.ZipFile(filename, "a",
                                  compression=zipfile.ZIP_DEFLATED,
                                  allowZip64=True)
+  else:
+    OPTIONS.info_dict = common.LoadInfoDict(filename, filename)
+    output_zip = None
+    images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES")
+    if not os.path.isdir(images_dir):
+      os.makedirs(images_dir)
+    images_dir = None
 
   has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")
   system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true")
@@ -422,7 +422,10 @@ def AddImagesToTargetFiles(filename):
     boot_image = common.GetBootableImage(
         "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
     if boot_image:
+      if output_zip:
         boot_image.AddToZip(output_zip)
+      else:
+        boot_image.WriteToDir(OPTIONS.input_tmp)
 
   recovery_image = None
   if has_recovery:
@@ -438,7 +441,10 @@ def AddImagesToTargetFiles(filename):
       recovery_image = common.GetBootableImage(
           "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")
       if recovery_image:
+        if output_zip:
           recovery_image.AddToZip(output_zip)
+        else:
+          recovery_image.WriteToDir(OPTIONS.input_tmp)
 
       banner("recovery (two-step image)")
       # The special recovery.img for two-step package use.
@@ -446,7 +452,10 @@ def AddImagesToTargetFiles(filename):
           "IMAGES/recovery-two-step.img", "recovery-two-step.img",
           OPTIONS.input_tmp, "RECOVERY", two_step_image=True)
       if recovery_two_step_image:
+        if output_zip:
           recovery_two_step_image.AddToZip(output_zip)
+        else:
+          recovery_two_step_image.WriteToDir(OPTIONS.input_tmp)
 
   banner("system")
   system_img_path = AddSystem(
@@ -502,23 +511,38 @@ def AddImagesToTargetFiles(filename):
       img_vendor_dir = os.path.join(
         OPTIONS.input_tmp, "VENDOR_IMAGES")
       if os.path.exists(img_radio_path):
+        if output_zip:
           common.ZipWrite(output_zip, img_radio_path,
                           os.path.join("IMAGES", img_name))
+        else:
+          shutil.copy(img_radio_path, prebuilt_path)
       else:
         for root, _, files in os.walk(img_vendor_dir):
           if img_name in files:
+            if output_zip:
               common.ZipWrite(output_zip, os.path.join(root, img_name),
                 os.path.join("IMAGES", img_name))
+            else:
+              shutil.copy(os.path.join(root, img_name), prebuilt_path)
             break
 
+      if output_zip:
         # Zip spec says: All slashes MUST be forward slashes.
         img_path = 'IMAGES/' + img_name
         assert img_path in output_zip.namelist(), "cannot find " + img_name
+      else:
+        img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
+        assert os.path.exists(img_path), "cannot find " + img_name
 
     if care_map_list:
       file_path = "META/care_map.txt"
+      if output_zip:
         common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
+      else:
+        with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp:
+          fp.write('\n'.join(care_map_list))
 
+  if output_zip:
     common.ZipClose(output_zip)
 
 def main(argv):

tools/releasetools/common.py

@@ -1191,6 +1191,10 @@ class File(object):
     t.flush()
     return t
 
+  def WriteToDir(self, d):
+    with open(os.path.join(d, self.name), "wb") as fp:
+      fp.write(self.data)
+
   def AddToZip(self, z, compression=None):
     ZipWriteStr(z, self.name, self.data, compress_type=compression)