Merge "releasetools: Support verifying files with non-monotonic ranges." am: 99d855db8f

am: bb20e1554b

Change-Id: I6406f40b539017baaf202917befd716e6528a6a1

tools/releasetools/sparse_img.py

@@ -249,8 +249,9 @@ class SparseImage(object):
 
     with open(fn) as f:
       for line in f:
-        fn, ranges = line.split(None, 1)
+        fn, ranges_text = line.rstrip().split(None, 1)
-        ranges = rangelib.RangeSet.parse(ranges)
+        ranges = rangelib.RangeSet.parse(ranges_text)
+        ranges.extra['text_str'] = ranges_text
 
         if allow_shared_blocks:
           # Find the shared blocks that have been claimed by others. If so, tag
@@ -261,9 +262,6 @@ class SparseImage(object):
             if not non_shared:
               continue
 
-            # There shouldn't anything in the extra dict yet.
-            assert not ranges.extra, "Non-empty RangeSet.extra"
-
             # Put the non-shared RangeSet as the value in the block map, which
             # has a copy of the original RangeSet.
             non_shared.extra['uses_shared_blocks'] = ranges

tools/releasetools/test_validate_target_files.py

@@ -238,14 +238,14 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase):
     system_root = os.path.join(input_tmp, "SYSTEM")
     os.mkdir(system_root)
 
-    # Write the test file that contain multiple blocks of zeros, and these
+    # Write test files that contain multiple blocks of zeros, and these zero
-    # zero blocks will be omitted by kernel. And the test files will occupy one
+    # blocks will be omitted by kernel. Each test file will occupy one block in
-    # block range each in the final system image.
+    # the final system image.
     with open(os.path.join(system_root, 'a'), 'w') as f:
-      f.write("aaa")
+      f.write('aaa')
       f.write('\0' * 4096 * 3)
     with open(os.path.join(system_root, 'b'), 'w') as f:
-      f.write("bbb")
+      f.write('bbb')
       f.write('\0' * 4096 * 3)
 
     raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')
@@ -254,7 +254,7 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase):
     # Parse the generated file map and update the block ranges for each file.
     file_map_list = {}
     image_ranges = RangeSet()
-    with open(raw_file_map, 'r') as f:
+    with open(raw_file_map) as f:
       for line in f.readlines():
         info = line.split()
         self.assertEqual(2, len(info))
@@ -265,7 +265,7 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase):
     mock_shared_block = RangeSet("10-20").subtract(image_ranges).first(1)
     with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:
       for key in sorted(file_map_list.keys()):
-        line = "{} {}\n".format(
+        line = '{} {}\n'.format(
             key, file_map_list[key].union(mock_shared_block))
         f.write(line)
 
@@ -277,9 +277,55 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase):
       for name in all_entries:
         input_zip.write(os.path.join(input_tmp, name), arcname=name)
 
-    input_zip = zipfile.ZipFile(input_file, 'r')
-    info_dict = {'extfs_sparse_flag': '-s'}
-
     # Expect the validation to pass and both files are skipped due to
     # 'incomplete' block range.
+    with zipfile.ZipFile(input_file) as input_zip:
+      info_dict = {'extfs_sparse_flag': '-s'}
+      ValidateFileConsistency(input_zip, input_tmp, info_dict)
+
+  @test_utils.SkipIfExternalToolsUnavailable()
+  def test_ValidateFileConsistency_nonMonotonicRanges(self):
+    input_tmp = common.MakeTempDir()
+    os.mkdir(os.path.join(input_tmp, 'IMAGES'))
+    system_image = os.path.join(input_tmp, 'IMAGES', 'system.img')
+    system_root = os.path.join(input_tmp, "SYSTEM")
+    os.mkdir(system_root)
+
+    # Write the test file that contain three blocks of 'a', 'b', 'c'.
+    with open(os.path.join(system_root, 'abc'), 'w') as f:
+      f.write('a' * 4096 + 'b' * 4096 + 'c' * 4096)
+    raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')
+    self._generate_system_image(system_image, system_root, raw_file_map)
+
+    # Parse the generated file map and manipulate the block ranges of 'abc' to
+    # be 'cba'.
+    file_map_list = {}
+    with open(raw_file_map) as f:
+      for line in f.readlines():
+        info = line.split()
+        self.assertEqual(2, len(info))
+        ranges = RangeSet(info[1])
+        self.assertTrue(ranges.monotonic)
+        blocks = reversed(list(ranges.next_item()))
+        file_map_list[info[0]] = ' '.join([str(block) for block in blocks])
+
+    # Update the contents of 'abc' to be 'cba'.
+    with open(os.path.join(system_root, 'abc'), 'w') as f:
+      f.write('c' * 4096 + 'b' * 4096 + 'a' * 4096)
+
+    # Update the system.map.
+    with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:
+      for key in sorted(file_map_list.keys()):
+        f.write('{} {}\n'.format(key, file_map_list[key]))
+
+    # Get the target zip file.
+    input_file = common.MakeTempFile()
+    all_entries = ['SYSTEM/', 'SYSTEM/abc', 'IMAGES/',
+                   'IMAGES/system.map', 'IMAGES/system.img']
+    with zipfile.ZipFile(input_file, 'w') as input_zip:
+      for name in all_entries:
+        input_zip.write(os.path.join(input_tmp, name), arcname=name)
+
+    with zipfile.ZipFile(input_file) as input_zip:
+      info_dict = {'extfs_sparse_flag': '-s'}
       ValidateFileConsistency(input_zip, input_tmp, info_dict)

tools/releasetools/validate_target_files.py

@@ -36,20 +36,21 @@ import logging
 import os.path
 import re
 import zipfile
+from hashlib import sha1
 
 import common
+import rangelib
 
 
 def _ReadFile(file_name, unpacked_name, round_up=False):
   """Constructs and returns a File object. Rounds up its size if needed."""
-
   assert os.path.exists(unpacked_name)
   with open(unpacked_name, 'rb') as f:
     file_data = f.read()
   file_size = len(file_data)
   if round_up:
     file_size_rounded_up = common.RoundUpTo4K(file_size)
-    file_data += '\0' * (file_size_rounded_up - file_size)
+    file_data += b'\0' * (file_size_rounded_up - file_size)
   return common.File(file_name, file_data)
 
 
@@ -96,12 +97,14 @@ def ValidateFileConsistency(input_zip, input_tmp, info_dict):
         logging.warning('Skipping %s that has incomplete block list', entry)
         continue
 
-      # TODO(b/79951650): Handle files with non-monotonic ranges.
+      # If the file has non-monotonic ranges, read each range in order.
       if not file_ranges.monotonic:
-        logging.warning(
+        h = sha1()
-            'Skipping %s that has non-monotonic ranges: %s', entry, file_ranges)
+        for file_range in file_ranges.extra['text_str'].split(' '):
-        continue
+          for data in image.ReadRangeSet(rangelib.RangeSet(file_range)):
-
+            h.update(data)
+        blocks_sha1 = h.hexdigest()
+      else:
         blocks_sha1 = image.RangeSha1(file_ranges)
 
       # The filename under unpacked directory, such as SYSTEM/bin/sh.