From 31b08073103d8e74565ddb4d5a505159c5aebcfd Mon Sep 17 00:00:00 2001 From: Tao Bao Date: Wed, 8 Nov 2017 15:50:59 -0800 Subject: [PATCH] releasetools: Reduce the memory use in test_common.py. test_common constructs a few 2GiB strings in memory, which leads to huge memory footprint (18GiB). This CL moves away from in-memory strings to generators, which reduces the memory use down to 41MiB. It also reduces the time cost from 294s to 139s as an extra benefit for free. The CL addresses some trivial pylint warnings as well. * Before $ /usr/bin/time -v python -m unittest -v test_common ... ---------------------------------------------------------------------- Ran 11 tests in 294.986s OK Command being timed: "python -m unittest -v test_common" User time (seconds): 110.51 System time (seconds): 109.34 Percent of CPU this job got: 74% Elapsed (wall clock) time (h:mm:ss or m:ss): 4:55.06 Average shared text size (kbytes): 0 Average unshared data size (kbytes): 0 Average stack size (kbytes): 0 Average total size (kbytes): 0 Maximum resident set size (kbytes): 18894172 Average resident set size (kbytes): 0 Major (requiring I/O) page faults: 1 Minor (reclaiming a frame) page faults: 20774908 Voluntary context switches: 48 Involuntary context switches: 3241 Swaps: 0 File system inputs: 184 File system outputs: 8406424 Socket messages sent: 0 Socket messages received: 0 Signals delivered: 0 Page size (bytes): 4096 Exit status: 0 * After $ /usr/bin/time -v python -m unittest -v test_common ... ---------------------------------------------------------------------- Ran 11 tests in 139.100s OK Command being timed: "python -m unittest -v test_common" User time (seconds): 59.00 System time (seconds): 4.73 Percent of CPU this job got: 45% Elapsed (wall clock) time (h:mm:ss or m:ss): 2:19.17 Average shared text size (kbytes): 0 Average unshared data size (kbytes): 0 Average stack size (kbytes): 0 Average total size (kbytes): 0 Maximum resident set size (kbytes): 41252 Average resident set size (kbytes): 0 Major (requiring I/O) page faults: 0 Minor (reclaiming a frame) page faults: 106569 Voluntary context switches: 44 Involuntary context switches: 103 Swaps: 0 File system inputs: 8 File system outputs: 8422808 Socket messages sent: 0 Socket messages received: 0 Signals delivered: 0 Page size (bytes): 4096 Exit status: 0 Fixes: 68988396 Test: See above. Change-Id: I00f16603a4ee59fb085b189c6f5b5ee9d2378690 --- tools/releasetools/test_common.py | 86 +++++++++++++++++-------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/tools/releasetools/test_common.py b/tools/releasetools/test_common.py index 3dac58950d..10ec0d3ed6 100644 --- a/tools/releasetools/test_common.py +++ b/tools/releasetools/test_common.py @@ -20,32 +20,27 @@ import time import unittest import zipfile +from hashlib import sha1 + import common import validate_target_files - -def random_string_with_holes(size, block_size, step_size): - data = ["\0"] * size - for begin in range(0, size, step_size): - end = begin + block_size - data[begin:end] = os.urandom(block_size) - return "".join(data) +KiB = 1024 +MiB = 1024 * KiB +GiB = 1024 * MiB def get_2gb_string(): - kilobytes = 1024 - megabytes = 1024 * kilobytes - gigabytes = 1024 * megabytes - - size = int(2 * gigabytes + 1) - block_size = 4 * kilobytes - step_size = 4 * megabytes - two_gb_string = random_string_with_holes( - size, block_size, step_size) - return two_gb_string + size = int(2 * GiB + 1) + block_size = 4 * KiB + step_size = 4 * MiB + # Generate a long string with holes, e.g. 'xyz\x00abc\x00...'. + for _ in range(0, size, step_size): + yield os.urandom(block_size) + yield '\0' * (step_size - block_size) class CommonZipTest(unittest.TestCase): - def _verify(self, zip_file, zip_file_name, arcname, contents, + def _verify(self, zip_file, zip_file_name, arcname, expected_hash, test_file_name=None, expected_stat=None, expected_mode=0o644, expected_compress_type=zipfile.ZIP_STORED): # Verify the stat if present. @@ -69,7 +64,11 @@ class CommonZipTest(unittest.TestCase): self.assertEqual(info.compress_type, expected_compress_type) # Verify the zip contents. - self.assertEqual(zip_file.read(arcname), contents) + entry = zip_file.open(arcname) + sha1_hash = sha1() + for chunk in iter(lambda: entry.read(4 * MiB), ''): + sha1_hash.update(chunk) + self.assertEqual(expected_hash, sha1_hash.hexdigest()) self.assertIsNone(zip_file.testzip()) def _test_ZipWrite(self, contents, extra_zipwrite_args=None): @@ -90,7 +89,10 @@ class CommonZipTest(unittest.TestCase): zip_file = zipfile.ZipFile(zip_file_name, "w") try: - test_file.write(contents) + sha1_hash = sha1() + for data in contents: + sha1_hash.update(data) + test_file.write(data) test_file.close() expected_stat = os.stat(test_file_name) @@ -102,8 +104,9 @@ class CommonZipTest(unittest.TestCase): common.ZipWrite(zip_file, test_file_name, **extra_zipwrite_args) common.ZipClose(zip_file) - self._verify(zip_file, zip_file_name, arcname, contents, test_file_name, - expected_stat, expected_mode, expected_compress_type) + self._verify(zip_file, zip_file_name, arcname, sha1_hash.hexdigest(), + test_file_name, expected_stat, expected_mode, + expected_compress_type) finally: os.remove(test_file_name) os.remove(zip_file_name) @@ -133,7 +136,7 @@ class CommonZipTest(unittest.TestCase): common.ZipWriteStr(zip_file, zinfo_or_arcname, contents, **extra_args) common.ZipClose(zip_file) - self._verify(zip_file, zip_file_name, arcname, contents, + self._verify(zip_file, zip_file_name, arcname, sha1(contents).hexdigest(), expected_mode=expected_mode, expected_compress_type=expected_compress_type) finally: @@ -159,7 +162,10 @@ class CommonZipTest(unittest.TestCase): zip_file = zipfile.ZipFile(zip_file_name, "w") try: - test_file.write(large) + sha1_hash = sha1() + for data in large: + sha1_hash.update(data) + test_file.write(data) test_file.close() expected_stat = os.stat(test_file_name) @@ -173,12 +179,13 @@ class CommonZipTest(unittest.TestCase): common.ZipClose(zip_file) # Verify the contents written by ZipWrite(). - self._verify(zip_file, zip_file_name, arcname_large, large, - test_file_name, expected_stat, expected_mode, - expected_compress_type) + self._verify(zip_file, zip_file_name, arcname_large, + sha1_hash.hexdigest(), test_file_name, expected_stat, + expected_mode, expected_compress_type) # Verify the contents written by ZipWriteStr(). - self._verify(zip_file, zip_file_name, arcname_small, small, + self._verify(zip_file, zip_file_name, arcname_small, + sha1(small).hexdigest(), expected_compress_type=expected_compress_type) finally: os.remove(zip_file_name) @@ -287,13 +294,17 @@ class CommonZipTest(unittest.TestCase): common.ZipWriteStr(zip_file, zinfo, random_string, perms=0o400) common.ZipClose(zip_file) - self._verify(zip_file, zip_file_name, "foo", random_string, + self._verify(zip_file, zip_file_name, "foo", + sha1(random_string).hexdigest(), expected_mode=0o644) - self._verify(zip_file, zip_file_name, "bar", random_string, + self._verify(zip_file, zip_file_name, "bar", + sha1(random_string).hexdigest(), expected_mode=0o755) - self._verify(zip_file, zip_file_name, "baz", random_string, + self._verify(zip_file, zip_file_name, "baz", + sha1(random_string).hexdigest(), expected_mode=0o740) - self._verify(zip_file, zip_file_name, "qux", random_string, + self._verify(zip_file, zip_file_name, "qux", + sha1(random_string).hexdigest(), expected_mode=0o400) finally: os.remove(zip_file_name) @@ -310,8 +321,7 @@ class InstallRecoveryScriptFormatTest(unittest.TestCase): dummy_fstab = \ ["/dev/soc.0/by-name/boot /boot emmc defaults defaults", "/dev/soc.0/by-name/recovery /recovery emmc defaults defaults"] - self._info["fstab"] = common.LoadRecoveryFSTab(lambda x : "\n".join(x), - 2, dummy_fstab) + self._info["fstab"] = common.LoadRecoveryFSTab("\n".join, 2, dummy_fstab) # Construct the gzipped recovery.img and boot.img self.recovery_data = bytearray([ 0x1f, 0x8b, 0x08, 0x00, 0x81, 0x11, 0x02, 0x5a, 0x00, 0x03, 0x2b, 0x4a, @@ -332,8 +342,8 @@ class InstallRecoveryScriptFormatTest(unittest.TestCase): f.write(data) def test_full_recovery(self): - recovery_image = common.File("recovery.img", self.recovery_data); - boot_image = common.File("boot.img", self.boot_data); + recovery_image = common.File("recovery.img", self.recovery_data) + boot_image = common.File("boot.img", self.boot_data) self._info["full_recovery_image"] = "true" common.MakeRecoveryPatch(self._tempdir, self._out_tmp_sink, @@ -342,9 +352,9 @@ class InstallRecoveryScriptFormatTest(unittest.TestCase): self._info) def test_recovery_from_boot(self): - recovery_image = common.File("recovery.img", self.recovery_data); + recovery_image = common.File("recovery.img", self.recovery_data) self._out_tmp_sink("recovery.img", recovery_image.data, "IMAGES") - boot_image = common.File("boot.img", self.boot_data); + boot_image = common.File("boot.img", self.boot_data) self._out_tmp_sink("boot.img", boot_image.data, "IMAGES") common.MakeRecoveryPatch(self._tempdir, self._out_tmp_sink,