build: tools: Add json collection of installed files with hashes.

Adds installed-files.json in addition to installed-files.txt
Further sorts the file list to be ordered within the same size bucket.

Test: manual, cross-checked checksums with sha256sum utility
      checked build outputs.

Bug: 19988819
Merged-in: Ifb632eb4df65ec48645c8f93e36bae44ccc52ba8
Change-Id: Ifb632eb4df65ec48645c8f93e36bae44ccc52ba8
Signed-off-by: Sasha Levitskiy <sanek@google.com>

core/Makefile

@@ -1092,7 +1092,8 @@ $(INSTALLED_FILES_FILE): $(FULL_SYSTEMIMAGE_DEPS)
 	@echo Installed file list: $@
 	@mkdir -p $(dir $@)
 	@rm -f $@
-	$(hide) build/tools/fileslist.py $(TARGET_OUT) > $@
+	$(hide) build/tools/fileslist.py $(TARGET_OUT) > $(@:.txt=.json)
+	$(hide) build/tools/fileslist_util.py -c $(@:.txt=.json) > $@
 
 .PHONY: installed-file-list
 installed-file-list: $(INSTALLED_FILES_FILE)
@@ -1495,7 +1496,8 @@ $(INSTALLED_FILES_FILE_VENDOR) : $(INTERNAL_VENDORIMAGE_FILES)
 	@echo Installed file list: $@
 	@mkdir -p $(dir $@)
 	@rm -f $@
-	$(hide) build/tools/fileslist.py $(TARGET_OUT_VENDOR) > $@
+	$(hide) build/tools/fileslist.py $(TARGET_OUT_VENDOR) > $(@:.txt=.json)
+	$(hide) build/tools/fileslist_util.py -c $(@:.txt=.json) > $@
 
 vendorimage_intermediates := \
     $(call intermediates-dir-for,PACKAGING,vendor)

tools/fileslist.py

@@ -15,12 +15,24 @@
 # limitations under the License.
 #
 
-import operator, os, sys
+import json, hashlib, operator, os, sys
 
 def get_file_size(path):
   st = os.lstat(path)
   return st.st_size;
 
+def get_file_digest(path):
+  if os.path.isfile(path) == False:
+    return "----------------------------------------------------------------"
+  digest = hashlib.sha256()
+  with open(path, 'rb') as f:
+    while True:
+      buf = f.read(1024*1024)
+      if not buf:
+        break
+      digest.update(buf)
+  return digest.hexdigest();
+
 def main(argv):
   output = []
   roots = argv[1:]
@@ -30,16 +42,17 @@ def main(argv):
       relative = dir[base:]
       for f in files:
         try:
-          row = (
+          path = os.path.sep.join((dir, f))
-              get_file_size(os.path.sep.join((dir, f))),
+          row = {
-              os.path.sep.join((relative, f)),
+              "Size": get_file_size(path),
-            )
+              "Name": os.path.sep.join((relative, f)),
+              "SHA256": get_file_digest(path),
+            }
           output.append(row)
         except os.error:
           pass
-  output.sort(key=operator.itemgetter(0), reverse=True)
+  output.sort(key=operator.itemgetter("Size", "Name"), reverse=True)
-  for row in output:
+  print json.dumps(output, indent=2, separators=(',',': '))
-    print "%12d  %s" % row
 
 if __name__ == '__main__':
   main(sys.argv)

tools/fileslist_util.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import getopt, json, sys
+
+def PrintFileNames(path):
+  with open(path) as jf:
+    data = json.load(jf)
+  for line in data:
+    print(line["Name"])
+
+def PrintCanonicalList(path):
+  with open(path) as jf:
+    data = json.load(jf)
+  for line in data:
+    print "{0:12d}  {1}".format(line["Size"], line["Name"])
+
+def PrintUsage(name):
+  print("""
+Usage: %s -[nc] json_files_list
+ -n produces list of files only
+ -c produces classic installed-files.txt
+""" % (name))
+
+def main(argv):
+  try:
+    opts, args = getopt.getopt(argv[1:], "nc", "")
+  except getopt.GetoptError, err:
+    print(err)
+    PrintUsage(argv[0])
+    sys.exit(2)
+
+  if len(opts) == 0:
+    print("No conversion option specified")
+    PrintUsage(argv[0])
+    sys.exit(2)
+
+  if len(args) == 0:
+    print("No input file specified")
+    PrintUsage(argv[0])
+    sys.exit(2)
+
+  for o, a in opts:
+    if o == ("-n"):
+      PrintFileNames(args[0])
+      sys.exit()
+    elif o == ("-c"):
+      PrintCanonicalList(args[0])
+      sys.exit()
+    else:
+      assert False, "Unsupported option"
+
+if __name__ == '__main__':
+  main(sys.argv)