Make file list by partitions for all lunch targets

1. Filtered out the Android.mk file if the Make modules on them are
   not in the PRODUCT_PACKAGES or the correspoding deps;
2. Merged the Android.mk file list by partitions for all lunch targets.

Bug: 225256154
Test: lunch aosp_coral-eng; m
out/target/product/coral/mk2bp_remaining.html
Test: python3 build/make/tools/mk2bp_partition.py

Change-Id: I40919e81ab3c6bd516379f0acd72932e8db50088

core/Makefile

@@ -641,26 +641,34 @@ $(SOONG_TO_CONVERT): $(SOONG_CONV_DATA) $(SOONG_TO_CONVERT_SCRIPT)
 	$(hide) $(SOONG_TO_CONVERT_SCRIPT) $< >$@
 $(call dist-for-goals,droidcore-unbundled,$(SOONG_TO_CONVERT))
 
+$(PRODUCT_OUT)/product_packages.txt:
+	@rm -f $@
+	echo "" > $@
+	$(foreach x,$(PRODUCT_PACKAGES),echo $(x) >> $@$(newline))
+
 MK2BP_CATALOG_SCRIPT := build/make/tools/mk2bp_catalog.py
+PRODUCT_PACKAGES_TXT := $(PRODUCT_OUT)/product_packages.txt
 MK2BP_REMAINING_HTML := $(PRODUCT_OUT)/mk2bp_remaining.html
 $(MK2BP_REMAINING_HTML): PRIVATE_CODE_SEARCH_BASE_URL := "https://cs.android.com/android/platform/superproject/+/master:"
-$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
+$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PACKAGES_TXT)
 	@rm -f $@
 	$(hide) $(MK2BP_CATALOG_SCRIPT) \
 		--device=$(TARGET_DEVICE) \
+		--product-packages=$(PRODUCT_PACKAGES_TXT) \
 		--title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \
 		--codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \
-		--out_dir="$(OUT_DIR)" \
+		--out-dir="$(OUT_DIR)" \
 		--mode=html \
 		> $@
 $(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_HTML))
 
 MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv
-$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
+$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PACKAGES_TXT)
 	@rm -f $@
 	$(hide) $(MK2BP_CATALOG_SCRIPT) \
 		--device=$(TARGET_DEVICE) \
-		--out_dir="$(OUT_DIR)" \
+		--product-packages=$(PRODUCT_PACKAGES_TXT) \
+		--out-dir="$(OUT_DIR)" \
 		--mode=csv \
 		> $@
 $(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_CSV))

tools/mk2bp_catalog.py

@@ -308,19 +308,31 @@ def print_analysis_header(link, title):
     print("""<th class="Count Warning">%s</th>""" % analyzer.title)
   print("      </tr>")
 
+# get all modules in $(PRODUCT_PACKAGE) and the corresponding deps
+def get_module_product_packages_plus_deps(initial_modules, result, soong_data):
+  for module in initial_modules:
+    if module in result:
+      continue
+    result.add(module)
+    if module in soong_data.deps:
+      get_module_product_packages_plus_deps(soong_data.deps[module], result, soong_data)
+
 def main():
   parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.")
   parser.add_argument("--device", type=str, required=True,
                       help="TARGET_DEVICE")
+  parser.add_argument("--product-packages", type=argparse.FileType('r'),
+                      default=None,
+                      help="PRODUCT_PACKAGES")
   parser.add_argument("--title", type=str,
                       help="page title")
   parser.add_argument("--codesearch", type=str,
                       default="https://cs.android.com/android/platform/superproject/+/master:",
                       help="page title")
-  parser.add_argument("--out_dir", type=str,
+  parser.add_argument("--out-dir", type=str,
                       default=None,
                       help="Equivalent of $OUT_DIR, which will also be checked if"
-                        + " --out_dir is unset. If neither is set, default is"
+                        + " --out-dir is unset. If neither is set, default is"
                         + " 'out'.")
   parser.add_argument("--mode", type=str,
                       default="html",
@@ -354,16 +366,25 @@ def main():
       continue
     all_makefiles[filename] = Makefile(filename)
 
+  # Get all the modules in $(PRODUCT_PACKAGES) and the correspoding deps
+  product_package_modules_plus_deps = set()
+  if args.product_packages:
+    product_package_top_modules = args.product_packages.read().strip().split('\n')
+    get_module_product_packages_plus_deps(product_package_top_modules, product_package_modules_plus_deps, soong)
+
   if args.mode == "html":
-    HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
+    HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles,
+        product_packages_modules=product_package_modules_plus_deps).execute()
   elif args.mode == "csv":
-    CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
+    CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles,
+        product_packages_modules=product_package_modules_plus_deps).execute()
 
 class HtmlProcessor(object):
-  def __init__(self, args, soong, all_makefiles):
+  def __init__(self, args, soong, all_makefiles, product_packages_modules):
     self.args = args
     self.soong = soong
     self.all_makefiles = all_makefiles
+    self.product_packages_modules = product_packages_modules
     self.annotations = Annotations()
 
   def execute(self):
@@ -376,6 +397,8 @@ class HtmlProcessor(object):
     modules_by_partition = dict()
     partitions = set()
     for installed, module in self.soong.installed.items():
+      if len(self.product_packages_modules) > 0 and module not in self.product_packages_modules:
+        continue
       partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed)
       modules_by_partition.setdefault(partition, []).append(module)
       partitions.add(partition)
@@ -985,10 +1008,11 @@ class HtmlProcessor(object):
       return "";
 
 class CsvProcessor(object):
-  def __init__(self, args, soong, all_makefiles):
+  def __init__(self, args, soong, all_makefiles, product_packages_modules):
     self.args = args
     self.soong = soong
     self.all_makefiles = all_makefiles
+    self.product_packages_modules = product_packages_modules
 
   def execute(self):
     csvout = csv.writer(sys.stdout)
@@ -1004,6 +1028,8 @@ class CsvProcessor(object):
     for filename in sorted(self.all_makefiles.keys()):
       makefile = self.all_makefiles[filename]
       for module in self.soong.reverse_makefiles[filename]:
+        if len(self.product_packages_modules) > 0 and module not in self.product_packages_modules:
+          continue
         row = [filename, module]
         # Partitions
         row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT,

tools/mk2bp_partition.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+"""
+The complete list of the remaining Make files in each partition for all lunch targets
+
+How to run?
+python3 $(path-to-file)/mk2bp_partition.py
+"""
+
+from pathlib import Path
+
+import csv
+import datetime
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+def get_top():
+  path = '.'
+  while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')):
+    if os.path.abspath(path) == '/':
+      sys.exit('Could not find android source tree root.')
+    path = os.path.join(path, '..')
+  return os.path.abspath(path)
+
+# get the values of a build variable
+def get_build_var(variable, product, build_variant):
+  """Returns the result of the shell command get_build_var."""
+  env = {
+      **os.environ,
+      'TARGET_PRODUCT': product if product else '',
+      'TARGET_BUILD_VARIANT': build_variant if build_variant else '',
+  }
+  return subprocess.run([
+      'build/soong/soong_ui.bash',
+      '--dumpvar-mode',
+      variable
+  ], check=True, capture_output=True, env=env, text=True).stdout.strip()
+
+def get_make_file_partitions():
+    lunch_targets = set(get_build_var("all_named_products", "", "").split())
+    total_lunch_targets = len(lunch_targets)
+    makefile_by_partition = dict()
+    partitions = set()
+    current_count = 0
+    start_time = time.time()
+    # cannot run command `m lunch_target`
+    broken_targets = {"mainline_sdk", "ndk"}
+    for lunch_target in sorted(lunch_targets):
+        current_count += 1
+        current_time = time.time()
+        print (current_count, "/", total_lunch_targets, lunch_target, datetime.timedelta(seconds=current_time - start_time))
+        if lunch_target in broken_targets:
+            continue
+        installed_product_out = get_build_var("PRODUCT_OUT", lunch_target, "userdebug")
+        filename = os.path.join(installed_product_out, "mk2bp_remaining.csv")
+        copy_filename = os.path.join(installed_product_out, lunch_target + "_mk2bp_remaining.csv")
+        # only generate if not exists
+        if not os.path.exists(copy_filename):
+            bash_cmd = "bash build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=" + lunch_target
+            bash_cmd += " TARGET_BUILD_VARIANT=userdebug " + filename
+            subprocess.run(bash_cmd, shell=True, text=True, check=True, stdout=subprocess.DEVNULL)
+            # generate a copied .csv file, to avoid possible overwritings
+            with open(copy_filename, "w") as file:
+                shutil.copyfile(filename, copy_filename)
+
+        # open mk2bp_remaining.csv file
+        with open(copy_filename, "r") as csvfile:
+            reader = csv.reader(csvfile, delimiter=",", quotechar='"')
+            # bypass the header row
+            next(reader, None)
+            for row in reader:
+                # read partition information
+                partition = row[2]
+                makefile_by_partition.setdefault(partition, set()).add(row[0])
+                partitions.add(partition)
+
+    # write merged make file list for each partition into a csv file
+    installed_path = Path(installed_product_out).parents[0].as_posix()
+    csv_path = installed_path + "/mk2bp_partition.csv"
+    with open(csv_path, "wt") as csvfile:
+        writer = csv.writer(csvfile, delimiter=",")
+        count_makefile = 0
+        for partition in sorted(partitions):
+            number_file = len(makefile_by_partition[partition])
+            count_makefile += number_file
+            writer.writerow([partition, number_file])
+            for makefile in sorted(makefile_by_partition[partition]):
+                writer.writerow([makefile])
+        row = ["The total count of make files is ", count_makefile]
+        writer.writerow(row)
+
+def main():
+    os.chdir(get_top())
+    get_make_file_partitions()
+
+if __name__ == "__main__":
+    main()