diff --git a/core/Makefile b/core/Makefile index 4c6ac37efc..5d0d0e895d 100644 --- a/core/Makefile +++ b/core/Makefile @@ -641,26 +641,34 @@ $(SOONG_TO_CONVERT): $(SOONG_CONV_DATA) $(SOONG_TO_CONVERT_SCRIPT) $(hide) $(SOONG_TO_CONVERT_SCRIPT) $< >$@ $(call dist-for-goals,droidcore-unbundled,$(SOONG_TO_CONVERT)) +$(PRODUCT_OUT)/product_packages.txt: + @rm -f $@ + echo "" > $@ + $(foreach x,$(PRODUCT_PACKAGES),echo $(x) >> $@$(newline)) + MK2BP_CATALOG_SCRIPT := build/make/tools/mk2bp_catalog.py +PRODUCT_PACKAGES_TXT := $(PRODUCT_OUT)/product_packages.txt MK2BP_REMAINING_HTML := $(PRODUCT_OUT)/mk2bp_remaining.html $(MK2BP_REMAINING_HTML): PRIVATE_CODE_SEARCH_BASE_URL := "https://cs.android.com/android/platform/superproject/+/master:" -$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) +$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PACKAGES_TXT) @rm -f $@ $(hide) $(MK2BP_CATALOG_SCRIPT) \ --device=$(TARGET_DEVICE) \ + --product-packages=$(PRODUCT_PACKAGES_TXT) \ --title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \ --codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \ - --out_dir="$(OUT_DIR)" \ + --out-dir="$(OUT_DIR)" \ --mode=html \ > $@ $(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_HTML)) MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv -$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) +$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PACKAGES_TXT) @rm -f $@ $(hide) $(MK2BP_CATALOG_SCRIPT) \ --device=$(TARGET_DEVICE) \ - --out_dir="$(OUT_DIR)" \ + --product-packages=$(PRODUCT_PACKAGES_TXT) \ + --out-dir="$(OUT_DIR)" \ --mode=csv \ > $@ $(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_CSV)) diff --git a/tools/mk2bp_catalog.py b/tools/mk2bp_catalog.py index c2afb9b948..3fc6236785 100755 --- a/tools/mk2bp_catalog.py +++ b/tools/mk2bp_catalog.py @@ -308,19 +308,31 @@ def print_analysis_header(link, title): print("""%s""" % analyzer.title) print(" ") +# get all modules in $(PRODUCT_PACKAGE) and the corresponding deps +def get_module_product_packages_plus_deps(initial_modules, result, soong_data): + for module in initial_modules: + if module in result: + continue + result.add(module) + if module in soong_data.deps: + get_module_product_packages_plus_deps(soong_data.deps[module], result, soong_data) + def main(): parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.") parser.add_argument("--device", type=str, required=True, help="TARGET_DEVICE") + parser.add_argument("--product-packages", type=argparse.FileType('r'), + default=None, + help="PRODUCT_PACKAGES") parser.add_argument("--title", type=str, help="page title") parser.add_argument("--codesearch", type=str, default="https://cs.android.com/android/platform/superproject/+/master:", help="page title") - parser.add_argument("--out_dir", type=str, + parser.add_argument("--out-dir", type=str, default=None, help="Equivalent of $OUT_DIR, which will also be checked if" - + " --out_dir is unset. If neither is set, default is" + + " --out-dir is unset. If neither is set, default is" + " 'out'.") parser.add_argument("--mode", type=str, default="html", @@ -354,16 +366,25 @@ def main(): continue all_makefiles[filename] = Makefile(filename) + # Get all the modules in $(PRODUCT_PACKAGES) and the correspoding deps + product_package_modules_plus_deps = set() + if args.product_packages: + product_package_top_modules = args.product_packages.read().strip().split('\n') + get_module_product_packages_plus_deps(product_package_top_modules, product_package_modules_plus_deps, soong) + if args.mode == "html": - HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() + HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles, + product_packages_modules=product_package_modules_plus_deps).execute() elif args.mode == "csv": - CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() + CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles, + product_packages_modules=product_package_modules_plus_deps).execute() class HtmlProcessor(object): - def __init__(self, args, soong, all_makefiles): + def __init__(self, args, soong, all_makefiles, product_packages_modules): self.args = args self.soong = soong self.all_makefiles = all_makefiles + self.product_packages_modules = product_packages_modules self.annotations = Annotations() def execute(self): @@ -376,6 +397,8 @@ class HtmlProcessor(object): modules_by_partition = dict() partitions = set() for installed, module in self.soong.installed.items(): + if len(self.product_packages_modules) > 0 and module not in self.product_packages_modules: + continue partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed) modules_by_partition.setdefault(partition, []).append(module) partitions.add(partition) @@ -985,10 +1008,11 @@ class HtmlProcessor(object): return ""; class CsvProcessor(object): - def __init__(self, args, soong, all_makefiles): + def __init__(self, args, soong, all_makefiles, product_packages_modules): self.args = args self.soong = soong self.all_makefiles = all_makefiles + self.product_packages_modules = product_packages_modules def execute(self): csvout = csv.writer(sys.stdout) @@ -1004,6 +1028,8 @@ class CsvProcessor(object): for filename in sorted(self.all_makefiles.keys()): makefile = self.all_makefiles[filename] for module in self.soong.reverse_makefiles[filename]: + if len(self.product_packages_modules) > 0 and module not in self.product_packages_modules: + continue row = [filename, module] # Partitions row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, diff --git a/tools/mk2bp_partition.py b/tools/mk2bp_partition.py new file mode 100644 index 0000000000..30c1135969 --- /dev/null +++ b/tools/mk2bp_partition.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +""" +The complete list of the remaining Make files in each partition for all lunch targets + +How to run? +python3 $(path-to-file)/mk2bp_partition.py +""" + +from pathlib import Path + +import csv +import datetime +import os +import shutil +import subprocess +import sys +import time + +def get_top(): + path = '.' + while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')): + if os.path.abspath(path) == '/': + sys.exit('Could not find android source tree root.') + path = os.path.join(path, '..') + return os.path.abspath(path) + +# get the values of a build variable +def get_build_var(variable, product, build_variant): + """Returns the result of the shell command get_build_var.""" + env = { + **os.environ, + 'TARGET_PRODUCT': product if product else '', + 'TARGET_BUILD_VARIANT': build_variant if build_variant else '', + } + return subprocess.run([ + 'build/soong/soong_ui.bash', + '--dumpvar-mode', + variable + ], check=True, capture_output=True, env=env, text=True).stdout.strip() + +def get_make_file_partitions(): + lunch_targets = set(get_build_var("all_named_products", "", "").split()) + total_lunch_targets = len(lunch_targets) + makefile_by_partition = dict() + partitions = set() + current_count = 0 + start_time = time.time() + # cannot run command `m lunch_target` + broken_targets = {"mainline_sdk", "ndk"} + for lunch_target in sorted(lunch_targets): + current_count += 1 + current_time = time.time() + print (current_count, "/", total_lunch_targets, lunch_target, datetime.timedelta(seconds=current_time - start_time)) + if lunch_target in broken_targets: + continue + installed_product_out = get_build_var("PRODUCT_OUT", lunch_target, "userdebug") + filename = os.path.join(installed_product_out, "mk2bp_remaining.csv") + copy_filename = os.path.join(installed_product_out, lunch_target + "_mk2bp_remaining.csv") + # only generate if not exists + if not os.path.exists(copy_filename): + bash_cmd = "bash build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=" + lunch_target + bash_cmd += " TARGET_BUILD_VARIANT=userdebug " + filename + subprocess.run(bash_cmd, shell=True, text=True, check=True, stdout=subprocess.DEVNULL) + # generate a copied .csv file, to avoid possible overwritings + with open(copy_filename, "w") as file: + shutil.copyfile(filename, copy_filename) + + # open mk2bp_remaining.csv file + with open(copy_filename, "r") as csvfile: + reader = csv.reader(csvfile, delimiter=",", quotechar='"') + # bypass the header row + next(reader, None) + for row in reader: + # read partition information + partition = row[2] + makefile_by_partition.setdefault(partition, set()).add(row[0]) + partitions.add(partition) + + # write merged make file list for each partition into a csv file + installed_path = Path(installed_product_out).parents[0].as_posix() + csv_path = installed_path + "/mk2bp_partition.csv" + with open(csv_path, "wt") as csvfile: + writer = csv.writer(csvfile, delimiter=",") + count_makefile = 0 + for partition in sorted(partitions): + number_file = len(makefile_by_partition[partition]) + count_makefile += number_file + writer.writerow([partition, number_file]) + for makefile in sorted(makefile_by_partition[partition]): + writer.writerow([makefile]) + row = ["The total count of make files is ", count_makefile] + writer.writerow(row) + +def main(): + os.chdir(get_top()) + get_make_file_partitions() + +if __name__ == "__main__": + main()