Add * support products and modules

Passing "*" to --products is equivalent to passing the all_named_producs
build variable

Passing "*" to --modules passes the contents of
PRODUCT_OUT/all_modules.txt

The total length of the text of all_modules can easily exceed the
maximum argument size for the OS. The proper solution is likely to call
getconf ARG_MAX, then concatenate the command to be executed to check
against the limit. Instead, the modules are processed in batches of 40k
modules. As long as module names don't get extremely long, this should
keep us under the ARG_MAX limit. In testing, using --modules "*" gets
split into two batches.

Test: build/make/tools/whichgit --modules "*"
Test: build/make/tools/whichgit --modules "*" --unused
Test: build/make/tools/whichgit --modules "*" --products "*"

Existing use-cases should remain unchanged:

TEST: build/make/tools/whichgit --modules framework
Change-Id: Ifa947daea2d439df0145e6def92637b67a8b5d22

tools/whichgit

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import itertools
 import os
 import subprocess
 import sys
@@ -10,15 +11,34 @@ def get_build_var(var):
                         check=True, capture_output=True, text=True).stdout.strip()
 
 
+def get_all_modules():
+  product_out = subprocess.run(["build/soong/soong_ui.bash", "--dumpvar-mode", "--abs", "PRODUCT_OUT"],
+                                check=True, capture_output=True, text=True).stdout.strip()
+  result = subprocess.run(["cat", product_out + "/all_modules.txt"], check=True, capture_output=True, text=True)
+  return result.stdout.strip().split("\n")
+
+
+def batched(iterable, n):
+  # introduced in itertools 3.12, could delete once that's universally available
+  if n < 1:
+    raise ValueError('n must be at least one')
+  it = iter(iterable)
+  while batch := tuple(itertools.islice(it, n)):
+    yield batch
+
+
 def get_sources(modules):
-  result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f",
+  sources = set()
-                           "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja",
+  for module_group in batched(modules, 40_000):
-                           "-t", "inputs", "-d", ] + modules,
+    result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f",
-                          stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True)
+                            "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja",
-  if result.returncode != 0:
+                            "-t", "inputs", "-d", ] + list(module_group),
-    sys.stderr.write(result.stdout)
+                            stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True)
-    sys.exit(1)
+    if result.returncode != 0:
-  return set([f for f in result.stdout.split("\n") if not f.startswith("out/")])
+      sys.stderr.write(result.stdout)
+      sys.exit(1)
+    sources.update(set([f for f in result.stdout.split("\n") if not f.startswith("out/")]))
+  return sources
 
 
 def m_nothing():
@@ -57,13 +77,13 @@ def main(argv):
   # Argument parsing
   ap = argparse.ArgumentParser(description="List the required git projects for the given modules")
   ap.add_argument("--products", nargs="*",
-                  help="The TARGET_PRODUCT to check. If not provided just uses whatever has"
+                  help="One or more TARGET_PRODUCT to check, or \"*\" for all. If not provided"
-                        + " already been built")
+                        + "just uses whatever has already been built")
   ap.add_argument("--variants", nargs="*",
                   help="The TARGET_BUILD_VARIANTS to check. If not provided just uses whatever has"
                         + " already been built, or eng if --products is supplied")
   ap.add_argument("--modules", nargs="*",
-                  help="The build modules to check, or droid if not supplied")
+                  help="The build modules to check, or \"*\" for all, or droid if not supplied")
   ap.add_argument("--why", nargs="*",
                   help="Also print the input files used in these projects, or \"*\" for all")
   ap.add_argument("--unused", help="List the unused git projects for the given modules rather than"
@@ -72,22 +92,33 @@ def main(argv):
 
   modules = args.modules if args.modules else ["droid"]
 
+  match args.products:
+    case ["*"]:
+      products = get_build_var("all_named_products").split(" ")
+    case _:
+      products = args.products
+
   # Get the list of sources for all of the requested build combos
-  if not args.products and not args.variants:
+  if not products and not args.variants:
+    m_nothing()
+    if args.modules == ["*"]:
+      modules = get_all_modules()
     sources = get_sources(modules)
   else:
-    if not args.products:
+    if not products:
       sys.stderr.write("Error: --products must be supplied if --variants is supplied")
       sys.exit(1)
     sources = set()
     build_num = 1
-    for product in args.products:
+    for product in products:
       os.environ["TARGET_PRODUCT"] = product
       variants = args.variants if args.variants else ["user", "userdebug", "eng"]
       for variant in variants:
-        sys.stderr.write(f"Analyzing build {build_num} of {len(args.products)*len(variants)}\r")
+        sys.stderr.write(f"Analyzing build {build_num} of {len(products)*len(variants)}\r")
         os.environ["TARGET_BUILD_VARIANT"] = variant
         m_nothing()
+        if args.modules == ["*"]:
+          modules = get_all_modules()
         sources.update(get_sources(modules))
         build_num += 1
     sys.stderr.write("\n\n")