#!/usr/bin/env python3 """Tool to find static libraries that maybe should be shared libraries and shared libraries that maybe should be static libraries. This tool only looks at the module-info.json for the current target. Example of "class" types for each of the modules in module-info.json "EXECUTABLES": 2307, "ETC": 9094, "NATIVE_TESTS": 10461, "APPS": 2885, "JAVA_LIBRARIES": 5205, "EXECUTABLES/JAVA_LIBRARIES": 119, "FAKE": 553, "SHARED_LIBRARIES/STATIC_LIBRARIES": 7591, "STATIC_LIBRARIES": 11535, "SHARED_LIBRARIES": 10852, "HEADER_LIBRARIES": 1897, "DYLIB_LIBRARIES": 1262, "RLIB_LIBRARIES": 3413, "ROBOLECTRIC": 39, "PACKAGING": 5, "PROC_MACRO_LIBRARIES": 36, "RENDERSCRIPT_BITCODE": 17, "DYLIB_LIBRARIES/RLIB_LIBRARIES": 8, "ETC/FAKE": 1 None of the "SHARED_LIBRARIES/STATIC_LIBRARIES" are double counted in the modules with one class RLIB/ All of these classes have shared_libs and/or static_libs "EXECUTABLES", "SHARED_LIBRARIES", "STATIC_LIBRARIES", "SHARED_LIBRARIES/STATIC_LIBRARIES", # cc_library "HEADER_LIBRARIES", "NATIVE_TESTS", # test modules "DYLIB_LIBRARIES", # rust "RLIB_LIBRARIES", # rust "ETC", # rust_bindgen """ from collections import defaultdict import json, os, argparse ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT") # If a shared library is used less than MAX_SHARED_INCLUSIONS times in a target, # then it will likely save memory by changing it to a static library # This move will also use less storage MAX_SHARED_INCLUSIONS = 2 # If a static library is used more than MAX_STATIC_INCLUSIONS times in a target, # then it will likely save memory by changing it to a shared library # This move will also likely use less storage MIN_STATIC_INCLUSIONS = 3 def parse_args(): parser = argparse.ArgumentParser( description=( "Parse module-info.jso and display information about static and" " shared library dependencies." ) ) parser.add_argument( "--module", dest="module", help="Print the info for the module." ) parser.add_argument( "--shared", dest="print_shared", action=argparse.BooleanOptionalAction, help=( "Print the list of libraries that are shared_libs for fewer than {}" " modules.".format(MAX_SHARED_INCLUSIONS) ), ) parser.add_argument( "--static", dest="print_static", action=argparse.BooleanOptionalAction, help=( "Print the list of libraries that are static_libs for more than {}" " modules.".format(MIN_STATIC_INCLUSIONS) ), ) parser.add_argument( "--recursive", dest="recursive", action=argparse.BooleanOptionalAction, default=True, help=( "Gather all dependencies of EXECUTABLES recursvily before calculating" " the stats. This eliminates duplicates from multiple libraries" " including the same dependencies in a single binary." ), ) parser.add_argument( "--both", dest="both", action=argparse.BooleanOptionalAction, default=False, help=( "Print a list of libraries that are including libraries as both" " static and shared" ), ) return parser.parse_args() class TransitiveHelper: def __init__(self): # keep a list of already expanded libraries so we don't end up in a cycle self.visited = defaultdict(lambda: defaultdict(set)) # module is an object from the module-info dictionary # module_info is the dictionary from module-info.json # modify the module's shared_libs and static_libs with all of the transient # dependencies required from all of the explicit dependencies def flattenDeps(self, module, module_info): libs_snapshot = dict(shared_libs = set(module["shared_libs"]), static_libs = set(module["static_libs"])) for lib_class in ["shared_libs", "static_libs"]: for lib in libs_snapshot[lib_class]: if not lib or lib not in module_info: continue if lib in self.visited: module[lib_class].update(self.visited[lib][lib_class]) else: res = self.flattenDeps(module_info[lib], module_info) module[lib_class].update(res[lib_class]) self.visited[lib][lib_class].update(res[lib_class]) return module def main(): module_info = json.load(open(ANDROID_PRODUCT_OUT + "/module-info.json")) # turn all of the static_libs and shared_libs lists into sets to make them # easier to update for _, module in module_info.items(): module["shared_libs"] = set(module["shared_libs"]) module["static_libs"] = set(module["static_libs"]) args = parse_args() if args.module: if args.module not in module_info: print("Module {} does not exist".format(args.module)) exit(1) includedStatically = defaultdict(set) includedSharedly = defaultdict(set) includedBothly = defaultdict(set) transitive = TransitiveHelper() for name, module in module_info.items(): if args.recursive: # in this recursive mode we only want to see what is included by the executables if "EXECUTABLES" not in module["class"]: continue module = transitive.flattenDeps(module, module_info) # filter out fuzzers by their dependency on clang if "libclang_rt.fuzzer" in module["static_libs"]: continue else: if "NATIVE_TESTS" in module["class"]: # We don't care about how tests are including libraries continue # count all of the shared and static libs included in this module for lib in module["shared_libs"]: includedSharedly[lib].add(name) for lib in module["static_libs"]: includedStatically[lib].add(name) intersection = set(module["shared_libs"]).intersection( module["static_libs"] ) if intersection: includedBothly[name] = intersection if args.print_shared: print( "Shared libraries that are included by fewer than {} modules on a" " device:".format(MAX_SHARED_INCLUSIONS) ) for name, libs in includedSharedly.items(): if len(libs) < MAX_SHARED_INCLUSIONS: print("{}: {} included by: {}".format(name, len(libs), libs)) if args.print_static: print( "Libraries that are included statically by more than {} modules on a" " device:".format(MIN_STATIC_INCLUSIONS) ) for name, libs in includedStatically.items(): if len(libs) > MIN_STATIC_INCLUSIONS: print("{}: {} included by: {}".format(name, len(libs), libs)) if args.both: allIncludedBothly = set() for name, libs in includedBothly.items(): allIncludedBothly.update(libs) print( "List of libraries used both statically and shared in the same" " processes:\n {}\n\n".format("\n".join(sorted(allIncludedBothly))) ) print( "List of libraries used both statically and shared in any processes:\n {}".format("\n".join(sorted(includedStatically.keys() & includedSharedly.keys())))) if args.module: print(json.dumps(module_info[args.module], default=list, indent=2)) print( "{} is included in shared_libs {} times by these modules: {}".format( args.module, len(includedSharedly[args.module]), includedSharedly[args.module] ) ) print( "{} is included in static_libs {} times by these modules: {}".format( args.module, len(includedStatically[args.module]), includedStatically[args.module] ) ) print("Shared libs included by this module that are used in fewer than {} processes:\n{}".format( MAX_SHARED_INCLUSIONS, [x for x in module_info[args.module]["shared_libs"] if len(includedSharedly[x]) < MAX_SHARED_INCLUSIONS])) if __name__ == "__main__": main()