Precompile python sources

This signifigantly improves the startup time of soong-built
python binaries. For example, running
`m apexer && time out/host/linux-x86/bin/apexer` gives
0.734s before this cl, and 0.094s after.

Fixes: 259718110
Test: Presubmits
Change-Id: Ib19e83e2c60c39a849525be117279c318de3afa7

cmd/merge_zips/merge_zips.go

@@ -359,7 +359,8 @@ func (oz *OutputZip) getUninitializedPythonPackages(inputZips []InputZip) ([]str
 		}
 		for _, file := range inputZip.Entries() {
 			pyPkg := getPackage(file.Name)
-			if filepath.Base(file.Name) == "__init__.py" {
+			baseName := filepath.Base(file.Name)
+			if baseName == "__init__.py" || baseName == "__init__.pyc" {
 				if _, found := initedPackages[pyPkg]; found {
 					panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q", file.Name))
 				}

java/java_test.go

@@ -30,7 +30,6 @@ import (
 	"android/soong/cc"
 	"android/soong/dexpreopt"
 	"android/soong/genrule"
-	"android/soong/python"
 )
 
 // Legacy preparer used for running tests within the java package.
@@ -49,7 +48,6 @@ var prepareForJavaTest = android.GroupFixturePreparers(
 	// Include all the default java modules.
 	PrepareForTestWithJavaDefaultModules,
 	PrepareForTestWithOverlayBuildComponents,
-	python.PrepareForTestWithPythonBuildComponents,
 	android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
 		ctx.RegisterPreSingletonType("sdk_versions", sdkPreSingletonFactory)
 	}),
@@ -1440,24 +1438,26 @@ func TestAidlEnforcePermissionsException(t *testing.T) {
 }
 
 func TestDataNativeBinaries(t *testing.T) {
-	ctx, _ := testJava(t, `
+	ctx := android.GroupFixturePreparers(
+		prepareForJavaTest,
+		android.PrepareForTestWithAllowMissingDependencies).RunTestWithBp(t, `
 		java_test_host {
 			name: "foo",
 			srcs: ["a.java"],
 			data_native_bins: ["bin"]
 		}
 
-		python_binary_host {
+		cc_binary_host {
 			name: "bin",
-			srcs: ["bin.py"],
+			srcs: ["bin.cpp"],
 		}
-	`)
+	`).TestContext
 
 	buildOS := ctx.Config().BuildOS.String()
 
 	test := ctx.ModuleForTests("foo", buildOS+"_common").Module().(*TestHost)
 	entries := android.AndroidMkEntriesForTest(t, ctx, test)[0]
-	expected := []string{"out/soong/.intermediates/bin/" + buildOS + "_x86_64_PY3/bin:bin"}
+	expected := []string{"out/soong/.intermediates/bin/" + buildOS + "_x86_64/bin:bin"}
 	actual := entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"]
 	android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_COMPATIBILITY_SUPPORT_FILES", ctx.Config(), expected, actual)
 }

python/Android.bp

@@ -9,6 +9,7 @@ bootstrap_go_package {
         "blueprint",
         "soong-android",
         "soong-tradefed",
+        "soong-cc",
     ],
     srcs: [
         "binary.go",

python/binary.go

@@ -21,8 +21,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/google/blueprint"
-
 	"android/soong/android"
 )
 
@@ -109,14 +107,14 @@ func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleConte
 }
 
 func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) {
-	depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx)
+	embeddedLauncher := p.isEmbeddedLauncherEnabled()
+	depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx, embeddedLauncher)
 	main := ""
 	if p.autorun() {
 		main = p.getPyMainFile(ctx, p.srcsPathMappings)
 	}
 
 	var launcherPath android.OptionalPath
-	embeddedLauncher := p.isEmbeddedLauncherEnabled()
 	if embeddedLauncher {
 		ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) {
 			if provider, ok := m.(IntermPathProvider); ok {
@@ -128,9 +126,16 @@ func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) {
 			}
 		})
 	}
+	srcsZips := make(android.Paths, 0, len(depsSrcsZips)+1)
+	if embeddedLauncher {
+		srcsZips = append(srcsZips, p.precompiledSrcsZip)
+	} else {
+		srcsZips = append(srcsZips, p.srcsZip)
+	}
+	srcsZips = append(srcsZips, depsSrcsZips...)
 	p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath,
 		p.getHostInterpreterName(ctx, p.properties.Actual_version),
-		main, p.getStem(ctx), append(android.Paths{p.srcsZip}, depsSrcsZips...))
+		main, p.getStem(ctx), srcsZips)
 
 	var sharedLibs []string
 	// if embedded launcher is enabled, we need to collect the shared library dependencies of the
@@ -170,64 +175,8 @@ func (p *PythonBinaryModule) AndroidMkEntries() []android.AndroidMkEntries {
 func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
 	p.PythonLibraryModule.DepsMutator(ctx)
 
-	versionVariation := []blueprint.Variation{
-		{"python_version", p.properties.Actual_version},
-	}
-
-	// If this module will be installed and has an embedded launcher, we need to add dependencies for:
-	//   * standard library
-	//   * launcher
-	//   * shared dependencies of the launcher
 	if p.isEmbeddedLauncherEnabled() {
-		var stdLib string
+		p.AddDepsOnPythonLauncherAndStdlib(ctx, pythonLibTag, launcherTag, launcherSharedLibTag, p.autorun(), ctx.Target())
-		var launcherModule string
-		// Add launcher shared lib dependencies. Ideally, these should be
-		// derived from the `shared_libs` property of the launcher. However, we
-		// cannot read the property at this stage and it will be too late to add
-		// dependencies later.
-		launcherSharedLibDeps := []string{
-			"libsqlite",
-		}
-		// Add launcher-specific dependencies for bionic
-		if ctx.Target().Os.Bionic() {
-			launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
-		}
-		if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
-			launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
-		}
-
-		switch p.properties.Actual_version {
-		case pyVersion2:
-			stdLib = "py2-stdlib"
-
-			launcherModule = "py2-launcher"
-			if p.autorun() {
-				launcherModule = "py2-launcher-autorun"
-			}
-
-			launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
-
-		case pyVersion3:
-			stdLib = "py3-stdlib"
-
-			launcherModule = "py3-launcher"
-			if p.autorun() {
-				launcherModule = "py3-launcher-autorun"
-			}
-			if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
-				launcherModule += "-static"
-			}
-
-			if ctx.Device() {
-				launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
-			}
-		default:
-			panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
-				p.properties.Actual_version, ctx.ModuleName()))
-		}
-		ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
-		ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
-		ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
 	}
 }
 

python/builder.go

@@ -70,6 +70,17 @@ var (
 			CommandDeps: []string{"$mergeParCmd"},
 		},
 		"srcsZips", "launcher")
+
+	precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{
+		Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` +
+			`PYTHONPATH=$stdlibZip/internal/stdlib ` +
+			`$launcher build/soong/python/scripts/precompile_python.py $in $out`,
+		CommandDeps: []string{
+			"$stdlibZip",
+			"$launcher",
+			"build/soong/python/scripts/precompile_python.py",
+		},
+	}, "stdlibZip", "launcher", "ldLibraryPath")
 )
 
 func init() {

python/python.go

@@ -146,8 +146,12 @@ type PythonLibraryModule struct {
 	// pathMapping: <dest: runfile_path, src: source_path>
 	dataPathMappings []pathMapping
 
-	// the zip filepath for zipping current module source/data files.
+	// The zip file containing the current module's source/data files.
 	srcsZip android.Path
+
+	// The zip file containing the current module's source/data files, with the
+	// source files precompiled.
+	precompiledSrcsZip android.Path
 }
 
 // newModule generates new Python base module
@@ -164,6 +168,7 @@ type pythonDependency interface {
 	getSrcsPathMappings() []pathMapping
 	getDataPathMappings() []pathMapping
 	getSrcsZip() android.Path
+	getPrecompiledSrcsZip() android.Path
 }
 
 // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
@@ -181,6 +186,11 @@ func (p *PythonLibraryModule) getSrcsZip() android.Path {
 	return p.srcsZip
 }
 
+// getSrcsZip returns the filepath where the current module's source/data files are zipped.
+func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
+	return p.precompiledSrcsZip
+}
+
 func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
 	return &p.properties
 }
@@ -213,16 +223,23 @@ type installDependencyTag struct {
 }
 
 var (
-	pythonLibTag         = dependencyTag{name: "pythonLib"}
+	pythonLibTag = dependencyTag{name: "pythonLib"}
-	javaDataTag          = dependencyTag{name: "javaData"}
+	javaDataTag  = dependencyTag{name: "javaData"}
+	// The python interpreter, with soong module name "py3-launcher" or "py3-launcher-autorun".
 	launcherTag          = dependencyTag{name: "launcher"}
 	launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
-	pathComponentRegexp  = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
+	// The python interpreter built for host so that we can precompile python sources.
-	pyExt                = ".py"
+	// This only works because the precompiled sources don't vary by architecture.
-	protoExt             = ".proto"
+	// The soong module name is "py3-launcher".
-	pyVersion2           = "PY2"
+	hostLauncherTag          = dependencyTag{name: "hostLauncher"}
-	pyVersion3           = "PY3"
+	hostlauncherSharedLibTag = dependencyTag{name: "hostlauncherSharedLib"}
-	internalPath         = "internal"
+	hostStdLibTag            = dependencyTag{name: "hostStdLib"}
+	pathComponentRegexp      = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
+	pyExt                    = ".py"
+	protoExt                 = ".proto"
+	pyVersion2               = "PY2"
+	pyVersion3               = "PY3"
+	internalPath             = "internal"
 )
 
 type basePropertiesProvider interface {
@@ -305,6 +322,76 @@ func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
 	// so that it can point to java modules.
 	javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
 	ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
+
+	p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, hostlauncherSharedLibTag, false, ctx.Config().BuildOSTarget)
+}
+
+// AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib,
+// launcher (interpreter), and the launcher's shared libraries. If autorun is true, it will use
+// the autorun launcher instead of the regular one. This function acceps a targetForDeps argument
+// as the target to use for these dependencies. For embedded launcher python binaries, the launcher
+// that will be embedded will be under the same target as the python module itself. But when
+// precompiling python code, we need to get the python launcher built for host, even if we're
+// compiling the python module for device, so we pass a different target to this function.
+func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.BottomUpMutatorContext,
+	stdLibTag, launcherTag, launcherSharedLibTag blueprint.DependencyTag,
+	autorun bool, targetForDeps android.Target) {
+	var stdLib string
+	var launcherModule string
+	// Add launcher shared lib dependencies. Ideally, these should be
+	// derived from the `shared_libs` property of the launcher. TODO: read these from
+	// the python launcher itself using ctx.OtherModuleProvider() or similar on the result
+	// of ctx.AddFarVariationDependencies()
+	launcherSharedLibDeps := []string{
+		"libsqlite",
+	}
+	// Add launcher-specific dependencies for bionic
+	if targetForDeps.Os.Bionic() {
+		launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
+	}
+	if targetForDeps.Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
+		launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
+	}
+
+	switch p.properties.Actual_version {
+	case pyVersion2:
+		stdLib = "py2-stdlib"
+
+		launcherModule = "py2-launcher"
+		if autorun {
+			launcherModule = "py2-launcher-autorun"
+		}
+
+		launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
+	case pyVersion3:
+		stdLib = "py3-stdlib"
+
+		launcherModule = "py3-launcher"
+		if autorun {
+			launcherModule = "py3-launcher-autorun"
+		}
+		if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl {
+			launcherModule += "-static"
+		}
+		if ctx.Device() {
+			launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
+		}
+	default:
+		panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+			p.properties.Actual_version, ctx.ModuleName()))
+	}
+	targetVariations := targetForDeps.Variations()
+	if ctx.ModuleName() != stdLib {
+		stdLibVariations := make([]blueprint.Variation, 0, len(targetVariations)+1)
+		stdLibVariations = append(stdLibVariations, blueprint.Variation{Mutator: "python_version", Variation: p.properties.Actual_version})
+		stdLibVariations = append(stdLibVariations, targetVariations...)
+		// Using AddFarVariationDependencies for all of these because they can be for a different
+		// platform, like if the python module itself was being compiled for device, we may want
+		// the python interpreter built for host so that we can precompile python sources.
+		ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib)
+	}
+	ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule)
+	ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...)
 }
 
 // GenerateAndroidBuildActions performs build actions common to all Python modules
@@ -342,6 +429,7 @@ func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleCont
 
 	// generate the zipfile of all source and data files
 	p.srcsZip = p.createSrcsZip(ctx, pkgPath)
+	p.precompiledSrcsZip = p.precompileSrcs(ctx)
 }
 
 func isValidPythonPath(path string) error {
@@ -397,12 +485,8 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p
 // createSrcsZip registers build actions to zip current module's sources and data.
 func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
 	relativeRootMap := make(map[string]android.Paths)
-	pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
-
 	var protoSrcs android.Paths
-	// "srcs" or "data" properties may contain filegroup so it might happen that
+	addPathMapping := func(path pathMapping) {
-	// the root directory for each source path is different.
-	for _, path := range pathMappings {
 		// handle proto sources separately
 		if path.src.Ext() == protoExt {
 			protoSrcs = append(protoSrcs, path.src)
@@ -411,6 +495,16 @@ func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath s
 			relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src)
 		}
 	}
+
+	// "srcs" or "data" properties may contain filegroups so it might happen that
+	// the root directory for each source path is different.
+	for _, path := range p.srcsPathMappings {
+		addPathMapping(path)
+	}
+	for _, path := range p.dataPathMappings {
+		addPathMapping(path)
+	}
+
 	var zips android.Paths
 	if len(protoSrcs) > 0 {
 		protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
@@ -484,6 +578,65 @@ func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath s
 	}
 }
 
+func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.Path {
+	// To precompile the python sources, we need a python interpreter and stdlib built
+	// for host. We then use those to compile the python sources, which may be used on either
+	// host of device. Python bytecode is architecture agnostic, so we're essentially
+	// "cross compiling" for device here purely by virtue of host and device python bytecode
+	// being the same.
+	var stdLib android.Path
+	var launcher android.Path
+	if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
+		stdLib = p.srcsZip
+	} else {
+		ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
+			if dep, ok := module.(pythonDependency); ok {
+				stdLib = dep.getPrecompiledSrcsZip()
+			}
+		})
+	}
+	ctx.VisitDirectDepsWithTag(hostLauncherTag, func(module android.Module) {
+		if dep, ok := module.(IntermPathProvider); ok {
+			optionalLauncher := dep.IntermPathForModuleOut()
+			if optionalLauncher.Valid() {
+				launcher = optionalLauncher.Path()
+			}
+		}
+	})
+	var launcherSharedLibs android.Paths
+	var ldLibraryPath []string
+	ctx.VisitDirectDepsWithTag(hostlauncherSharedLibTag, func(module android.Module) {
+		if dep, ok := module.(IntermPathProvider); ok {
+			optionalPath := dep.IntermPathForModuleOut()
+			if optionalPath.Valid() {
+				launcherSharedLibs = append(launcherSharedLibs, optionalPath.Path())
+				ldLibraryPath = append(ldLibraryPath, filepath.Dir(optionalPath.Path().String()))
+			}
+		}
+	})
+
+	out := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszipprecompiled")
+	if stdLib == nil || launcher == nil {
+		// This shouldn't happen in a real build because we'll error out when adding dependencies
+		// on the stdlib and launcher if they don't exist. But some tests set
+		// AllowMissingDependencies.
+		return out
+	}
+	ctx.Build(pctx, android.BuildParams{
+		Rule:        precompile,
+		Input:       p.srcsZip,
+		Output:      out,
+		Implicits:   launcherSharedLibs,
+		Description: "Precompile the python sources of " + ctx.ModuleName(),
+		Args: map[string]string{
+			"stdlibZip":     stdLib.String(),
+			"launcher":      launcher.String(),
+			"ldLibraryPath": strings.Join(ldLibraryPath, ":"),
+		},
+	})
+	return out
+}
+
 // isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not
 func isPythonLibModule(module blueprint.Module) bool {
 	if _, ok := module.(*PythonLibraryModule); ok {
@@ -497,7 +650,7 @@ func isPythonLibModule(module blueprint.Module) bool {
 // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
 // for module and its transitive dependencies and collects list of data/source file
 // zips for transitive dependencies.
-func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext) android.Paths {
+func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext, precompiled bool) android.Paths {
 	// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
 	// check duplicates.
 	destToPySrcs := make(map[string]string)
@@ -541,7 +694,11 @@ func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleC
 				checkForDuplicateOutputPath(ctx, destToPyData,
 					path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
 			}
-			result = append(result, dep.getSrcsZip())
+			if precompiled {
+				result = append(result, dep.getPrecompiledSrcsZip())
+			} else {
+				result = append(result, dep.getSrcsZip())
+			}
 		}
 		return true
 	})

python/python_test.go

@@ -18,10 +18,10 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 	"testing"
 
 	"android/soong/android"
+	"android/soong/cc"
 )
 
 type pyModule struct {
@@ -33,8 +33,10 @@ type pyModule struct {
 }
 
 var (
-	buildNamePrefix          = "soong_python_test"
+	buildNamePrefix = "soong_python_test"
-	moduleVariantErrTemplate = "%s: module %q variant %q: "
+	// We allow maching almost anything before the actual variant so that the os/arch variant
+	// is matched.
+	moduleVariantErrTemplate = `%s: module %q variant "[a-zA-Z0-9_]*%s": `
 	pkgPathErrTemplate       = moduleVariantErrTemplate +
 		"pkg_path: %q must be a relative path contained in par file."
 	badIdentifierErrTemplate = moduleVariantErrTemplate +
@@ -323,17 +325,26 @@ func TestPythonModule(t *testing.T) {
 		if d.desc != "module with duplicate runfile path" {
 			continue
 		}
-		errorPatterns := make([]string, len(d.errors))
+		d.mockFiles[filepath.Join("common", bpFile)] = []byte(`
-		for i, s := range d.errors {
+python_library {
-			errorPatterns[i] = regexp.QuoteMeta(s)
+  name: "py3-stdlib",
-		}
+  host_supported: true,
+}
+cc_binary {
+  name: "py3-launcher",
+  host_supported: true,
+}
+`)
 
 		t.Run(d.desc, func(t *testing.T) {
 			result := android.GroupFixturePreparers(
 				android.PrepareForTestWithDefaults,
+				android.PrepareForTestWithArchMutator,
+				android.PrepareForTestWithAllowMissingDependencies,
+				cc.PrepareForTestWithCcDefaultModules,
 				PrepareForTestWithPythonBuildComponents,
 				d.mockFiles.AddToFixture(),
-			).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(errorPatterns)).
+			).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(d.errors)).
 				RunTest(t)
 
 			if len(result.Errs) > 0 {

python/scripts/precompile_python.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Copyright 2023 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import py_compile
+import os
+import shutil
+import tempfile
+import zipfile
+
+# This file needs to support both python 2 and 3.
+
+
+def process_one_file(name, inf, outzip):
+    if not name.endswith('.py'):
+        outzip.writestr(name, inf.read())
+        return
+
+    # Unfortunately py_compile requires the input/output files to be written
+    # out to disk.
+    with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp:
+        shutil.copyfileobj(inf, tmp)
+        in_name = tmp.name
+    with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp:
+        out_name = tmp.name
+    try:
+        py_compile.compile(in_name, out_name, name, doraise=True)
+        with open(out_name, 'rb') as f:
+            outzip.writestr(name + 'c', f.read())
+    finally:
+        os.remove(in_name)
+        os.remove(out_name)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('src_zip')
+    parser.add_argument('dst_zip')
+    args = parser.parse_args()
+
+    with open(args.dst_zip, 'wb') as outf, open(args.src_zip, 'rb') as inf:
+        with zipfile.ZipFile(outf, mode='w') as outzip, zipfile.ZipFile(inf, mode='r') as inzip:
+            for name in inzip.namelist():
+                with inzip.open(name, mode='r') as inzipf:
+                    process_one_file(name, inzipf, outzip)
+
+
+if __name__ == "__main__":
+    main()

python/tests/par_test.py

@@ -27,7 +27,10 @@ def assert_equal(what, a, b):
         failed = True
 
 assert_equal("__name__", __name__, "__main__")
-assert_equal("os.path.basename(__file__)", os.path.basename(__file__), "par_test.py")
+fileName = os.path.basename(__file__)
+if fileName.endswith('.pyc'):
+    fileName = fileName[:-1]
+assert_equal("os.path.basename(__file__)", fileName, "par_test.py")
 
 archive = os.path.dirname(__file__)
 

python/tests/testpkg/par_test.py

@@ -28,7 +28,10 @@ def assert_equal(what, a, b):
 archive = sys.modules["__main__"].__loader__.archive
 
 assert_equal("__name__", __name__, "testpkg.par_test")
-assert_equal("__file__", __file__, os.path.join(archive, "testpkg/par_test.py"))
+fileName = __file__
+if fileName.endswith('.pyc'):
+    fileName = fileName[:-1]
+assert_equal("__file__", fileName, os.path.join(archive, "testpkg/par_test.py"))
 
 # Python3 is returning None here for me, and I haven't found any problems caused by this.
 if sys.version_info[0] == 2:

tests/lib.sh

@@ -94,6 +94,8 @@ function create_mock_soong {
   symlink_directory external/go-cmp
   symlink_directory external/golang-protobuf
   symlink_directory external/starlark-go
+  symlink_directory external/python
+  symlink_directory external/sqlite
 
   touch "$MOCK_TOP/Android.bp"
 }