Precompile python sources
This signifigantly improves the startup time of soong-built python binaries. For example, running `m apexer && time out/host/linux-x86/bin/apexer` gives 0.734s before this cl, and 0.094s after. Fixes: 259718110 Test: Presubmits Change-Id: Ib19e83e2c60c39a849525be117279c318de3afa7
This commit is contained in:
Cole Faust
parent
3420d978cf
commit
5c503d1c43
11 changed files with 295 additions and 96 deletions
|
|
@ -359,7 +359,8 @@ func (oz *OutputZip) getUninitializedPythonPackages(inputZips []InputZip) ([]str
|
|||
}
|
||||
for _, file := range inputZip.Entries() {
|
||||
pyPkg := getPackage(file.Name)
|
||||
if filepath.Base(file.Name) == "__init__.py" {
|
||||
baseName := filepath.Base(file.Name)
|
||||
if baseName == "__init__.py" || baseName == "__init__.pyc" {
|
||||
if _, found := initedPackages[pyPkg]; found {
|
||||
panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q", file.Name))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import (
|
|||
"android/soong/cc"
|
||||
"android/soong/dexpreopt"
|
||||
"android/soong/genrule"
|
||||
"android/soong/python"
|
||||
)
|
||||
|
||||
// Legacy preparer used for running tests within the java package.
|
||||
|
|
@ -49,7 +48,6 @@ var prepareForJavaTest = android.GroupFixturePreparers(
|
|||
// Include all the default java modules.
|
||||
PrepareForTestWithJavaDefaultModules,
|
||||
PrepareForTestWithOverlayBuildComponents,
|
||||
python.PrepareForTestWithPythonBuildComponents,
|
||||
android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
|
||||
ctx.RegisterPreSingletonType("sdk_versions", sdkPreSingletonFactory)
|
||||
}),
|
||||
|
|
@ -1440,24 +1438,26 @@ func TestAidlEnforcePermissionsException(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDataNativeBinaries(t *testing.T) {
|
||||
ctx, _ := testJava(t, `
|
||||
ctx := android.GroupFixturePreparers(
|
||||
prepareForJavaTest,
|
||||
android.PrepareForTestWithAllowMissingDependencies).RunTestWithBp(t, `
|
||||
java_test_host {
|
||||
name: "foo",
|
||||
srcs: ["a.java"],
|
||||
data_native_bins: ["bin"]
|
||||
}
|
||||
|
||||
python_binary_host {
|
||||
cc_binary_host {
|
||||
name: "bin",
|
||||
srcs: ["bin.py"],
|
||||
srcs: ["bin.cpp"],
|
||||
}
|
||||
`)
|
||||
`).TestContext
|
||||
|
||||
buildOS := ctx.Config().BuildOS.String()
|
||||
|
||||
test := ctx.ModuleForTests("foo", buildOS+"_common").Module().(*TestHost)
|
||||
entries := android.AndroidMkEntriesForTest(t, ctx, test)[0]
|
||||
expected := []string{"out/soong/.intermediates/bin/" + buildOS + "_x86_64_PY3/bin:bin"}
|
||||
expected := []string{"out/soong/.intermediates/bin/" + buildOS + "_x86_64/bin:bin"}
|
||||
actual := entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"]
|
||||
android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_COMPATIBILITY_SUPPORT_FILES", ctx.Config(), expected, actual)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ bootstrap_go_package {
|
|||
"blueprint",
|
||||
"soong-android",
|
||||
"soong-tradefed",
|
||||
"soong-cc",
|
||||
],
|
||||
srcs: [
|
||||
"binary.go",
|
||||
|
|
|
|||
|
|
@ -21,8 +21,6 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
|
||||
"android/soong/android"
|
||||
)
|
||||
|
||||
|
|
@ -109,14 +107,14 @@ func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleConte
|
|||
}
|
||||
|
||||
func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) {
|
||||
depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx)
|
||||
embeddedLauncher := p.isEmbeddedLauncherEnabled()
|
||||
depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx, embeddedLauncher)
|
||||
main := ""
|
||||
if p.autorun() {
|
||||
main = p.getPyMainFile(ctx, p.srcsPathMappings)
|
||||
}
|
||||
|
||||
var launcherPath android.OptionalPath
|
||||
embeddedLauncher := p.isEmbeddedLauncherEnabled()
|
||||
if embeddedLauncher {
|
||||
ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) {
|
||||
if provider, ok := m.(IntermPathProvider); ok {
|
||||
|
|
@ -128,9 +126,16 @@ func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) {
|
|||
}
|
||||
})
|
||||
}
|
||||
srcsZips := make(android.Paths, 0, len(depsSrcsZips)+1)
|
||||
if embeddedLauncher {
|
||||
srcsZips = append(srcsZips, p.precompiledSrcsZip)
|
||||
} else {
|
||||
srcsZips = append(srcsZips, p.srcsZip)
|
||||
}
|
||||
srcsZips = append(srcsZips, depsSrcsZips...)
|
||||
p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath,
|
||||
p.getHostInterpreterName(ctx, p.properties.Actual_version),
|
||||
main, p.getStem(ctx), append(android.Paths{p.srcsZip}, depsSrcsZips...))
|
||||
main, p.getStem(ctx), srcsZips)
|
||||
|
||||
var sharedLibs []string
|
||||
// if embedded launcher is enabled, we need to collect the shared library dependencies of the
|
||||
|
|
@ -170,64 +175,8 @@ func (p *PythonBinaryModule) AndroidMkEntries() []android.AndroidMkEntries {
|
|||
func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
|
||||
p.PythonLibraryModule.DepsMutator(ctx)
|
||||
|
||||
versionVariation := []blueprint.Variation{
|
||||
{"python_version", p.properties.Actual_version},
|
||||
}
|
||||
|
||||
// If this module will be installed and has an embedded launcher, we need to add dependencies for:
|
||||
// * standard library
|
||||
// * launcher
|
||||
// * shared dependencies of the launcher
|
||||
if p.isEmbeddedLauncherEnabled() {
|
||||
var stdLib string
|
||||
var launcherModule string
|
||||
// Add launcher shared lib dependencies. Ideally, these should be
|
||||
// derived from the `shared_libs` property of the launcher. However, we
|
||||
// cannot read the property at this stage and it will be too late to add
|
||||
// dependencies later.
|
||||
launcherSharedLibDeps := []string{
|
||||
"libsqlite",
|
||||
}
|
||||
// Add launcher-specific dependencies for bionic
|
||||
if ctx.Target().Os.Bionic() {
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
|
||||
}
|
||||
if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
|
||||
}
|
||||
|
||||
switch p.properties.Actual_version {
|
||||
case pyVersion2:
|
||||
stdLib = "py2-stdlib"
|
||||
|
||||
launcherModule = "py2-launcher"
|
||||
if p.autorun() {
|
||||
launcherModule = "py2-launcher-autorun"
|
||||
}
|
||||
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
|
||||
|
||||
case pyVersion3:
|
||||
stdLib = "py3-stdlib"
|
||||
|
||||
launcherModule = "py3-launcher"
|
||||
if p.autorun() {
|
||||
launcherModule = "py3-launcher-autorun"
|
||||
}
|
||||
if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
|
||||
launcherModule += "-static"
|
||||
}
|
||||
|
||||
if ctx.Device() {
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
|
||||
}
|
||||
default:
|
||||
panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
|
||||
p.properties.Actual_version, ctx.ModuleName()))
|
||||
}
|
||||
ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
|
||||
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
|
||||
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
|
||||
p.AddDepsOnPythonLauncherAndStdlib(ctx, pythonLibTag, launcherTag, launcherSharedLibTag, p.autorun(), ctx.Target())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -70,6 +70,17 @@ var (
|
|||
CommandDeps: []string{"$mergeParCmd"},
|
||||
},
|
||||
"srcsZips", "launcher")
|
||||
|
||||
precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{
|
||||
Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` +
|
||||
`PYTHONPATH=$stdlibZip/internal/stdlib ` +
|
||||
`$launcher build/soong/python/scripts/precompile_python.py $in $out`,
|
||||
CommandDeps: []string{
|
||||
"$stdlibZip",
|
||||
"$launcher",
|
||||
"build/soong/python/scripts/precompile_python.py",
|
||||
},
|
||||
}, "stdlibZip", "launcher", "ldLibraryPath")
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
189
python/python.go
189
python/python.go
|
|
@ -146,8 +146,12 @@ type PythonLibraryModule struct {
|
|||
// pathMapping: <dest: runfile_path, src: source_path>
|
||||
dataPathMappings []pathMapping
|
||||
|
||||
// the zip filepath for zipping current module source/data files.
|
||||
// The zip file containing the current module's source/data files.
|
||||
srcsZip android.Path
|
||||
|
||||
// The zip file containing the current module's source/data files, with the
|
||||
// source files precompiled.
|
||||
precompiledSrcsZip android.Path
|
||||
}
|
||||
|
||||
// newModule generates new Python base module
|
||||
|
|
@ -164,6 +168,7 @@ type pythonDependency interface {
|
|||
getSrcsPathMappings() []pathMapping
|
||||
getDataPathMappings() []pathMapping
|
||||
getSrcsZip() android.Path
|
||||
getPrecompiledSrcsZip() android.Path
|
||||
}
|
||||
|
||||
// getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
|
||||
|
|
@ -181,6 +186,11 @@ func (p *PythonLibraryModule) getSrcsZip() android.Path {
|
|||
return p.srcsZip
|
||||
}
|
||||
|
||||
// getSrcsZip returns the filepath where the current module's source/data files are zipped.
|
||||
func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
|
||||
return p.precompiledSrcsZip
|
||||
}
|
||||
|
||||
func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
|
||||
return &p.properties
|
||||
}
|
||||
|
|
@ -213,16 +223,23 @@ type installDependencyTag struct {
|
|||
}
|
||||
|
||||
var (
|
||||
pythonLibTag = dependencyTag{name: "pythonLib"}
|
||||
javaDataTag = dependencyTag{name: "javaData"}
|
||||
pythonLibTag = dependencyTag{name: "pythonLib"}
|
||||
javaDataTag = dependencyTag{name: "javaData"}
|
||||
// The python interpreter, with soong module name "py3-launcher" or "py3-launcher-autorun".
|
||||
launcherTag = dependencyTag{name: "launcher"}
|
||||
launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
|
||||
pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
|
||||
pyExt = ".py"
|
||||
protoExt = ".proto"
|
||||
pyVersion2 = "PY2"
|
||||
pyVersion3 = "PY3"
|
||||
internalPath = "internal"
|
||||
// The python interpreter built for host so that we can precompile python sources.
|
||||
// This only works because the precompiled sources don't vary by architecture.
|
||||
// The soong module name is "py3-launcher".
|
||||
hostLauncherTag = dependencyTag{name: "hostLauncher"}
|
||||
hostlauncherSharedLibTag = dependencyTag{name: "hostlauncherSharedLib"}
|
||||
hostStdLibTag = dependencyTag{name: "hostStdLib"}
|
||||
pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
|
||||
pyExt = ".py"
|
||||
protoExt = ".proto"
|
||||
pyVersion2 = "PY2"
|
||||
pyVersion3 = "PY3"
|
||||
internalPath = "internal"
|
||||
)
|
||||
|
||||
type basePropertiesProvider interface {
|
||||
|
|
@ -305,6 +322,76 @@ func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
|
|||
// so that it can point to java modules.
|
||||
javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
|
||||
ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
|
||||
|
||||
p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, hostlauncherSharedLibTag, false, ctx.Config().BuildOSTarget)
|
||||
}
|
||||
|
||||
// AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib,
|
||||
// launcher (interpreter), and the launcher's shared libraries. If autorun is true, it will use
|
||||
// the autorun launcher instead of the regular one. This function acceps a targetForDeps argument
|
||||
// as the target to use for these dependencies. For embedded launcher python binaries, the launcher
|
||||
// that will be embedded will be under the same target as the python module itself. But when
|
||||
// precompiling python code, we need to get the python launcher built for host, even if we're
|
||||
// compiling the python module for device, so we pass a different target to this function.
|
||||
func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.BottomUpMutatorContext,
|
||||
stdLibTag, launcherTag, launcherSharedLibTag blueprint.DependencyTag,
|
||||
autorun bool, targetForDeps android.Target) {
|
||||
var stdLib string
|
||||
var launcherModule string
|
||||
// Add launcher shared lib dependencies. Ideally, these should be
|
||||
// derived from the `shared_libs` property of the launcher. TODO: read these from
|
||||
// the python launcher itself using ctx.OtherModuleProvider() or similar on the result
|
||||
// of ctx.AddFarVariationDependencies()
|
||||
launcherSharedLibDeps := []string{
|
||||
"libsqlite",
|
||||
}
|
||||
// Add launcher-specific dependencies for bionic
|
||||
if targetForDeps.Os.Bionic() {
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
|
||||
}
|
||||
if targetForDeps.Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
|
||||
}
|
||||
|
||||
switch p.properties.Actual_version {
|
||||
case pyVersion2:
|
||||
stdLib = "py2-stdlib"
|
||||
|
||||
launcherModule = "py2-launcher"
|
||||
if autorun {
|
||||
launcherModule = "py2-launcher-autorun"
|
||||
}
|
||||
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
|
||||
case pyVersion3:
|
||||
stdLib = "py3-stdlib"
|
||||
|
||||
launcherModule = "py3-launcher"
|
||||
if autorun {
|
||||
launcherModule = "py3-launcher-autorun"
|
||||
}
|
||||
if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl {
|
||||
launcherModule += "-static"
|
||||
}
|
||||
if ctx.Device() {
|
||||
launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
|
||||
}
|
||||
default:
|
||||
panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
|
||||
p.properties.Actual_version, ctx.ModuleName()))
|
||||
}
|
||||
targetVariations := targetForDeps.Variations()
|
||||
if ctx.ModuleName() != stdLib {
|
||||
stdLibVariations := make([]blueprint.Variation, 0, len(targetVariations)+1)
|
||||
stdLibVariations = append(stdLibVariations, blueprint.Variation{Mutator: "python_version", Variation: p.properties.Actual_version})
|
||||
stdLibVariations = append(stdLibVariations, targetVariations...)
|
||||
// Using AddFarVariationDependencies for all of these because they can be for a different
|
||||
// platform, like if the python module itself was being compiled for device, we may want
|
||||
// the python interpreter built for host so that we can precompile python sources.
|
||||
ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib)
|
||||
}
|
||||
ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule)
|
||||
ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...)
|
||||
}
|
||||
|
||||
// GenerateAndroidBuildActions performs build actions common to all Python modules
|
||||
|
|
@ -342,6 +429,7 @@ func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleCont
|
|||
|
||||
// generate the zipfile of all source and data files
|
||||
p.srcsZip = p.createSrcsZip(ctx, pkgPath)
|
||||
p.precompiledSrcsZip = p.precompileSrcs(ctx)
|
||||
}
|
||||
|
||||
func isValidPythonPath(path string) error {
|
||||
|
|
@ -397,12 +485,8 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p
|
|||
// createSrcsZip registers build actions to zip current module's sources and data.
|
||||
func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
|
||||
relativeRootMap := make(map[string]android.Paths)
|
||||
pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
|
||||
|
||||
var protoSrcs android.Paths
|
||||
// "srcs" or "data" properties may contain filegroup so it might happen that
|
||||
// the root directory for each source path is different.
|
||||
for _, path := range pathMappings {
|
||||
addPathMapping := func(path pathMapping) {
|
||||
// handle proto sources separately
|
||||
if path.src.Ext() == protoExt {
|
||||
protoSrcs = append(protoSrcs, path.src)
|
||||
|
|
@ -411,6 +495,16 @@ func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath s
|
|||
relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src)
|
||||
}
|
||||
}
|
||||
|
||||
// "srcs" or "data" properties may contain filegroups so it might happen that
|
||||
// the root directory for each source path is different.
|
||||
for _, path := range p.srcsPathMappings {
|
||||
addPathMapping(path)
|
||||
}
|
||||
for _, path := range p.dataPathMappings {
|
||||
addPathMapping(path)
|
||||
}
|
||||
|
||||
var zips android.Paths
|
||||
if len(protoSrcs) > 0 {
|
||||
protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
|
||||
|
|
@ -484,6 +578,65 @@ func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath s
|
|||
}
|
||||
}
|
||||
|
||||
func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.Path {
|
||||
// To precompile the python sources, we need a python interpreter and stdlib built
|
||||
// for host. We then use those to compile the python sources, which may be used on either
|
||||
// host of device. Python bytecode is architecture agnostic, so we're essentially
|
||||
// "cross compiling" for device here purely by virtue of host and device python bytecode
|
||||
// being the same.
|
||||
var stdLib android.Path
|
||||
var launcher android.Path
|
||||
if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
|
||||
stdLib = p.srcsZip
|
||||
} else {
|
||||
ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
|
||||
if dep, ok := module.(pythonDependency); ok {
|
||||
stdLib = dep.getPrecompiledSrcsZip()
|
||||
}
|
||||
})
|
||||
}
|
||||
ctx.VisitDirectDepsWithTag(hostLauncherTag, func(module android.Module) {
|
||||
if dep, ok := module.(IntermPathProvider); ok {
|
||||
optionalLauncher := dep.IntermPathForModuleOut()
|
||||
if optionalLauncher.Valid() {
|
||||
launcher = optionalLauncher.Path()
|
||||
}
|
||||
}
|
||||
})
|
||||
var launcherSharedLibs android.Paths
|
||||
var ldLibraryPath []string
|
||||
ctx.VisitDirectDepsWithTag(hostlauncherSharedLibTag, func(module android.Module) {
|
||||
if dep, ok := module.(IntermPathProvider); ok {
|
||||
optionalPath := dep.IntermPathForModuleOut()
|
||||
if optionalPath.Valid() {
|
||||
launcherSharedLibs = append(launcherSharedLibs, optionalPath.Path())
|
||||
ldLibraryPath = append(ldLibraryPath, filepath.Dir(optionalPath.Path().String()))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
out := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszipprecompiled")
|
||||
if stdLib == nil || launcher == nil {
|
||||
// This shouldn't happen in a real build because we'll error out when adding dependencies
|
||||
// on the stdlib and launcher if they don't exist. But some tests set
|
||||
// AllowMissingDependencies.
|
||||
return out
|
||||
}
|
||||
ctx.Build(pctx, android.BuildParams{
|
||||
Rule: precompile,
|
||||
Input: p.srcsZip,
|
||||
Output: out,
|
||||
Implicits: launcherSharedLibs,
|
||||
Description: "Precompile the python sources of " + ctx.ModuleName(),
|
||||
Args: map[string]string{
|
||||
"stdlibZip": stdLib.String(),
|
||||
"launcher": launcher.String(),
|
||||
"ldLibraryPath": strings.Join(ldLibraryPath, ":"),
|
||||
},
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not
|
||||
func isPythonLibModule(module blueprint.Module) bool {
|
||||
if _, ok := module.(*PythonLibraryModule); ok {
|
||||
|
|
@ -497,7 +650,7 @@ func isPythonLibModule(module blueprint.Module) bool {
|
|||
// collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
|
||||
// for module and its transitive dependencies and collects list of data/source file
|
||||
// zips for transitive dependencies.
|
||||
func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext) android.Paths {
|
||||
func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext, precompiled bool) android.Paths {
|
||||
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
|
||||
// check duplicates.
|
||||
destToPySrcs := make(map[string]string)
|
||||
|
|
@ -541,7 +694,11 @@ func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleC
|
|||
checkForDuplicateOutputPath(ctx, destToPyData,
|
||||
path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
|
||||
}
|
||||
result = append(result, dep.getSrcsZip())
|
||||
if precompiled {
|
||||
result = append(result, dep.getPrecompiledSrcsZip())
|
||||
} else {
|
||||
result = append(result, dep.getSrcsZip())
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"android/soong/android"
|
||||
"android/soong/cc"
|
||||
)
|
||||
|
||||
type pyModule struct {
|
||||
|
|
@ -33,8 +33,10 @@ type pyModule struct {
|
|||
}
|
||||
|
||||
var (
|
||||
buildNamePrefix = "soong_python_test"
|
||||
moduleVariantErrTemplate = "%s: module %q variant %q: "
|
||||
buildNamePrefix = "soong_python_test"
|
||||
// We allow maching almost anything before the actual variant so that the os/arch variant
|
||||
// is matched.
|
||||
moduleVariantErrTemplate = `%s: module %q variant "[a-zA-Z0-9_]*%s": `
|
||||
pkgPathErrTemplate = moduleVariantErrTemplate +
|
||||
"pkg_path: %q must be a relative path contained in par file."
|
||||
badIdentifierErrTemplate = moduleVariantErrTemplate +
|
||||
|
|
@ -323,17 +325,26 @@ func TestPythonModule(t *testing.T) {
|
|||
if d.desc != "module with duplicate runfile path" {
|
||||
continue
|
||||
}
|
||||
errorPatterns := make([]string, len(d.errors))
|
||||
for i, s := range d.errors {
|
||||
errorPatterns[i] = regexp.QuoteMeta(s)
|
||||
}
|
||||
d.mockFiles[filepath.Join("common", bpFile)] = []byte(`
|
||||
python_library {
|
||||
name: "py3-stdlib",
|
||||
host_supported: true,
|
||||
}
|
||||
cc_binary {
|
||||
name: "py3-launcher",
|
||||
host_supported: true,
|
||||
}
|
||||
`)
|
||||
|
||||
t.Run(d.desc, func(t *testing.T) {
|
||||
result := android.GroupFixturePreparers(
|
||||
android.PrepareForTestWithDefaults,
|
||||
android.PrepareForTestWithArchMutator,
|
||||
android.PrepareForTestWithAllowMissingDependencies,
|
||||
cc.PrepareForTestWithCcDefaultModules,
|
||||
PrepareForTestWithPythonBuildComponents,
|
||||
d.mockFiles.AddToFixture(),
|
||||
).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(errorPatterns)).
|
||||
).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(d.errors)).
|
||||
RunTest(t)
|
||||
|
||||
if len(result.Errs) > 0 {
|
||||
|
|
|
|||
61
python/scripts/precompile_python.py
Normal file
61
python/scripts/precompile_python.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import py_compile
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
# This file needs to support both python 2 and 3.
|
||||
|
||||
|
||||
def process_one_file(name, inf, outzip):
|
||||
if not name.endswith('.py'):
|
||||
outzip.writestr(name, inf.read())
|
||||
return
|
||||
|
||||
# Unfortunately py_compile requires the input/output files to be written
|
||||
# out to disk.
|
||||
with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp:
|
||||
shutil.copyfileobj(inf, tmp)
|
||||
in_name = tmp.name
|
||||
with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp:
|
||||
out_name = tmp.name
|
||||
try:
|
||||
py_compile.compile(in_name, out_name, name, doraise=True)
|
||||
with open(out_name, 'rb') as f:
|
||||
outzip.writestr(name + 'c', f.read())
|
||||
finally:
|
||||
os.remove(in_name)
|
||||
os.remove(out_name)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('src_zip')
|
||||
parser.add_argument('dst_zip')
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.dst_zip, 'wb') as outf, open(args.src_zip, 'rb') as inf:
|
||||
with zipfile.ZipFile(outf, mode='w') as outzip, zipfile.ZipFile(inf, mode='r') as inzip:
|
||||
for name in inzip.namelist():
|
||||
with inzip.open(name, mode='r') as inzipf:
|
||||
process_one_file(name, inzipf, outzip)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -27,7 +27,10 @@ def assert_equal(what, a, b):
|
|||
failed = True
|
||||
|
||||
assert_equal("__name__", __name__, "__main__")
|
||||
assert_equal("os.path.basename(__file__)", os.path.basename(__file__), "par_test.py")
|
||||
fileName = os.path.basename(__file__)
|
||||
if fileName.endswith('.pyc'):
|
||||
fileName = fileName[:-1]
|
||||
assert_equal("os.path.basename(__file__)", fileName, "par_test.py")
|
||||
|
||||
archive = os.path.dirname(__file__)
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ def assert_equal(what, a, b):
|
|||
archive = sys.modules["__main__"].__loader__.archive
|
||||
|
||||
assert_equal("__name__", __name__, "testpkg.par_test")
|
||||
assert_equal("__file__", __file__, os.path.join(archive, "testpkg/par_test.py"))
|
||||
fileName = __file__
|
||||
if fileName.endswith('.pyc'):
|
||||
fileName = fileName[:-1]
|
||||
assert_equal("__file__", fileName, os.path.join(archive, "testpkg/par_test.py"))
|
||||
|
||||
# Python3 is returning None here for me, and I haven't found any problems caused by this.
|
||||
if sys.version_info[0] == 2:
|
||||
|
|
|
|||
|
|
@ -94,6 +94,8 @@ function create_mock_soong {
|
|||
symlink_directory external/go-cmp
|
||||
symlink_directory external/golang-protobuf
|
||||
symlink_directory external/starlark-go
|
||||
symlink_directory external/python
|
||||
symlink_directory external/sqlite
|
||||
|
||||
touch "$MOCK_TOP/Android.bp"
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue