tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor python rules

Browse source 
The goal of this cl is to simplify the python rules,
mostly by removing the "decorator" pattern that they
currently use, and instead making separate module
types for libraries, binaries, and tests that inherit
from each other.

Bug: 259718110
Test: Verified ninja files are unchanged (they only change in the list of soong sources because I added/deleted files)
Change-Id: I1e836e2cc4782c7818f91db7df7895de3b8db7ca
This commit is contained in:
Cole Faust 2023-01-23 10:14:58 -08:00
parent 6cf5e0d9cb
commit 4d247e6f21
10 changed files with 528 additions and 691 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
apex/apex.go
View file

@ -1657,7 +1657,7 @@ func apexFileForRustLibrary(ctx android.BaseModuleContext, rustm *rust.Module) a
return newApexFile(ctx, fileToCopy, androidMkModuleName, dirInApex, nativeSharedLib, rustm) return newApexFile(ctx, fileToCopy, androidMkModuleName, dirInApex, nativeSharedLib, rustm)
} }
func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.Module) apexFile { func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.PythonBinaryModule) apexFile {
dirInApex := "bin" dirInApex := "bin"
fileToCopy := py.HostToolPath().Path() fileToCopy := py.HostToolPath().Path()
return newApexFile(ctx, fileToCopy, py.BaseModuleName(), dirInApex, pyBinary, py) return newApexFile(ctx, fileToCopy, py.BaseModuleName(), dirInApex, pyBinary, py)
@ -2147,7 +2147,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext,
case *cc.Module: case *cc.Module:
vctx.filesInfo = append(vctx.filesInfo, apexFileForExecutable(ctx, ch)) vctx.filesInfo = append(vctx.filesInfo, apexFileForExecutable(ctx, ch))
return true // track transitive dependencies return true // track transitive dependencies
case *python.Module: case *python.PythonBinaryModule:
if ch.HostToolPath().Valid() { if ch.HostToolPath().Valid() {
vctx.filesInfo = append(vctx.filesInfo, apexFileForPyBinary(ctx, ch)) vctx.filesInfo = append(vctx.filesInfo, apexFileForPyBinary(ctx, ch))
} }

3
python/Android.bp
View file

@ -11,11 +11,10 @@ bootstrap_go_package {
"soong-tradefed", "soong-tradefed",
], ],
srcs: [ srcs: [
"androidmk.go",
"binary.go", "binary.go",
"bp2build.go",
"builder.go", "builder.go",
"defaults.go", "defaults.go",
"installer.go",
"library.go", "library.go",
"proto.go", "proto.go",
"python.go", "python.go",

90
python/androidmk.go
View file

@ -1,90 +0,0 @@
// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"path/filepath"
"strings"
"android/soong/android"
)
type subAndroidMkProvider interface {
AndroidMk(*Module, *android.AndroidMkEntries)
}
func (p *Module) subAndroidMk(entries *android.AndroidMkEntries, obj interface{}) {
if p.subAndroidMkOnce == nil {
p.subAndroidMkOnce = make(map[subAndroidMkProvider]bool)
}
if androidmk, ok := obj.(subAndroidMkProvider); ok {
if !p.subAndroidMkOnce[androidmk] {
p.subAndroidMkOnce[androidmk] = true
androidmk.AndroidMk(p, entries)
}
}
}
func (p *Module) AndroidMkEntries() []android.AndroidMkEntries {
entries := android.AndroidMkEntries{OutputFile: p.installSource}
p.subAndroidMk(&entries, p.installer)
return []android.AndroidMkEntries{entries}
}
func (p *binaryDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) {
entries.Class = "EXECUTABLES"
entries.ExtraEntries = append(entries.ExtraEntries,
func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...)
})
base.subAndroidMk(entries, p.pythonInstaller)
}
func (p *testDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) {
entries.Class = "NATIVE_TESTS"
entries.ExtraEntries = append(entries.ExtraEntries,
func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
entries.AddCompatibilityTestSuites(p.binaryDecorator.binaryProperties.Test_suites...)
if p.testConfig != nil {
entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String())
}
entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true))
entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...)
p.testProperties.Test_options.SetAndroidMkEntries(entries)
})
base.subAndroidMk(entries, p.binaryDecorator.pythonInstaller)
}
func (installer *pythonInstaller) AndroidMk(base *Module, entries *android.AndroidMkEntries) {
entries.Required = append(entries.Required, "libc++")
entries.ExtraEntries = append(entries.ExtraEntries,
func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
path, file := filepath.Split(installer.path.String())
stem := strings.TrimSuffix(file, filepath.Ext(file))
entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file))
entries.SetString("LOCAL_MODULE_PATH", path)
entries.SetString("LOCAL_MODULE_STEM", stem)
entries.AddStrings("LOCAL_SHARED_LIBRARIES", installer.androidMkSharedLibs...)
entries.SetBool("LOCAL_CHECK_ELF_FILES", false)
})
}

279
python/binary.go
View file

@ -18,11 +18,12 @@ package python
import ( import (
"fmt" "fmt"
"path/filepath"
"strings"
"github.com/google/blueprint"
"android/soong/android" "android/soong/android"
"android/soong/bazel"
"github.com/google/blueprint/proptools"
) )
func init() { func init() {
@ -33,63 +34,6 @@ func registerPythonBinaryComponents(ctx android.RegistrationContext) {
ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory)
} }
type bazelPythonBinaryAttributes struct {
Main *bazel.Label
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
Python_version *string
Imports bazel.StringListAttribute
}
func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *Module) {
// TODO(b/182306917): this doesn't fully handle all nested props versioned
// by the python version, which would have been handled by the version split
// mutator. This is sufficient for very simple python_binary_host modules
// under Bionic.
py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false)
py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false)
var python_version *string
if py3Enabled && py2Enabled {
panic(fmt.Errorf(
"error for '%s' module: bp2build's python_binary_host converter does not support "+
"converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name()))
} else if py2Enabled {
python_version = &pyVersion2
} else {
// do nothing, since python_version defaults to PY3.
}
baseAttrs := m.makeArchVariantBaseAttributes(ctx)
attrs := &bazelPythonBinaryAttributes{
Main: nil,
Srcs: baseAttrs.Srcs,
Deps: baseAttrs.Deps,
Python_version: python_version,
Imports: baseAttrs.Imports,
}
for _, propIntf := range m.GetProperties() {
if props, ok := propIntf.(*BinaryProperties); ok {
// main is optional.
if props.Main != nil {
main := android.BazelLabelForModuleSrcSingle(ctx, *props.Main)
attrs.Main = &main
break
}
}
}
props := bazel.BazelTargetModuleProperties{
// Use the native py_binary rule.
Rule_class: "py_binary",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{
Name: m.Name(),
Data: baseAttrs.Data,
}, attrs)
}
type BinaryProperties struct { type BinaryProperties struct {
// the name of the source file that is the main entry point of the program. // the name of the source file that is the main entry point of the program.
// this file must also be listed in srcs. // this file must also be listed in srcs.
@ -118,52 +62,61 @@ type BinaryProperties struct {
Auto_gen_config *bool Auto_gen_config *bool
} }
type binaryDecorator struct { type PythonBinaryModule struct {
PythonLibraryModule
binaryProperties BinaryProperties binaryProperties BinaryProperties
*pythonInstaller // (.intermediate) module output path as installation source.
installSource android.Path
// Final installation path.
installedDest android.Path
androidMkSharedLibs []string
} }
var _ android.AndroidMkEntriesProvider = (*PythonBinaryModule)(nil)
var _ android.Module = (*PythonBinaryModule)(nil)
type IntermPathProvider interface { type IntermPathProvider interface {
IntermPathForModuleOut() android.OptionalPath IntermPathForModuleOut() android.OptionalPath
} }
func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { func NewBinary(hod android.HostOrDeviceSupported) *PythonBinaryModule {
module := newModule(hod, android.MultilibFirst) return &PythonBinaryModule{
decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "")} PythonLibraryModule: *newModule(hod, android.MultilibFirst),
}
module.bootstrapper = decorator
module.installer = decorator
return module, decorator
} }
func PythonBinaryHostFactory() android.Module { func PythonBinaryHostFactory() android.Module {
module, _ := NewBinary(android.HostSupported) return NewBinary(android.HostSupported).init()
android.InitBazelModule(module)
return module.init()
} }
func (binary *binaryDecorator) autorun() bool { func (p *PythonBinaryModule) init() android.Module {
return BoolDefault(binary.binaryProperties.Autorun, true) p.AddProperties(&p.properties, &p.protoProperties)
p.AddProperties(&p.binaryProperties)
android.InitAndroidArchModule(p, p.hod, p.multilib)
android.InitDefaultableModule(p)
android.InitBazelModule(p)
return p
} }
func (binary *binaryDecorator) bootstrapperProps() []interface{} { func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
return []interface{}{&binary.binaryProperties} p.PythonLibraryModule.GenerateAndroidBuildActions(ctx)
p.buildBinary(ctx)
p.installedDest = ctx.InstallFile(installDir(ctx, "bin", "", ""),
p.installSource.Base(), p.installSource)
} }
func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersion string, func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) {
embeddedLauncher bool, srcsPathMappings []pathMapping, srcsZip android.Path, depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx)
depsSrcsZips android.Paths) android.OptionalPath {
main := "" main := ""
if binary.autorun() { if p.autorun() {
main = binary.getPyMainFile(ctx, srcsPathMappings) main = p.getPyMainFile(ctx, p.srcsPathMappings)
} }
var launcherPath android.OptionalPath var launcherPath android.OptionalPath
embeddedLauncher := p.isEmbeddedLauncherEnabled()
if embeddedLauncher { if embeddedLauncher {
ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) { ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) {
if provider, ok := m.(IntermPathProvider); ok { if provider, ok := m.(IntermPathProvider); ok {
@ -175,15 +128,137 @@ func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersio
} }
}) })
} }
binFile := registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath,
binary.getHostInterpreterName(ctx, actualVersion), p.getHostInterpreterName(ctx, p.properties.Actual_version),
main, binary.getStem(ctx), append(android.Paths{srcsZip}, depsSrcsZips...)) main, p.getStem(ctx), append(android.Paths{p.srcsZip}, depsSrcsZips...))
return android.OptionalPathForPath(binFile) var sharedLibs []string
// if embedded launcher is enabled, we need to collect the shared library dependencies of the
// launcher
for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) {
sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep))
}
p.androidMkSharedLibs = sharedLibs
}
func (p *PythonBinaryModule) AndroidMkEntries() []android.AndroidMkEntries {
entries := android.AndroidMkEntries{OutputFile: android.OptionalPathForPath(p.installSource)}
entries.Class = "EXECUTABLES"
entries.ExtraEntries = append(entries.ExtraEntries,
func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...)
})
entries.Required = append(entries.Required, "libc++")
entries.ExtraEntries = append(entries.ExtraEntries,
func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
path, file := filepath.Split(p.installedDest.String())
stem := strings.TrimSuffix(file, filepath.Ext(file))
entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file))
entries.SetString("LOCAL_MODULE_PATH", path)
entries.SetString("LOCAL_MODULE_STEM", stem)
entries.AddStrings("LOCAL_SHARED_LIBRARIES", p.androidMkSharedLibs...)
entries.SetBool("LOCAL_CHECK_ELF_FILES", false)
})
return []android.AndroidMkEntries{entries}
}
func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
p.PythonLibraryModule.DepsMutator(ctx)
versionVariation := []blueprint.Variation{
{"python_version", p.properties.Actual_version},
}
// If this module will be installed and has an embedded launcher, we need to add dependencies for:
// * standard library
// * launcher
// * shared dependencies of the launcher
if p.isEmbeddedLauncherEnabled() {
var stdLib string
var launcherModule string
// Add launcher shared lib dependencies. Ideally, these should be
// derived from the `shared_libs` property of the launcher. However, we
// cannot read the property at this stage and it will be too late to add
// dependencies later.
launcherSharedLibDeps := []string{
"libsqlite",
}
// Add launcher-specific dependencies for bionic
if ctx.Target().Os.Bionic() {
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
}
if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
}
switch p.properties.Actual_version {
case pyVersion2:
stdLib = "py2-stdlib"
launcherModule = "py2-launcher"
if p.autorun() {
launcherModule = "py2-launcher-autorun"
}
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
case pyVersion3:
stdLib = "py3-stdlib"
launcherModule = "py3-launcher"
if p.autorun() {
launcherModule = "py3-launcher-autorun"
}
if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
launcherModule += "-static"
}
if ctx.Device() {
launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
}
default:
panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
p.properties.Actual_version, ctx.ModuleName()))
}
ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
}
}
// HostToolPath returns a path if appropriate such that this module can be used as a host tool,
// fulfilling the android.HostToolProvider interface.
func (p *PythonBinaryModule) HostToolPath() android.OptionalPath {
// TODO: This should only be set when building host binaries -- tests built for device would be
// setting this incorrectly.
return android.OptionalPathForPath(p.installedDest)
}
// OutputFiles returns output files based on given tag, returns an error if tag is unsupported.
func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) {
switch tag {
case "":
return android.Paths{p.installSource}, nil
default:
return nil, fmt.Errorf("unsupported module reference tag %q", tag)
}
}
func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool {
return Bool(p.properties.Embedded_launcher)
}
func (b *PythonBinaryModule) autorun() bool {
return BoolDefault(b.binaryProperties.Autorun, true)
} }
// get host interpreter name. // get host interpreter name.
func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext, func (p *PythonBinaryModule) getHostInterpreterName(ctx android.ModuleContext,
actualVersion string) string { actualVersion string) string {
var interp string var interp string
switch actualVersion { switch actualVersion {
@ -200,13 +275,13 @@ func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext,
} }
// find main program path within runfiles tree. // find main program path within runfiles tree.
func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext, func (p *PythonBinaryModule) getPyMainFile(ctx android.ModuleContext,
srcsPathMappings []pathMapping) string { srcsPathMappings []pathMapping) string {
var main string var main string
if String(binary.binaryProperties.Main) == "" { if String(p.binaryProperties.Main) == "" {
main = ctx.ModuleName() + pyExt main = ctx.ModuleName() + pyExt
} else { } else {
main = String(binary.binaryProperties.Main) main = String(p.binaryProperties.Main)
} }
for _, path := range srcsPathMappings { for _, path := range srcsPathMappings {
@ -219,11 +294,21 @@ func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext,
return "" return ""
} }
func (binary *binaryDecorator) getStem(ctx android.ModuleContext) string { func (p *PythonBinaryModule) getStem(ctx android.ModuleContext) string {
stem := ctx.ModuleName() stem := ctx.ModuleName()
if String(binary.binaryProperties.Stem) != "" { if String(p.binaryProperties.Stem) != "" {
stem = String(binary.binaryProperties.Stem) stem = String(p.binaryProperties.Stem)
} }
return stem + String(binary.binaryProperties.Suffix) return stem + String(p.binaryProperties.Suffix)
}
func installDir(ctx android.ModuleContext, dir, dir64, relative string) android.InstallPath {
if ctx.Arch().ArchType.Multilib == "lib64" && dir64 != "" {
dir = dir64
}
if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) {
dir = filepath.Join(dir, ctx.Arch().ArchType.String())
}
return android.PathForModuleInstall(ctx, dir, relative)
} }

226
python/bp2build.go Normal file
View file

@ -0,0 +1,226 @@
// Copyright 2023 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"fmt"
"path/filepath"
"strings"
"github.com/google/blueprint/proptools"
"android/soong/android"
"android/soong/bazel"
)
type bazelPythonLibraryAttributes struct {
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
Imports bazel.StringListAttribute
Srcs_version *string
}
type bazelPythonProtoLibraryAttributes struct {
Deps bazel.LabelListAttribute
}
type baseAttributes struct {
// TODO(b/200311466): Probably not translate b/c Bazel has no good equiv
//Pkg_path bazel.StringAttribute
// TODO: Related to Pkg_bath and similarLy gated
//Is_internal bazel.BoolAttribute
// Combines Srcs and Exclude_srcs
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
// Combines Data and Java_data (invariant)
Data bazel.LabelListAttribute
Imports bazel.StringListAttribute
}
func (m *PythonLibraryModule) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes {
var attrs baseAttributes
archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{})
for axis, configToProps := range archVariantBaseProps {
for config, props := range configToProps {
if baseProps, ok := props.(*BaseProperties); ok {
attrs.Srcs.SetSelectValue(axis, config,
android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs))
attrs.Deps.SetSelectValue(axis, config,
android.BazelLabelForModuleDeps(ctx, baseProps.Libs))
data := android.BazelLabelForModuleSrc(ctx, baseProps.Data)
data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data))
attrs.Data.SetSelectValue(axis, config, data)
}
}
}
partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{
"proto": android.ProtoSrcLabelPartition,
"py": bazel.LabelPartition{Keep_remainder: true},
})
attrs.Srcs = partitionedSrcs["py"]
if !partitionedSrcs["proto"].IsEmpty() {
protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"])
protoLabel := bazel.Label{Label: ":" + protoInfo.Name}
pyProtoLibraryName := m.Name() + "_py_proto"
ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{
Rule_class: "py_proto_library",
Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl",
}, android.CommonAttributes{
Name: pyProtoLibraryName,
}, &bazelPythonProtoLibraryAttributes{
Deps: bazel.MakeSingleLabelListAttribute(protoLabel),
})
attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName))
}
// Bazel normally requires `import path.from.top.of.tree` statements in
// python code, but with soong you can directly import modules from libraries.
// Add "imports" attributes to the bazel library so it matches soong's behavior.
imports := "."
if m.properties.Pkg_path != nil {
// TODO(b/215119317) This is a hack to handle the fact that we don't convert
// pkg_path properly right now. If the folder structure that contains this
// Android.bp file matches pkg_path, we can set imports to an appropriate
// number of ../..s to emulate moving the files under a pkg_path folder.
pkg_path := filepath.Clean(*m.properties.Pkg_path)
if strings.HasPrefix(pkg_path, "/") {
ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path)
}
if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path {
ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir())
}
numFolders := strings.Count(pkg_path, "/") + 1
dots := make([]string, numFolders)
for i := 0; i < numFolders; i++ {
dots[i] = ".."
}
imports = strings.Join(dots, "/")
}
attrs.Imports = bazel.MakeStringListAttribute([]string{imports})
return attrs
}
func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *PythonLibraryModule) {
// TODO(b/182306917): this doesn't fully handle all nested props versioned
// by the python version, which would have been handled by the version split
// mutator. This is sufficient for very simple python_library modules under
// Bionic.
py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true)
py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false)
var python_version *string
if py2Enabled && !py3Enabled {
python_version = &pyVersion2
} else if !py2Enabled && py3Enabled {
python_version = &pyVersion3
} else if !py2Enabled && !py3Enabled {
ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled")
} else {
// do nothing, since python_version defaults to PY2ANDPY3
}
baseAttrs := m.makeArchVariantBaseAttributes(ctx)
attrs := &bazelPythonLibraryAttributes{
Srcs: baseAttrs.Srcs,
Deps: baseAttrs.Deps,
Srcs_version: python_version,
Imports: baseAttrs.Imports,
}
props := bazel.BazelTargetModuleProperties{
// Use the native py_library rule.
Rule_class: "py_library",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{
Name: m.Name(),
Data: baseAttrs.Data,
}, attrs)
}
type bazelPythonBinaryAttributes struct {
Main *bazel.Label
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
Python_version *string
Imports bazel.StringListAttribute
}
func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *PythonBinaryModule) {
// TODO(b/182306917): this doesn't fully handle all nested props versioned
// by the python version, which would have been handled by the version split
// mutator. This is sufficient for very simple python_binary_host modules
// under Bionic.
py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false)
py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false)
var python_version *string
if py3Enabled && py2Enabled {
panic(fmt.Errorf(
"error for '%s' module: bp2build's python_binary_host converter does not support "+
"converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name()))
} else if py2Enabled {
python_version = &pyVersion2
} else {
// do nothing, since python_version defaults to PY3.
}
baseAttrs := m.makeArchVariantBaseAttributes(ctx)
attrs := &bazelPythonBinaryAttributes{
Main: nil,
Srcs: baseAttrs.Srcs,
Deps: baseAttrs.Deps,
Python_version: python_version,
Imports: baseAttrs.Imports,
}
for _, propIntf := range m.GetProperties() {
if props, ok := propIntf.(*BinaryProperties); ok {
// main is optional.
if props.Main != nil {
main := android.BazelLabelForModuleSrcSingle(ctx, *props.Main)
attrs.Main = &main
break
}
}
}
props := bazel.BazelTargetModuleProperties{
// Use the native py_binary rule.
Rule_class: "py_binary",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{
Name: m.Name(),
Data: baseAttrs.Data,
}, attrs)
}
func (p *PythonLibraryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
pythonLibBp2Build(ctx, p)
}
func (p *PythonBinaryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
pythonBinaryBp2Build(ctx, p)
}
func (p *PythonTestModule) ConvertWithBp2build(_ android.TopDownMutatorContext) {
// Tests are currently unsupported
}

67
python/installer.go
View file

@ -1,67 +0,0 @@
// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"path/filepath"
"android/soong/android"
)
// This file handles installing python executables into their final location
type installLocation int
const (
InstallInData installLocation = iota
)
type pythonInstaller struct {
dir string
dir64 string
relative string
path android.InstallPath
androidMkSharedLibs []string
}
func NewPythonInstaller(dir, dir64 string) *pythonInstaller {
return &pythonInstaller{
dir: dir,
dir64: dir64,
}
}
var _ installer = (*pythonInstaller)(nil)
func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.InstallPath {
dir := installer.dir
if ctx.Arch().ArchType.Multilib == "lib64" && installer.dir64 != "" {
dir = installer.dir64
}
if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) {
dir = filepath.Join(dir, ctx.Arch().ArchType.String())
}
return android.PathForModuleInstall(ctx, dir, installer.relative)
}
func (installer *pythonInstaller) install(ctx android.ModuleContext, file android.Path) {
installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file)
}
func (installer *pythonInstaller) setAndroidMkSharedLibs(sharedLibs []string) {
installer.androidMkSharedLibs = sharedLibs
}

64
python/library.go
View file

@ -18,9 +18,6 @@ package python
import ( import (
"android/soong/android" "android/soong/android"
"android/soong/bazel"
"github.com/google/blueprint/proptools"
) )
func init() { func init() {
@ -33,66 +30,9 @@ func registerPythonLibraryComponents(ctx android.RegistrationContext) {
} }
func PythonLibraryHostFactory() android.Module { func PythonLibraryHostFactory() android.Module {
module := newModule(android.HostSupported, android.MultilibFirst) return newModule(android.HostSupported, android.MultilibFirst).init()
android.InitBazelModule(module)
return module.init()
}
type bazelPythonLibraryAttributes struct {
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
Imports bazel.StringListAttribute
Srcs_version *string
}
type bazelPythonProtoLibraryAttributes struct {
Deps bazel.LabelListAttribute
}
func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *Module) {
// TODO(b/182306917): this doesn't fully handle all nested props versioned
// by the python version, which would have been handled by the version split
// mutator. This is sufficient for very simple python_library modules under
// Bionic.
py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true)
py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false)
var python_version *string
if py2Enabled && !py3Enabled {
python_version = &pyVersion2
} else if !py2Enabled && py3Enabled {
python_version = &pyVersion3
} else if !py2Enabled && !py3Enabled {
ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled")
} else {
// do nothing, since python_version defaults to PY2ANDPY3
}
baseAttrs := m.makeArchVariantBaseAttributes(ctx)
attrs := &bazelPythonLibraryAttributes{
Srcs: baseAttrs.Srcs,
Deps: baseAttrs.Deps,
Srcs_version: python_version,
Imports: baseAttrs.Imports,
}
props := bazel.BazelTargetModuleProperties{
// Use the native py_library rule.
Rule_class: "py_library",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{
Name: m.Name(),
Data: baseAttrs.Data,
}, attrs)
} }
func PythonLibraryFactory() android.Module { func PythonLibraryFactory() android.Module {
module := newModule(android.HostAndDeviceSupported, android.MultilibBoth) return newModule(android.HostAndDeviceSupported, android.MultilibBoth).init()
android.InitBazelModule(module)
return module.init()
} }

361
python/python.go
View file

@ -22,8 +22,6 @@ import (
"regexp" "regexp"
"strings" "strings"
"android/soong/bazel"
"github.com/google/blueprint" "github.com/google/blueprint"
"github.com/google/blueprint/proptools" "github.com/google/blueprint/proptools"
@ -122,26 +120,13 @@ type BaseProperties struct {
Embedded_launcher *bool `blueprint:"mutated"` Embedded_launcher *bool `blueprint:"mutated"`
} }
type baseAttributes struct {
// TODO(b/200311466): Probably not translate b/c Bazel has no good equiv
//Pkg_path bazel.StringAttribute
// TODO: Related to Pkg_bath and similarLy gated
//Is_internal bazel.BoolAttribute
// Combines Srcs and Exclude_srcs
Srcs bazel.LabelListAttribute
Deps bazel.LabelListAttribute
// Combines Data and Java_data (invariant)
Data bazel.LabelListAttribute
Imports bazel.StringListAttribute
}
// Used to store files of current module after expanding dependencies // Used to store files of current module after expanding dependencies
type pathMapping struct { type pathMapping struct {
dest string dest string
src android.Path src android.Path
} }
type Module struct { type PythonLibraryModule struct {
android.ModuleBase android.ModuleBase
android.DefaultableModuleBase android.DefaultableModuleBase
android.BazelModuleBase android.BazelModuleBase
@ -153,16 +138,6 @@ type Module struct {
hod android.HostOrDeviceSupported hod android.HostOrDeviceSupported
multilib android.Multilib multilib android.Multilib
// interface used to bootstrap .par executable when embedded_launcher is true
// this should be set by Python modules which are runnable, e.g. binaries and tests
// bootstrapper might be nil (e.g. Python library module).
bootstrapper bootstrapper
// interface that implements functions required for installation
// this should be set by Python modules which are runnable, e.g. binaries and tests
// installer might be nil (e.g. Python library module).
installer installer
// the Python files of current module after expanding source dependencies. // the Python files of current module after expanding source dependencies.
// pathMapping: <dest: runfile_path, src: source_path> // pathMapping: <dest: runfile_path, src: source_path>
srcsPathMappings []pathMapping srcsPathMappings []pathMapping
@ -173,110 +148,16 @@ type Module struct {
// the zip filepath for zipping current module source/data files. // the zip filepath for zipping current module source/data files.
srcsZip android.Path srcsZip android.Path
// dependency modules' zip filepath for zipping current module source/data files.
depsSrcsZips android.Paths
// (.intermediate) module output path as installation source.
installSource android.OptionalPath
// Map to ensure sub-part of the AndroidMk for this module is only added once
subAndroidMkOnce map[subAndroidMkProvider]bool
} }
// newModule generates new Python base module // newModule generates new Python base module
func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *PythonLibraryModule {
return &Module{ return &PythonLibraryModule{
hod: hod, hod: hod,
multilib: multilib, multilib: multilib,
} }
} }
func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes {
var attrs baseAttributes
archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{})
for axis, configToProps := range archVariantBaseProps {
for config, props := range configToProps {
if baseProps, ok := props.(*BaseProperties); ok {
attrs.Srcs.SetSelectValue(axis, config,
android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs))
attrs.Deps.SetSelectValue(axis, config,
android.BazelLabelForModuleDeps(ctx, baseProps.Libs))
data := android.BazelLabelForModuleSrc(ctx, baseProps.Data)
data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data))
attrs.Data.SetSelectValue(axis, config, data)
}
}
}
partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{
"proto": android.ProtoSrcLabelPartition,
"py": bazel.LabelPartition{Keep_remainder: true},
})
attrs.Srcs = partitionedSrcs["py"]
if !partitionedSrcs["proto"].IsEmpty() {
protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"])
protoLabel := bazel.Label{Label: ":" + protoInfo.Name}
pyProtoLibraryName := m.Name() + "_py_proto"
ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{
Rule_class: "py_proto_library",
Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl",
}, android.CommonAttributes{
Name: pyProtoLibraryName,
}, &bazelPythonProtoLibraryAttributes{
Deps: bazel.MakeSingleLabelListAttribute(protoLabel),
})
attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName))
}
// Bazel normally requires `import path.from.top.of.tree` statements in
// python code, but with soong you can directly import modules from libraries.
// Add "imports" attributes to the bazel library so it matches soong's behavior.
imports := "."
if m.properties.Pkg_path != nil {
// TODO(b/215119317) This is a hack to handle the fact that we don't convert
// pkg_path properly right now. If the folder structure that contains this
// Android.bp file matches pkg_path, we can set imports to an appropriate
// number of ../..s to emulate moving the files under a pkg_path folder.
pkg_path := filepath.Clean(*m.properties.Pkg_path)
if strings.HasPrefix(pkg_path, "/") {
ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path)
}
if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path {
ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir())
}
numFolders := strings.Count(pkg_path, "/") + 1
dots := make([]string, numFolders)
for i := 0; i < numFolders; i++ {
dots[i] = ".."
}
imports = strings.Join(dots, "/")
}
attrs.Imports = bazel.MakeStringListAttribute([]string{imports})
return attrs
}
// bootstrapper interface should be implemented for runnable modules, e.g. binary and test
type bootstrapper interface {
bootstrapperProps() []interface{}
bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool,
srcsPathMappings []pathMapping, srcsZip android.Path,
depsSrcsZips android.Paths) android.OptionalPath
autorun() bool
}
// installer interface should be implemented for installable modules, e.g. binary and test
type installer interface {
install(ctx android.ModuleContext, path android.Path)
setAndroidMkSharedLibs(sharedLibs []string)
}
// interface implemented by Python modules to provide source and data mappings and zip to python // interface implemented by Python modules to provide source and data mappings and zip to python
// modules that depend on it // modules that depend on it
type pythonDependency interface { type pythonDependency interface {
@ -286,37 +167,31 @@ type pythonDependency interface {
} }
// getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
func (p *Module) getSrcsPathMappings() []pathMapping { func (p *PythonLibraryModule) getSrcsPathMappings() []pathMapping {
return p.srcsPathMappings return p.srcsPathMappings
} }
// getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination
func (p *Module) getDataPathMappings() []pathMapping { func (p *PythonLibraryModule) getDataPathMappings() []pathMapping {
return p.dataPathMappings return p.dataPathMappings
} }
// getSrcsZip returns the filepath where the current module's source/data files are zipped. // getSrcsZip returns the filepath where the current module's source/data files are zipped.
func (p *Module) getSrcsZip() android.Path { func (p *PythonLibraryModule) getSrcsZip() android.Path {
return p.srcsZip return p.srcsZip
} }
var _ pythonDependency = (*Module)(nil) func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
return &p.properties
var _ android.AndroidMkEntriesProvider = (*Module)(nil)
func (p *Module) init(additionalProps ...interface{}) android.Module {
p.AddProperties(&p.properties, &p.protoProperties)
// Add additional properties for bootstrapping/installation
// This is currently tied to the bootstrapper interface;
// however, these are a combination of properties for the installation and bootstrapping of a module
if p.bootstrapper != nil {
p.AddProperties(p.bootstrapper.bootstrapperProps()...)
} }
var _ pythonDependency = (*PythonLibraryModule)(nil)
func (p *PythonLibraryModule) init() android.Module {
p.AddProperties(&p.properties, &p.protoProperties)
android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitAndroidArchModule(p, p.hod, p.multilib)
android.InitDefaultableModule(p) android.InitDefaultableModule(p)
android.InitBazelModule(p)
return p return p
} }
@ -350,24 +225,29 @@ var (
internalPath = "internal" internalPath = "internal"
) )
type basePropertiesProvider interface {
getBaseProperties() *BaseProperties
}
// versionSplitMutator creates version variants for modules and appends the version-specific // versionSplitMutator creates version variants for modules and appends the version-specific
// properties for a given variant to the properties in the variant module // properties for a given variant to the properties in the variant module
func versionSplitMutator() func(android.BottomUpMutatorContext) { func versionSplitMutator() func(android.BottomUpMutatorContext) {
return func(mctx android.BottomUpMutatorContext) { return func(mctx android.BottomUpMutatorContext) {
if base, ok := mctx.Module().(*Module); ok { if base, ok := mctx.Module().(basePropertiesProvider); ok {
versionNames := []string{} props := base.getBaseProperties()
var versionNames []string
// collect version specific properties, so that we can merge version-specific properties // collect version specific properties, so that we can merge version-specific properties
// into the module's overall properties // into the module's overall properties
versionProps := []VersionProperties{} var versionProps []VersionProperties
// PY3 is first so that we alias the PY3 variant rather than PY2 if both // PY3 is first so that we alias the PY3 variant rather than PY2 if both
// are available // are available
if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) { if proptools.BoolDefault(props.Version.Py3.Enabled, true) {
versionNames = append(versionNames, pyVersion3) versionNames = append(versionNames, pyVersion3)
versionProps = append(versionProps, base.properties.Version.Py3) versionProps = append(versionProps, props.Version.Py3)
} }
if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) { if proptools.BoolDefault(props.Version.Py2.Enabled, false) {
versionNames = append(versionNames, pyVersion2) versionNames = append(versionNames, pyVersion2)
versionProps = append(versionProps, base.properties.Version.Py2) versionProps = append(versionProps, props.Version.Py2)
} }
modules := mctx.CreateLocalVariations(versionNames...) modules := mctx.CreateLocalVariations(versionNames...)
// Alias module to the first variant // Alias module to the first variant
@ -376,9 +256,10 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) {
} }
for i, v := range versionNames { for i, v := range versionNames {
// set the actual version for Python module. // set the actual version for Python module.
modules[i].(*Module).properties.Actual_version = v newProps := modules[i].(basePropertiesProvider).getBaseProperties()
newProps.Actual_version = v
// append versioned properties for the Python module to the overall properties // append versioned properties for the Python module to the overall properties
err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil) err := proptools.AppendMatchingProperties([]interface{}{newProps}, &versionProps[i], nil)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -387,38 +268,6 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) {
} }
} }
// HostToolPath returns a path if appropriate such that this module can be used as a host tool,
// fulfilling HostToolProvider interface.
func (p *Module) HostToolPath() android.OptionalPath {
if p.installer != nil {
if bin, ok := p.installer.(*binaryDecorator); ok {
// TODO: This should only be set when building host binaries -- tests built for device would be
// setting this incorrectly.
return android.OptionalPathForPath(bin.path)
}
}
return android.OptionalPath{}
}
// OutputFiles returns output files based on given tag, returns an error if tag is unsupported.
func (p *Module) OutputFiles(tag string) (android.Paths, error) {
switch tag {
case "":
if outputFile := p.installSource; outputFile.Valid() {
return android.Paths{outputFile.Path()}, nil
}
return android.Paths{}, nil
default:
return nil, fmt.Errorf("unsupported module reference tag %q", tag)
}
}
func (p *Module) isEmbeddedLauncherEnabled() bool {
return p.installer != nil && Bool(p.properties.Embedded_launcher)
}
func anyHasExt(paths []string, ext string) bool { func anyHasExt(paths []string, ext string) bool {
for _, p := range paths { for _, p := range paths {
if filepath.Ext(p) == ext { if filepath.Ext(p) == ext {
@ -429,7 +278,7 @@ func anyHasExt(paths []string, ext string) bool {
return false return false
} }
func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { func (p *PythonLibraryModule) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool {
return anyHasExt(p.properties.Srcs, ext) return anyHasExt(p.properties.Srcs, ext)
} }
@ -437,7 +286,7 @@ func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bo
// - handles proto dependencies, // - handles proto dependencies,
// - if required, specifies launcher and adds launcher dependencies, // - if required, specifies launcher and adds launcher dependencies,
// - applies python version mutations to Python dependencies // - applies python version mutations to Python dependencies
func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
android.ProtoDeps(ctx, &p.protoProperties) android.ProtoDeps(ctx, &p.protoProperties)
versionVariation := []blueprint.Variation{ versionVariation := []blueprint.Variation{
@ -452,111 +301,15 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
// Add python library dependencies for this python version variation // Add python library dependencies for this python version variation
ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...) ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
// If this module will be installed and has an embedded launcher, we need to add dependencies for:
// * standard library
// * launcher
// * shared dependencies of the launcher
if p.installer != nil && p.isEmbeddedLauncherEnabled() {
var stdLib string
var launcherModule string
// Add launcher shared lib dependencies. Ideally, these should be
// derived from the `shared_libs` property of the launcher. However, we
// cannot read the property at this stage and it will be too late to add
// dependencies later.
launcherSharedLibDeps := []string{
"libsqlite",
}
// Add launcher-specific dependencies for bionic
if ctx.Target().Os.Bionic() {
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
}
if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
}
switch p.properties.Actual_version {
case pyVersion2:
stdLib = "py2-stdlib"
launcherModule = "py2-launcher"
if p.bootstrapper.autorun() {
launcherModule = "py2-launcher-autorun"
}
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
case pyVersion3:
stdLib = "py3-stdlib"
launcherModule = "py3-launcher"
if p.bootstrapper.autorun() {
launcherModule = "py3-launcher-autorun"
}
if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
launcherModule += "-static"
}
if ctx.Device() {
launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
}
default:
panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
p.properties.Actual_version, ctx.ModuleName()))
}
ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
}
// Emulate the data property for java_data but with the arch variation overridden to "common" // Emulate the data property for java_data but with the arch variation overridden to "common"
// so that it can point to java modules. // so that it can point to java modules.
javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}} javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...) ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
} }
func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { // GenerateAndroidBuildActions performs build actions common to all Python modules
p.generatePythonBuildActions(ctx) func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// Only Python binary and test modules have non-empty bootstrapper.
if p.bootstrapper != nil {
// if the module is being installed, we need to collect all transitive dependencies to embed in
// the final par
p.collectPathsFromTransitiveDeps(ctx)
// bootstrap the module, including resolving main file, getting launcher path, and
// registering actions to build the par file
// bootstrap returns the binary output path
p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips)
}
// Only Python binary and test modules have non-empty installer.
if p.installer != nil {
var sharedLibs []string
// if embedded launcher is enabled, we need to collect the shared library depenendencies of the
// launcher
for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) {
sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep))
}
p.installer.setAndroidMkSharedLibs(sharedLibs)
// Install the par file from installSource
if p.installSource.Valid() {
p.installer.install(ctx, p.installSource.Path())
}
}
}
// generatePythonBuildActions performs build actions common to all Python modules
func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) {
expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs)
requiresSrcs := true
if p.bootstrapper != nil && !p.bootstrapper.autorun() {
requiresSrcs = false
}
if len(expandedSrcs) == 0 && requiresSrcs {
ctx.ModuleErrorf("doesn't have any source files!")
}
// expand data files from "data" property. // expand data files from "data" property.
expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) expandedData := android.PathsForModuleSrc(ctx, p.properties.Data)
@ -607,7 +360,7 @@ func isValidPythonPath(path string) error {
// For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path> // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path>
// for python/data files expanded from properties. // for python/data files expanded from properties.
func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string, func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, pkgPath string,
expandedSrcs, expandedData android.Paths) { expandedSrcs, expandedData android.Paths) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check current module duplicates. // check current module duplicates.
@ -642,7 +395,7 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string
} }
// createSrcsZip registers build actions to zip current module's sources and data. // createSrcsZip registers build actions to zip current module's sources and data.
func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
relativeRootMap := make(map[string]android.Paths) relativeRootMap := make(map[string]android.Paths)
pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
@ -654,13 +407,8 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi
if path.src.Ext() == protoExt { if path.src.Ext() == protoExt {
protoSrcs = append(protoSrcs, path.src) protoSrcs = append(protoSrcs, path.src)
} else { } else {
var relativeRoot string relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel()) relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src)
if v, found := relativeRootMap[relativeRoot]; found {
relativeRootMap[relativeRoot] = append(v, path.src)
} else {
relativeRootMap[relativeRoot] = android.Paths{path.src}
}
} }
} }
var zips android.Paths var zips android.Paths
@ -736,30 +484,20 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi
} }
} }
// isPythonLibModule returns whether the given module is a Python library Module or not // isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not
func isPythonLibModule(module blueprint.Module) bool { func isPythonLibModule(module blueprint.Module) bool {
if m, ok := module.(*Module); ok { if _, ok := module.(*PythonLibraryModule); ok {
return m.isLibrary() if _, ok := module.(*PythonBinaryModule); !ok {
return true
}
} }
return false return false
} }
// This is distinguished by the fact that Python libraries are not installable, while other Python
// modules are.
func (p *Module) isLibrary() bool {
// Python library has no bootstrapper or installer
return p.bootstrapper == nil && p.installer == nil
}
func (p *Module) isBinary() bool {
_, ok := p.bootstrapper.(*binaryDecorator)
return ok
}
// collectPathsFromTransitiveDeps checks for source/data files for duplicate paths // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
// for module and its transitive dependencies and collects list of data/source file // for module and its transitive dependencies and collects list of data/source file
// zips for transitive dependencies. // zips for transitive dependencies.
func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext) android.Paths {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates. // check duplicates.
destToPySrcs := make(map[string]string) destToPySrcs := make(map[string]string)
@ -773,6 +511,8 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) {
seen := make(map[android.Module]bool) seen := make(map[android.Module]bool)
var result android.Paths
// visit all its dependencies in depth first. // visit all its dependencies in depth first.
ctx.WalkDeps(func(child, parent android.Module) bool { ctx.WalkDeps(func(child, parent android.Module) bool {
// we only collect dependencies tagged as python library deps // we only collect dependencies tagged as python library deps
@ -801,10 +541,11 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) {
checkForDuplicateOutputPath(ctx, destToPyData, checkForDuplicateOutputPath(ctx, destToPyData,
path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
} }
p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip()) result = append(result, dep.getSrcsZip())
} }
return true return true
}) })
return result
} }
// chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which
@ -825,18 +566,10 @@ func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string,
} }
// InstallInData returns true as Python is not supported in the system partition // InstallInData returns true as Python is not supported in the system partition
func (p *Module) InstallInData() bool { func (p *PythonLibraryModule) InstallInData() bool {
return true return true
} }
func (p *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
if p.isLibrary() {
pythonLibBp2Build(ctx, p)
} else if p.isBinary() {
pythonBinaryBp2Build(ctx, p)
}
}
var Bool = proptools.Bool var Bool = proptools.Bool
var BoolDefault = proptools.BoolDefault var BoolDefault = proptools.BoolDefault
var String = proptools.String var String = proptools.String

12
python/python_test.go
View file

@ -312,10 +312,6 @@ var (
"e/file4.py", "e/file4.py",
}, },
srcsZip: "out/soong/.intermediates/dir/bin/PY3/bin.py.srcszip", srcsZip: "out/soong/.intermediates/dir/bin/PY3/bin.py.srcszip",
depsSrcsZips: []string{
"out/soong/.intermediates/dir/lib5/PY3/lib5.py.srcszip",
"out/soong/.intermediates/dir/lib6/PY3/lib6.py.srcszip",
},
}, },
}, },
}, },
@ -346,17 +342,17 @@ func TestPythonModule(t *testing.T) {
for _, e := range d.expectedBinaries { for _, e := range d.expectedBinaries {
t.Run(e.name, func(t *testing.T) { t.Run(e.name, func(t *testing.T) {
expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles, e.depsSrcsZips) expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles)
}) })
} }
}) })
} }
} }
func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles, expectedDepsSrcsZips []string) { func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles []string) {
module := ctx.ModuleForTests(name, variant) module := ctx.ModuleForTests(name, variant)
base, baseOk := module.Module().(*Module) base, baseOk := module.Module().(*PythonLibraryModule)
if !baseOk { if !baseOk {
t.Fatalf("%s is not Python module!", name) t.Fatalf("%s is not Python module!", name)
} }
@ -369,8 +365,6 @@ func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expecte
android.AssertDeepEquals(t, "pyRunfiles", expectedPyRunfiles, actualPyRunfiles) android.AssertDeepEquals(t, "pyRunfiles", expectedPyRunfiles, actualPyRunfiles)
android.AssertPathRelativeToTopEquals(t, "srcsZip", expectedSrcsZip, base.srcsZip) android.AssertPathRelativeToTopEquals(t, "srcsZip", expectedSrcsZip, base.srcsZip)
android.AssertPathsRelativeToTopEquals(t, "depsSrcsZips", expectedDepsSrcsZips, base.depsSrcsZips)
} }
func TestMain(m *testing.M) { func TestMain(m *testing.M) {

107
python/test.go
View file

@ -32,6 +32,20 @@ func registerPythonTestComponents(ctx android.RegistrationContext) {
ctx.RegisterModuleType("python_test", PythonTestFactory) ctx.RegisterModuleType("python_test", PythonTestFactory)
} }
func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule {
return &PythonTestModule{PythonBinaryModule: *NewBinary(hod)}
}
func PythonTestHostFactory() android.Module {
return NewTest(android.HostSupportedNoCross).init()
}
func PythonTestFactory() android.Module {
module := NewTest(android.HostAndDeviceSupported)
module.multilib = android.MultilibBoth
return module.init()
}
type TestProperties struct { type TestProperties struct {
// the name of the test configuration (for example "AndroidTest.xml") that should be // the name of the test configuration (for example "AndroidTest.xml") that should be
// installed with the module. // installed with the module.
@ -52,76 +66,79 @@ type TestProperties struct {
Test_options android.CommonTestOptions Test_options android.CommonTestOptions
} }
type testDecorator struct { type PythonTestModule struct {
*binaryDecorator PythonBinaryModule
testProperties TestProperties testProperties TestProperties
testConfig android.Path testConfig android.Path
data []android.DataPath data []android.DataPath
} }
func (test *testDecorator) bootstrapperProps() []interface{} { func (p *PythonTestModule) init() android.Module {
return append(test.binaryDecorator.bootstrapperProps(), &test.testProperties) p.AddProperties(&p.properties, &p.protoProperties)
p.AddProperties(&p.binaryProperties)
p.AddProperties(&p.testProperties)
android.InitAndroidArchModule(p, p.hod, p.multilib)
android.InitDefaultableModule(p)
android.InitBazelModule(p)
if p.hod == android.HostSupportedNoCross && p.testProperties.Test_options.Unit_test == nil {
p.testProperties.Test_options.Unit_test = proptools.BoolPtr(true)
}
return p
} }
func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) { func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
test.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ // We inherit from only the library's GenerateAndroidBuildActions, and then
TestConfigProp: test.testProperties.Test_config, // just use buildBinary() so that the binary is not installed into the location
TestConfigTemplateProp: test.testProperties.Test_config_template, // it would be for regular binaries.
TestSuites: test.binaryDecorator.binaryProperties.Test_suites, p.PythonLibraryModule.GenerateAndroidBuildActions(ctx)
AutoGenConfig: test.binaryDecorator.binaryProperties.Auto_gen_config, p.buildBinary(ctx)
p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{
TestConfigProp: p.testProperties.Test_config,
TestConfigTemplateProp: p.testProperties.Test_config_template,
TestSuites: p.binaryProperties.Test_suites,
AutoGenConfig: p.binaryProperties.Auto_gen_config,
DeviceTemplate: "${PythonBinaryHostTestConfigTemplate}", DeviceTemplate: "${PythonBinaryHostTestConfigTemplate}",
HostTemplate: "${PythonBinaryHostTestConfigTemplate}", HostTemplate: "${PythonBinaryHostTestConfigTemplate}",
}) })
test.binaryDecorator.pythonInstaller.dir = "nativetest" p.installedDest = ctx.InstallFile(installDir(ctx, "nativetest", "nativetest64", ctx.ModuleName()), p.installSource.Base(), p.installSource)
test.binaryDecorator.pythonInstaller.dir64 = "nativetest64"
test.binaryDecorator.pythonInstaller.relative = ctx.ModuleName() for _, dataSrcPath := range android.PathsForModuleSrc(ctx, p.testProperties.Data) {
p.data = append(p.data, android.DataPath{SrcPath: dataSrcPath})
test.binaryDecorator.pythonInstaller.install(ctx, file)
dataSrcPaths := android.PathsForModuleSrc(ctx, test.testProperties.Data)
for _, dataSrcPath := range dataSrcPaths {
test.data = append(test.data, android.DataPath{SrcPath: dataSrcPath})
} }
// Emulate the data property for java_data dependencies. // Emulate the data property for java_data dependencies.
for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) {
for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") { for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") {
test.data = append(test.data, android.DataPath{SrcPath: javaDataSrcPath}) p.data = append(p.data, android.DataPath{SrcPath: javaDataSrcPath})
} }
} }
} }
func NewTest(hod android.HostOrDeviceSupported) *Module { func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries {
module, binary := NewBinary(hod) entriesList := p.PythonBinaryModule.AndroidMkEntries()
if len(entriesList) != 1 {
panic("Expected 1 entry")
}
entries := &entriesList[0]
binary.pythonInstaller = NewPythonInstaller("nativetest", "nativetest64") entries.Class = "NATIVE_TESTS"
test := &testDecorator{binaryDecorator: binary} entries.ExtraEntries = append(entries.ExtraEntries,
if hod == android.HostSupportedNoCross && test.testProperties.Test_options.Unit_test == nil { func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
test.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) //entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...)
if p.testConfig != nil {
entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String())
} }
module.bootstrapper = test entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true))
module.installer = test
return module entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...)
}
p.testProperties.Test_options.SetAndroidMkEntries(entries)
func PythonTestHostFactory() android.Module { })
module := NewTest(android.HostSupportedNoCross)
return entriesList
return module.init()
}
func PythonTestFactory() android.Module {
module := NewTest(android.HostAndDeviceSupported)
module.multilib = android.MultilibBoth
return module.init()
} }