tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Prepare soong for python 3.11

Browse source 
Due to upstream changes in python path calculations, I'm simplifying the
python launcher to work more like a standard python distribution. This
is effectively changing the stdlib path from `internal/stdlib` to
`internal/python3.10` (or `3.11`, etc). This allows us to specify the
zip file as PYTHONHOME, set PYTHONPLATLIBDIR to `internal` and use the
default detection after that.

That does mean during upgrades that the stdlib pkg path will change, so
move the source vs prebuilt calculation from the Android.bp into Soong
to choose which stdlib module to pick up (with the corresponding
`pkg_path`)

Bug: 278602456
Test: treehugger with python3.10
Test: a python3.11 source + 3.10 prebuilt build
Test: a python3.11 source+prebuilt build
Change-Id: I8b02e7b22a1f1d1e02819ae1a31a99cdc985542c
This commit is contained in:
Dan Willemsen 2023-08-15 22:17:03 -04:00
parent d89ee08bbf
commit 339a63f044
4 changed files with 48 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/builder.go
View file

@ -73,14 +73,14 @@ var (
precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{
Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` +
`PYTHONPATH=$stdlibZip/internal/stdlib ` +
`PYTHONPATH=$stdlibZip/internal/$stdlibPkg ` +
`$launcher build/soong/python/scripts/precompile_python.py $in $out`,
CommandDeps: []string{
"$stdlibZip",
"$launcher",
"build/soong/python/scripts/precompile_python.py",
},
}, "stdlibZip", "launcher", "ldLibraryPath")
}, "stdlibZip", "stdlibPkg", "launcher", "ldLibraryPath")
)
func init() {

30
python/python.go
View file

@ -169,6 +169,7 @@ type pythonDependency interface {
getDataPathMappings() []pathMapping
getSrcsZip() android.Path
getPrecompiledSrcsZip() android.Path
getPkgPath() string
}
// getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
@ -191,6 +192,11 @@ func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
return p.precompiledSrcsZip
}
// getPkgPath returns the pkg_path value
func (p *PythonLibraryModule) getPkgPath() string {
return String(p.properties.Pkg_path)
}
func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
return &p.properties
}
@ -370,7 +376,20 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto
launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
case pyVersion3:
var prebuiltStdLib bool
if targetForDeps.Os.Bionic() {
prebuiltStdLib = false
} else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") {
prebuiltStdLib = false
} else {
prebuiltStdLib = true
}
if prebuiltStdLib {
stdLib = "py3-stdlib-prebuilt"
} else {
stdLib = "py3-stdlib"
}
launcherModule = "py3-launcher"
if autorun {
@ -461,15 +480,20 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p
destToPySrcs := make(map[string]string)
destToPyData := make(map[string]string)
// Disable path checks for the stdlib, as it includes a "." in the version string
isInternal := proptools.BoolDefault(p.properties.Is_internal, false)
for _, s := range expandedSrcs {
if s.Ext() != pyExt && s.Ext() != protoExt {
ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
continue
}
runfilesPath := filepath.Join(pkgPath, s.Rel())
if !isInternal {
if err := isValidPythonPath(runfilesPath); err != nil {
ctx.PropertyErrorf("srcs", err.Error())
}
}
if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
}
@ -591,13 +615,16 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.
// "cross compiling" for device here purely by virtue of host and device python bytecode
// being the same.
var stdLib android.Path
var stdLibPkg string
var launcher android.Path
if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
if proptools.BoolDefault(p.properties.Is_internal, false) {
stdLib = p.srcsZip
stdLibPkg = p.getPkgPath()
} else {
ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
if dep, ok := module.(pythonDependency); ok {
stdLib = dep.getPrecompiledSrcsZip()
stdLibPkg = dep.getPkgPath()
}
})
}
@ -636,6 +663,7 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.
Description: "Precompile the python sources of " + ctx.ModuleName(),
Args: map[string]string{
"stdlibZip": stdLib.String(),
"stdlibPkg": stdLibPkg,
"launcher": launcher.String(),
"ldLibraryPath": strings.Join(ldLibraryPath, ":"),
},

9
python/tests/par_test.py
View file

@ -33,6 +33,8 @@ if fileName.endswith('.pyc'):
assert_equal("os.path.basename(__file__)", fileName, "par_test.py")
archive = os.path.dirname(__file__)
major = sys.version_info.major
minor = sys.version_info.minor
assert_equal("__package__", __package__, "")
assert_equal("sys.argv[0]", sys.argv[0], archive)
@ -42,10 +44,11 @@ assert_equal("sys.prefix", sys.prefix, archive)
assert_equal("__loader__.archive", __loader__.archive, archive)
assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None)
assert_equal("len(sys.path)", len(sys.path), 3)
assert_equal("len(sys.path)", len(sys.path), 4)
assert_equal("sys.path[0]", sys.path[0], archive)
assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal"))
assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", "stdlib"))
assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal", f"python{major}{minor}.zip"))
assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", f"python{major}.{minor}"))
assert_equal("sys.path[3]", sys.path[3], os.path.join(archive, "internal", f"python{major}.{minor}", "lib-dynload"))
if os.getenv('ARGTEST', False):
assert_equal("len(sys.argv)", len(sys.argv), 3)

16
python/tests/py-cmd_test.py
View file

@ -55,22 +55,22 @@ assert_equal("sys.exec_prefix", sys.exec_prefix, sys.executable)
assert_equal("sys.prefix", sys.prefix, sys.executable)
assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None)
if sys.version_info[0] == 2:
major = sys.version_info.major
minor = sys.version_info.minor
if major == 2:
assert_equal("len(sys.path)", len(sys.path), 4)
assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__)))
assert_equal("sys.path[1]", sys.path[1], "/extra")
assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal"))
assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib"))
else:
assert_equal("len(sys.path)", len(sys.path), 8)
assert_equal("len(sys.path)", len(sys.path), 5)
assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__)))
assert_equal("sys.path[1]", sys.path[1], "/extra")
assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip'))
assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), '..'))
assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])))
assert_equal("sys.path[5]", sys.path[5], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload'))
assert_equal("sys.path[6]", sys.path[6], os.path.join(sys.executable, "internal"))
assert_equal("sys.path[7]", sys.path[7], os.path.join(sys.executable, "internal", "stdlib"))
assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip'))
assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])))
assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload'))
if failed:
sys.exit(1)