Prepare soong for python 3.11

Due to upstream changes in python path calculations, I'm simplifying the
python launcher to work more like a standard python distribution. This
is effectively changing the stdlib path from `internal/stdlib` to
`internal/python3.10` (or `3.11`, etc). This allows us to specify the
zip file as PYTHONHOME, set PYTHONPLATLIBDIR to `internal` and use the
default detection after that.

That does mean during upgrades that the stdlib pkg path will change, so
move the source vs prebuilt calculation from the Android.bp into Soong
to choose which stdlib module to pick up (with the corresponding
`pkg_path`)

Bug: 278602456
Test: treehugger with python3.10
Test: a python3.11 source + 3.10 prebuilt build
Test: a python3.11 source+prebuilt build
Change-Id: I8b02e7b22a1f1d1e02819ae1a31a99cdc985542c

python/builder.go

@@ -73,14 +73,14 @@ var (
 
 	precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{
 		Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` +
-			`PYTHONPATH=$stdlibZip/internal/stdlib ` +
+			`PYTHONPATH=$stdlibZip/internal/$stdlibPkg ` +
 			`$launcher build/soong/python/scripts/precompile_python.py $in $out`,
 		CommandDeps: []string{
 			"$stdlibZip",
 			"$launcher",
 			"build/soong/python/scripts/precompile_python.py",
 		},
-	}, "stdlibZip", "launcher", "ldLibraryPath")
+	}, "stdlibZip", "stdlibPkg", "launcher", "ldLibraryPath")
 )
 
 func init() {

python/python.go

@@ -169,6 +169,7 @@ type pythonDependency interface {
 	getDataPathMappings() []pathMapping
 	getSrcsZip() android.Path
 	getPrecompiledSrcsZip() android.Path
+	getPkgPath() string
 }
 
 // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
@@ -191,6 +192,11 @@ func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
 	return p.precompiledSrcsZip
 }
 
+// getPkgPath returns the pkg_path value
+func (p *PythonLibraryModule) getPkgPath() string {
+	return String(p.properties.Pkg_path)
+}
+
 func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
 	return &p.properties
 }
@@ -370,7 +376,20 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto
 
 		launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
 	case pyVersion3:
-		stdLib = "py3-stdlib"
+		var prebuiltStdLib bool
+		if targetForDeps.Os.Bionic() {
+			prebuiltStdLib = false
+		} else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") {
+			prebuiltStdLib = false
+		} else {
+			prebuiltStdLib = true
+		}
+
+		if prebuiltStdLib {
+			stdLib = "py3-stdlib-prebuilt"
+		} else {
+			stdLib = "py3-stdlib"
+		}
 
 		launcherModule = "py3-launcher"
 		if autorun {
@@ -461,14 +480,19 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p
 	destToPySrcs := make(map[string]string)
 	destToPyData := make(map[string]string)
 
+	// Disable path checks for the stdlib, as it includes a "." in the version string
+	isInternal := proptools.BoolDefault(p.properties.Is_internal, false)
+
 	for _, s := range expandedSrcs {
 		if s.Ext() != pyExt && s.Ext() != protoExt {
 			ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
 			continue
 		}
 		runfilesPath := filepath.Join(pkgPath, s.Rel())
-		if err := isValidPythonPath(runfilesPath); err != nil {
-			ctx.PropertyErrorf("srcs", err.Error())
+		if !isInternal {
+			if err := isValidPythonPath(runfilesPath); err != nil {
+				ctx.PropertyErrorf("srcs", err.Error())
+			}
 		}
 		if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
 			p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
@@ -591,13 +615,16 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.
 	// "cross compiling" for device here purely by virtue of host and device python bytecode
 	// being the same.
 	var stdLib android.Path
+	var stdLibPkg string
 	var launcher android.Path
-	if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
+	if proptools.BoolDefault(p.properties.Is_internal, false) {
 		stdLib = p.srcsZip
+		stdLibPkg = p.getPkgPath()
 	} else {
 		ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
 			if dep, ok := module.(pythonDependency); ok {
 				stdLib = dep.getPrecompiledSrcsZip()
+				stdLibPkg = dep.getPkgPath()
 			}
 		})
 	}
@@ -636,6 +663,7 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.
 		Description: "Precompile the python sources of " + ctx.ModuleName(),
 		Args: map[string]string{
 			"stdlibZip":     stdLib.String(),
+			"stdlibPkg":     stdLibPkg,
 			"launcher":      launcher.String(),
 			"ldLibraryPath": strings.Join(ldLibraryPath, ":"),
 		},

python/tests/par_test.py

@@ -33,6 +33,8 @@ if fileName.endswith('.pyc'):
 assert_equal("os.path.basename(__file__)", fileName, "par_test.py")
 
 archive = os.path.dirname(__file__)
+major = sys.version_info.major
+minor = sys.version_info.minor
 
 assert_equal("__package__", __package__, "")
 assert_equal("sys.argv[0]", sys.argv[0], archive)
@@ -42,10 +44,11 @@ assert_equal("sys.prefix", sys.prefix, archive)
 assert_equal("__loader__.archive", __loader__.archive, archive)
 assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None)
 
-assert_equal("len(sys.path)", len(sys.path), 3)
+assert_equal("len(sys.path)", len(sys.path), 4)
 assert_equal("sys.path[0]", sys.path[0], archive)
-assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal"))
-assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", "stdlib"))
+assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal", f"python{major}{minor}.zip"))
+assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", f"python{major}.{minor}"))
+assert_equal("sys.path[3]", sys.path[3], os.path.join(archive, "internal", f"python{major}.{minor}", "lib-dynload"))
 
 if os.getenv('ARGTEST', False):
     assert_equal("len(sys.argv)", len(sys.argv), 3)

python/tests/py-cmd_test.py

@@ -55,22 +55,22 @@ assert_equal("sys.exec_prefix", sys.exec_prefix, sys.executable)
 assert_equal("sys.prefix", sys.prefix, sys.executable)
 assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None)
 
-if sys.version_info[0] == 2:
+major = sys.version_info.major
+minor = sys.version_info.minor
+
+if major == 2:
     assert_equal("len(sys.path)", len(sys.path), 4)
     assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__)))
     assert_equal("sys.path[1]", sys.path[1], "/extra")
     assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal"))
     assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib"))
 else:
-    assert_equal("len(sys.path)", len(sys.path), 8)
+    assert_equal("len(sys.path)", len(sys.path), 5)
     assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__)))
     assert_equal("sys.path[1]", sys.path[1], "/extra")
-    assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip'))
-    assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), '..'))
-    assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])))
-    assert_equal("sys.path[5]", sys.path[5], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload'))
-    assert_equal("sys.path[6]", sys.path[6], os.path.join(sys.executable, "internal"))
-    assert_equal("sys.path[7]", sys.path[7], os.path.join(sys.executable, "internal", "stdlib"))
+    assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip'))
+    assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])))
+    assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload'))
 
 if failed:
     sys.exit(1)