Merge "Prepare soong for python 3.11" into main

python/builder.go

@@ -73,14 +73,14 @@ var (
 
 	precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{
 		Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` +
-			`PYTHONPATH=$stdlibZip/internal/stdlib ` +
+			`PYTHONPATH=$stdlibZip/internal/$stdlibPkg ` +
 			`$launcher build/soong/python/scripts/precompile_python.py $in $out`,
 		CommandDeps: []string{
 			"$stdlibZip",
 			"$launcher",
 			"build/soong/python/scripts/precompile_python.py",
 		},
-	}, "stdlibZip", "launcher", "ldLibraryPath")
+	}, "stdlibZip", "stdlibPkg", "launcher", "ldLibraryPath")
 )
 
 func init() {

python/python.go

@@ -169,6 +169,7 @@ type pythonDependency interface {
 	getDataPathMappings() []pathMapping
 	getSrcsZip() android.Path
 	getPrecompiledSrcsZip() android.Path
+	getPkgPath() string
 }
 
 // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
@@ -191,6 +192,11 @@ func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
 	return p.precompiledSrcsZip
 }
 
+// getPkgPath returns the pkg_path value
+func (p *PythonLibraryModule) getPkgPath() string {
+	return String(p.properties.Pkg_path)
+}
+
 func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
 	return &p.properties
 }
@@ -370,7 +376,20 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto
 
 		launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
 	case pyVersion3:
-		stdLib = "py3-stdlib"
+		var prebuiltStdLib bool
+		if targetForDeps.Os.Bionic() {
+			prebuiltStdLib = false
+		} else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") {
+			prebuiltStdLib = false
+		} else {
+			prebuiltStdLib = true
+		}
+
+		if prebuiltStdLib {
+			stdLib = "py3-stdlib-prebuilt"
+		} else {
+			stdLib = "py3-stdlib"
+		}
 
 		launcherModule = "py3-launcher"
 		if autorun {
@@ -461,14 +480,19 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p
 	destToPySrcs := make(map[string]string)
 	destToPyData := make(map[string]string)
 
+	// Disable path checks for the stdlib, as it includes a "." in the version string
+	isInternal := proptools.BoolDefault(p.properties.Is_internal, false)
+
 	for _, s := range expandedSrcs {
 		if s.Ext() != pyExt && s.Ext() != protoExt {
 			ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
 			continue
 		}
 		runfilesPath := filepath.Join(pkgPath, s.Rel())
-		if err := isValidPythonPath(runfilesPath); err != nil {
+		if !isInternal {
-			ctx.PropertyErrorf("srcs", err.Error())
+			if err := isValidPythonPath(runfilesPath); err != nil {
+				ctx.PropertyErrorf("srcs", err.Error())
+			}
 		}
 		if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
 			p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
@@ -591,13 +615,16 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.
 	// "cross compiling" for device here purely by virtue of host and device python bytecode
 	// being the same.
 	var stdLib android.Path
+	var stdLibPkg string
 	var launcher android.Path
-	if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
+	if proptools.BoolDefault(p.properties.Is_internal, false) {
 		stdLib = p.srcsZip
+		stdLibPkg = p.getPkgPath()
 	} else {
 		ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
 			if dep, ok := module.(pythonDependency); ok {
 				stdLib = dep.getPrecompiledSrcsZip()
+				stdLibPkg = dep.getPkgPath()
 			}
 		})
 	}
@@ -636,6 +663,7 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.
 		Description: "Precompile the python sources of " + ctx.ModuleName(),
 		Args: map[string]string{
 			"stdlibZip":     stdLib.String(),
+			"stdlibPkg":     stdLibPkg,
 			"launcher":      launcher.String(),
 			"ldLibraryPath": strings.Join(ldLibraryPath, ":"),
 		},

python/tests/par_test.py

@@ -33,6 +33,8 @@ if fileName.endswith('.pyc'):
 assert_equal("os.path.basename(__file__)", fileName, "par_test.py")
 
 archive = os.path.dirname(__file__)
+major = sys.version_info.major
+minor = sys.version_info.minor
 
 assert_equal("__package__", __package__, "")
 assert_equal("sys.argv[0]", sys.argv[0], archive)
@@ -42,10 +44,11 @@ assert_equal("sys.prefix", sys.prefix, archive)
 assert_equal("__loader__.archive", __loader__.archive, archive)
 assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None)
 
-assert_equal("len(sys.path)", len(sys.path), 3)
+assert_equal("len(sys.path)", len(sys.path), 4)
 assert_equal("sys.path[0]", sys.path[0], archive)
-assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal"))
+assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal", f"python{major}{minor}.zip"))
-assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", "stdlib"))
+assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", f"python{major}.{minor}"))
+assert_equal("sys.path[3]", sys.path[3], os.path.join(archive, "internal", f"python{major}.{minor}", "lib-dynload"))
 
 if os.getenv('ARGTEST', False):
     assert_equal("len(sys.argv)", len(sys.argv), 3)

python/tests/py-cmd_test.py

@@ -55,22 +55,22 @@ assert_equal("sys.exec_prefix", sys.exec_prefix, sys.executable)
 assert_equal("sys.prefix", sys.prefix, sys.executable)
 assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None)
 
-if sys.version_info[0] == 2:
+major = sys.version_info.major
+minor = sys.version_info.minor
+
+if major == 2:
     assert_equal("len(sys.path)", len(sys.path), 4)
     assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__)))
     assert_equal("sys.path[1]", sys.path[1], "/extra")
     assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal"))
     assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib"))
 else:
-    assert_equal("len(sys.path)", len(sys.path), 8)
+    assert_equal("len(sys.path)", len(sys.path), 5)
     assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__)))
     assert_equal("sys.path[1]", sys.path[1], "/extra")
-    assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip'))
+    assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip'))
-    assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), '..'))
+    assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])))
-    assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])))
+    assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload'))
-    assert_equal("sys.path[5]", sys.path[5], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload'))
-    assert_equal("sys.path[6]", sys.path[6], os.path.join(sys.executable, "internal"))
-    assert_equal("sys.path[7]", sys.path[7], os.path.join(sys.executable, "internal", "stdlib"))
 
 if failed:
     sys.exit(1)