From 339a63f04469ebf69a0b3ca729243eefd838a572 Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Tue, 15 Aug 2023 22:17:03 -0400 Subject: [PATCH] Prepare soong for python 3.11 Due to upstream changes in python path calculations, I'm simplifying the python launcher to work more like a standard python distribution. This is effectively changing the stdlib path from `internal/stdlib` to `internal/python3.10` (or `3.11`, etc). This allows us to specify the zip file as PYTHONHOME, set PYTHONPLATLIBDIR to `internal` and use the default detection after that. That does mean during upgrades that the stdlib pkg path will change, so move the source vs prebuilt calculation from the Android.bp into Soong to choose which stdlib module to pick up (with the corresponding `pkg_path`) Bug: 278602456 Test: treehugger with python3.10 Test: a python3.11 source + 3.10 prebuilt build Test: a python3.11 source+prebuilt build Change-Id: I8b02e7b22a1f1d1e02819ae1a31a99cdc985542c --- python/builder.go | 4 ++-- python/python.go | 36 ++++++++++++++++++++++++++++++++---- python/tests/par_test.py | 9 ++++++--- python/tests/py-cmd_test.py | 16 ++++++++-------- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/python/builder.go b/python/builder.go index 106649398..2553a7714 100644 --- a/python/builder.go +++ b/python/builder.go @@ -73,14 +73,14 @@ var ( precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{ Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` + - `PYTHONPATH=$stdlibZip/internal/stdlib ` + + `PYTHONPATH=$stdlibZip/internal/$stdlibPkg ` + `$launcher build/soong/python/scripts/precompile_python.py $in $out`, CommandDeps: []string{ "$stdlibZip", "$launcher", "build/soong/python/scripts/precompile_python.py", }, - }, "stdlibZip", "launcher", "ldLibraryPath") + }, "stdlibZip", "stdlibPkg", "launcher", "ldLibraryPath") ) func init() { diff --git a/python/python.go b/python/python.go index 8fde638b4..6c837a888 100644 --- a/python/python.go +++ b/python/python.go @@ -169,6 +169,7 @@ type pythonDependency interface { getDataPathMappings() []pathMapping getSrcsZip() android.Path getPrecompiledSrcsZip() android.Path + getPkgPath() string } // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination @@ -191,6 +192,11 @@ func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path { return p.precompiledSrcsZip } +// getPkgPath returns the pkg_path value +func (p *PythonLibraryModule) getPkgPath() string { + return String(p.properties.Pkg_path) +} + func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { return &p.properties } @@ -370,7 +376,20 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") case pyVersion3: - stdLib = "py3-stdlib" + var prebuiltStdLib bool + if targetForDeps.Os.Bionic() { + prebuiltStdLib = false + } else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") { + prebuiltStdLib = false + } else { + prebuiltStdLib = true + } + + if prebuiltStdLib { + stdLib = "py3-stdlib-prebuilt" + } else { + stdLib = "py3-stdlib" + } launcherModule = "py3-launcher" if autorun { @@ -461,14 +480,19 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p destToPySrcs := make(map[string]string) destToPyData := make(map[string]string) + // Disable path checks for the stdlib, as it includes a "." in the version string + isInternal := proptools.BoolDefault(p.properties.Is_internal, false) + for _, s := range expandedSrcs { if s.Ext() != pyExt && s.Ext() != protoExt { ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String()) continue } runfilesPath := filepath.Join(pkgPath, s.Rel()) - if err := isValidPythonPath(runfilesPath); err != nil { - ctx.PropertyErrorf("srcs", err.Error()) + if !isInternal { + if err := isValidPythonPath(runfilesPath); err != nil { + ctx.PropertyErrorf("srcs", err.Error()) + } } if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) { p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s}) @@ -591,13 +615,16 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android. // "cross compiling" for device here purely by virtue of host and device python bytecode // being the same. var stdLib android.Path + var stdLibPkg string var launcher android.Path - if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" { + if proptools.BoolDefault(p.properties.Is_internal, false) { stdLib = p.srcsZip + stdLibPkg = p.getPkgPath() } else { ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) { if dep, ok := module.(pythonDependency); ok { stdLib = dep.getPrecompiledSrcsZip() + stdLibPkg = dep.getPkgPath() } }) } @@ -636,6 +663,7 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android. Description: "Precompile the python sources of " + ctx.ModuleName(), Args: map[string]string{ "stdlibZip": stdLib.String(), + "stdlibPkg": stdLibPkg, "launcher": launcher.String(), "ldLibraryPath": strings.Join(ldLibraryPath, ":"), }, diff --git a/python/tests/par_test.py b/python/tests/par_test.py index 1e03f1669..96b42ae83 100644 --- a/python/tests/par_test.py +++ b/python/tests/par_test.py @@ -33,6 +33,8 @@ if fileName.endswith('.pyc'): assert_equal("os.path.basename(__file__)", fileName, "par_test.py") archive = os.path.dirname(__file__) +major = sys.version_info.major +minor = sys.version_info.minor assert_equal("__package__", __package__, "") assert_equal("sys.argv[0]", sys.argv[0], archive) @@ -42,10 +44,11 @@ assert_equal("sys.prefix", sys.prefix, archive) assert_equal("__loader__.archive", __loader__.archive, archive) assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None) -assert_equal("len(sys.path)", len(sys.path), 3) +assert_equal("len(sys.path)", len(sys.path), 4) assert_equal("sys.path[0]", sys.path[0], archive) -assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal")) -assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", "stdlib")) +assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal", f"python{major}{minor}.zip")) +assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", f"python{major}.{minor}")) +assert_equal("sys.path[3]", sys.path[3], os.path.join(archive, "internal", f"python{major}.{minor}", "lib-dynload")) if os.getenv('ARGTEST', False): assert_equal("len(sys.argv)", len(sys.argv), 3) diff --git a/python/tests/py-cmd_test.py b/python/tests/py-cmd_test.py index c7ba0ab4b..8aed78289 100644 --- a/python/tests/py-cmd_test.py +++ b/python/tests/py-cmd_test.py @@ -55,22 +55,22 @@ assert_equal("sys.exec_prefix", sys.exec_prefix, sys.executable) assert_equal("sys.prefix", sys.prefix, sys.executable) assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None) -if sys.version_info[0] == 2: +major = sys.version_info.major +minor = sys.version_info.minor + +if major == 2: assert_equal("len(sys.path)", len(sys.path), 4) assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__))) assert_equal("sys.path[1]", sys.path[1], "/extra") assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal")) assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib")) else: - assert_equal("len(sys.path)", len(sys.path), 8) + assert_equal("len(sys.path)", len(sys.path), 5) assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__))) assert_equal("sys.path[1]", sys.path[1], "/extra") - assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip')) - assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), '..')) - assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]))) - assert_equal("sys.path[5]", sys.path[5], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload')) - assert_equal("sys.path[6]", sys.path[6], os.path.join(sys.executable, "internal")) - assert_equal("sys.path[7]", sys.path[7], os.path.join(sys.executable, "internal", "stdlib")) + assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip')) + assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]))) + assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload')) if failed: sys.exit(1)