<Hermetic> Replace Soong Python bootstrap process with embedded

launcher. For Python2, we bundle embedded launcher as bootstrapper within every .par file. This feature is only enabled for linux_x86_64 for now. We provide a user flag: hermetic_enabled within bp file. By default, Pyhon2 still use classic bootstrapping way to construct .par file and relys on host interpreter. Once embedded_launcher is enabled, launcher will be used to bootstrap .par file and execute user program. For Python3, the launcher will be ready soon, and for now it still relys on classic bootstrapping. Test: Real example is used to test. Bug: b/63018041 Change-Id: I28deba413d8ad3af407595e46f77d663e79a3705
2017-07-12 12:55:28 -07:00
parent ff2abe56da
commit d4e641b6e9
8 changed files with 374 additions and 236 deletions
--- a/python/python.go
+++ b/python/python.go
@@ -24,6 +24,7 @@ import (
 	"strings"

 	"github.com/google/blueprint"
+	"github.com/google/blueprint/proptools"

 	"android/soong/android"
 )
@@ -35,58 +36,70 @@ func init() {
 }

 // the version properties that apply to python libraries and binaries.
-type PythonVersionProperties struct {
+type VersionProperties struct {
 	// true, if the module is required to be built with this version.
-	Enabled *bool
-
-	// if specified, common src files are converted to specific version with converter tool.
-	// Converter bool
+	Enabled *bool `android:"arch_variant"`

 	// non-empty list of .py files under this strict Python version.
 	// srcs may reference the outputs of other modules that produce source files like genrule
 	// or filegroup using the syntax ":module".
-	Srcs []string
+	Srcs []string `android:"arch_variant"`
+
+	// list of source files that should not be used to build the Python module.
+	// This is most useful in the arch/multilib variants to remove non-common files
+	Exclude_srcs []string `android:"arch_variant"`

 	// list of the Python libraries under this Python version.
-	Libs []string
+	Libs []string `android:"arch_variant"`
+
+	// true, if the binary is required to be built with embedded launcher.
+	// TODO(nanzhang): Remove this flag when embedded Python3 is supported later.
+	Embedded_launcher *bool `android:"arch_variant"`
 }

 // properties that apply to python libraries and binaries.
-type PythonBaseModuleProperties struct {
+type BaseProperties struct {
 	// the package path prefix within the output artifact at which to place the source/data
 	// files of the current module.
 	// eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
 	// (from a.b.c import ...) statement.
 	// if left unspecified, all the source/data files of current module are copied to
 	// "runfiles/" tree directory directly.
-	Pkg_path string
+	Pkg_path string `android:"arch_variant"`
+
+	// true, if the Python module is used internally, eg, Python std libs.
+	Is_internal *bool `android:"arch_variant"`

 	// list of source (.py) files compatible both with Python2 and Python3 used to compile the
 	// Python module.
 	// srcs may reference the outputs of other modules that produce source files like genrule
 	// or filegroup using the syntax ":module".
 	// Srcs has to be non-empty.
-	Srcs []string
+	Srcs []string `android:"arch_variant"`
+
+	// list of source files that should not be used to build the C/C++ module.
+	// This is most useful in the arch/multilib variants to remove non-common files
+	Exclude_srcs []string `android:"arch_variant"`

 	// list of files or filegroup modules that provide data that should be installed alongside
 	// the test. the file extension can be arbitrary except for (.py).
-	Data []string
+	Data []string `android:"arch_variant"`

 	// list of the Python libraries compatible both with Python2 and Python3.
-	Libs []string
+	Libs []string `android:"arch_variant"`

 	Version struct {
 		// all the "srcs" or Python dependencies that are to be used only for Python2.
-		Py2 PythonVersionProperties
+		Py2 VersionProperties `android:"arch_variant"`

 		// all the "srcs" or Python dependencies that are to be used only for Python3.
-		Py3 PythonVersionProperties
-	}
+		Py3 VersionProperties `android:"arch_variant"`
+	} `android:"arch_variant"`

 	// the actual version each module uses after variations created.
 	// this property name is hidden from users' perspectives, and soong will populate it during
 	// runtime.
-	ActualVersion string `blueprint:"mutated"`
+	Actual_version string `blueprint:"mutated"`
 }

 type pathMapping struct {
@@ -94,11 +107,21 @@ type pathMapping struct {
 	src  android.Path
 }

-type pythonBaseModule struct {
+type Module struct {
 	android.ModuleBase
-	subModule PythonSubModule

-	properties PythonBaseModuleProperties
+	properties BaseProperties
+
+	// initialize before calling Init
+	hod      android.HostOrDeviceSupported
+	multilib android.Multilib
+
+	// the bootstrapper is used to bootstrap .par executable.
+	// bootstrapper might be nil (Python library module).
+	bootstrapper bootstrapper
+
+	// the installer might be nil.
+	installer installer

 	// the Python files of current module after expanding source dependencies.
 	// pathMapping: <dest: runfile_path, src: source_path>
@@ -108,17 +131,37 @@ type pythonBaseModule struct {
 	// pathMapping: <dest: runfile_path, src: source_path>
 	dataPathMappings []pathMapping

+	// soong_zip arguments of all its dependencies.
+	depsParSpecs []parSpec
+
+	// Python runfiles paths of all its dependencies.
+	depsPyRunfiles []string
+
+	// (.intermediate) module output path as installation source.
+	installSource android.OptionalPath
+
 	// the soong_zip arguments for zipping current module source/data files.
 	parSpec parSpec

-	// the installer might be nil.
-	installer installer
-
 	subAndroidMkOnce map[subAndroidMkProvider]bool
 }

-type PythonSubModule interface {
-	GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath
+func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
+	return &Module{
+		hod:      hod,
+		multilib: multilib,
+	}
+}
+
+type bootstrapper interface {
+	bootstrapperProps() []interface{}
+	bootstrap(ctx android.ModuleContext, Actual_version string, embedded_launcher bool,
+		srcsPathMappings []pathMapping, parSpec parSpec,
+		depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath
+}
+
+type installer interface {
+	install(ctx android.ModuleContext, path android.Path)
 }

 type PythonDependency interface {
@@ -127,64 +170,58 @@ type PythonDependency interface {
 	GetParSpec() parSpec
 }

-type pythonDecorator struct {
-	baseInstaller *pythonInstaller
-}
-
-type installer interface {
-	install(ctx android.ModuleContext, path android.Path)
-}
-
-func (p *pythonBaseModule) GetSrcsPathMappings() []pathMapping {
+func (p *Module) GetSrcsPathMappings() []pathMapping {
 	return p.srcsPathMappings
 }

-func (p *pythonBaseModule) GetDataPathMappings() []pathMapping {
+func (p *Module) GetDataPathMappings() []pathMapping {
 	return p.dataPathMappings
 }

-func (p *pythonBaseModule) GetParSpec() parSpec {
+func (p *Module) GetParSpec() parSpec {
 	return p.parSpec
 }

-var _ PythonDependency = (*pythonBaseModule)(nil)
+var _ PythonDependency = (*Module)(nil)

-var _ android.AndroidMkDataProvider = (*pythonBaseModule)(nil)
+var _ android.AndroidMkDataProvider = (*Module)(nil)

-func InitPythonBaseModule(baseModule *pythonBaseModule, subModule PythonSubModule,
-	hod android.HostOrDeviceSupported) android.Module {
+func (p *Module) Init() android.Module {

-	baseModule.subModule = subModule
+	p.AddProperties(&p.properties)
+	if p.bootstrapper != nil {
+		p.AddProperties(p.bootstrapper.bootstrapperProps()...)
+	}

-	baseModule.AddProperties(&baseModule.properties)
+	android.InitAndroidArchModule(p, p.hod, p.multilib)

-	android.InitAndroidArchModule(baseModule, hod, android.MultilibCommon)
-
-	return baseModule
+	return p
 }

-// the tag used to mark dependencies within "py_libs" attribute.
-type pythonDependencyTag struct {
+type dependencyTag struct {
 	blueprint.BaseDependencyTag
+	name string
 }

-var pyDependencyTag pythonDependencyTag
-
 var (
+	pythonLibTag       = dependencyTag{name: "pythonLib"}
+	launcherTag        = dependencyTag{name: "launcher"}
 	pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`)
 	pyExt              = ".py"
 	pyVersion2         = "PY2"
 	pyVersion3         = "PY3"
 	initFileName       = "__init__.py"
 	mainFileName       = "__main__.py"
+	entryPointFile     = "entry_point.txt"
 	parFileExt         = ".zip"
 	runFiles           = "runfiles"
+	internal           = "internal"
 )

 // create version variants for modules.
 func versionSplitMutator() func(android.BottomUpMutatorContext) {
 	return func(mctx android.BottomUpMutatorContext) {
-		if base, ok := mctx.Module().(*pythonBaseModule); ok {
+		if base, ok := mctx.Module().(*Module); ok {
 			versionNames := []string{}
 			if base.properties.Version.Py2.Enabled != nil &&
 				*(base.properties.Version.Py2.Enabled) == true {
@@ -197,36 +234,61 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) {
 			modules := mctx.CreateVariations(versionNames...)
 			for i, v := range versionNames {
 				// set the actual version for Python module.
-				modules[i].(*pythonBaseModule).properties.ActualVersion = v
+				modules[i].(*Module).properties.Actual_version = v
 			}
 		}
 	}
 }

-func (p *pythonBaseModule) DepsMutator(ctx android.BottomUpMutatorContext) {
+func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool {
+	switch actual_version {
+	case pyVersion2:
+		return proptools.Bool(p.properties.Version.Py2.Embedded_launcher)
+	case pyVersion3:
+		return proptools.Bool(p.properties.Version.Py3.Embedded_launcher)
+	}
+
+	return false
+}
+
+func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
 	// deps from "data".
 	android.ExtractSourcesDeps(ctx, p.properties.Data)
 	// deps from "srcs".
 	android.ExtractSourcesDeps(ctx, p.properties.Srcs)

-	switch p.properties.ActualVersion {
+	switch p.properties.Actual_version {
 	case pyVersion2:
 		// deps from "version.py2.srcs" property.
 		android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs)

-		ctx.AddVariationDependencies(nil, pyDependencyTag,
+		ctx.AddVariationDependencies(nil, pythonLibTag,
 			uniqueLibs(ctx, p.properties.Libs, "version.py2.libs",
 				p.properties.Version.Py2.Libs)...)
+
+		if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) {
+			ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib")
+			ctx.AddFarVariationDependencies([]blueprint.Variation{
+				{"arch", ctx.Target().String()},
+			}, launcherTag, "py2-launcher")
+		}
+
 	case pyVersion3:
 		// deps from "version.py3.srcs" property.
 		android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs)

-		ctx.AddVariationDependencies(nil, pyDependencyTag,
+		ctx.AddVariationDependencies(nil, pythonLibTag,
 			uniqueLibs(ctx, p.properties.Libs, "version.py3.libs",
 				p.properties.Version.Py3.Libs)...)
+
+		if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion3) {
+			//TODO(nanzhang): Add embedded launcher for Python3.
+			ctx.PropertyErrorf("version.py3.embedded_launcher",
+				"is not supported yet for Python3.")
+		}
 	default:
-		panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
-			p.properties.ActualVersion, ctx.ModuleName()))
+		panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+			p.properties.Actual_version, ctx.ModuleName()))
 	}
 }

@@ -258,27 +320,43 @@ func uniqueLibs(ctx android.BottomUpMutatorContext,
 	return ret
 }

-func (p *pythonBaseModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
-	installSource := p.subModule.GeneratePythonBuildActions(ctx)
+func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+	p.GeneratePythonBuildActions(ctx)

-	if p.installer != nil && installSource.Valid() {
-		p.installer.install(ctx, installSource.Path())
+	if p.bootstrapper != nil {
+		// TODO(nanzhang): Since embedded launcher is not supported for Python3 for now,
+		// so we initialize "embedded_launcher" to false.
+		embedded_launcher := false
+		if p.properties.Actual_version == pyVersion2 {
+			embedded_launcher = p.isEmbeddedLauncherEnabled(pyVersion2)
+		}
+		p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
+			embedded_launcher, p.srcsPathMappings, p.parSpec, p.depsPyRunfiles,
+			p.depsParSpecs)
 	}
+
+	if p.installer != nil && p.installSource.Valid() {
+		p.installer.install(ctx, p.installSource.Path())
+	}
+
 }

-func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath {
+func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) {
 	// expand python files from "srcs" property.
 	srcs := p.properties.Srcs
-	switch p.properties.ActualVersion {
+	exclude_srcs := p.properties.Exclude_srcs
+	switch p.properties.Actual_version {
 	case pyVersion2:
 		srcs = append(srcs, p.properties.Version.Py2.Srcs...)
+		exclude_srcs = append(exclude_srcs, p.properties.Version.Py2.Exclude_srcs...)
 	case pyVersion3:
 		srcs = append(srcs, p.properties.Version.Py3.Srcs...)
+		exclude_srcs = append(exclude_srcs, p.properties.Version.Py3.Exclude_srcs...)
 	default:
-		panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
-			p.properties.ActualVersion, ctx.ModuleName()))
+		panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+			p.properties.Actual_version, ctx.ModuleName()))
 	}
-	expandedSrcs := ctx.ExpandSources(srcs, nil)
+	expandedSrcs := ctx.ExpandSources(srcs, exclude_srcs)
 	if len(expandedSrcs) == 0 {
 		ctx.ModuleErrorf("doesn't have any source files!")
 	}
@@ -292,15 +370,26 @@ func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext)
 		pkg_path = filepath.Clean(p.properties.Pkg_path)
 		if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") ||
 			strings.HasPrefix(pkg_path, "/") {
-			ctx.PropertyErrorf("pkg_path", "%q is not a valid format.",
+			ctx.PropertyErrorf("pkg_path",
+				"%q must be a relative path contained in par file.",
 				p.properties.Pkg_path)
-			return android.OptionalPath{}
+			return
+		}
+		if p.properties.Is_internal != nil && *p.properties.Is_internal {
+			// pkg_path starts from "internal/" implicitly.
+			pkg_path = filepath.Join(internal, pkg_path)
+		} else {
+			// pkg_path starts from "runfiles/" implicitly.
+			pkg_path = filepath.Join(runFiles, pkg_path)
 		}
-		// pkg_path starts from "runfiles/" implicitly.
-		pkg_path = filepath.Join(runFiles, pkg_path)
 	} else {
-		// pkg_path starts from "runfiles/" implicitly.
-		pkg_path = runFiles
+		if p.properties.Is_internal != nil && *p.properties.Is_internal {
+			// pkg_path starts from "runfiles/" implicitly.
+			pkg_path = internal
+		} else {
+			// pkg_path starts from "runfiles/" implicitly.
+			pkg_path = runFiles
+		}
 	}

 	p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData)
@@ -308,13 +397,11 @@ func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext)
 	p.parSpec = p.dumpFileList(ctx, pkg_path)

 	p.uniqWholeRunfilesTree(ctx)
-
-	return android.OptionalPath{}
 }

 // generate current module unique pathMappings: <dest: runfiles_path, src: source_path>
 // for python/data files.
-func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
+func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
 	expandedSrcs, expandedData android.Paths) {
 	// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
 	// check duplicates.
@@ -355,7 +442,7 @@ func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_
 }

 // register build actions to dump filelist to disk.
-func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
+func (p *Module) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
 	relativeRootMap := make(map[string]android.Paths)
 	// the soong_zip params in order to pack current module's Python/data files.
 	ret := parSpec{rootPrefix: pkg_path}
@@ -365,7 +452,8 @@ func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path stri
 	// "srcs" or "data" properties may have filegroup so it might happen that
 	// the relative root for each source path is different.
 	for _, path := range pathMappings {
-		relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
+		var relativeRoot string
+		relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
 		if v, found := relativeRootMap[relativeRoot]; found {
 			relativeRootMap[relativeRoot] = append(v, path.src)
 		} else {
@@ -392,8 +480,19 @@ func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path stri
 	return ret
 }

-// check Python/data files duplicates from current module and its whole dependencies.
-func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
+func isPythonLibModule(module blueprint.Module) bool {
+	if m, ok := module.(*Module); ok {
+		// Python library has no bootstrapper or installer.
+		if m.bootstrapper != nil || m.installer != nil {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// check Python source/data files duplicates from current module and its whole dependencies.
+func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) {
 	// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
 	// check duplicates.
 	destToPySrcs := make(map[string]string)
@@ -408,16 +507,15 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {

 	// visit all its dependencies in depth first.
 	ctx.VisitDepsDepthFirst(func(module blueprint.Module) {
-		// module can only depend on Python library.
-		if base, ok := module.(*pythonBaseModule); ok {
-			if _, ok := base.subModule.(*PythonLibrary); !ok {
-				panic(fmt.Errorf(
-					"the dependency %q of module %q is not Python library!",
-					ctx.ModuleName(), ctx.OtherModuleName(module)))
-			}
-		} else {
+		if ctx.OtherModuleDependencyTag(module) != pythonLibTag {
 			return
 		}
+		// Python module cannot depend on modules, except for Python library.
+		if !isPythonLibModule(module) {
+			panic(fmt.Errorf(
+				"the dependency %q of module %q is not Python library!",
+				ctx.ModuleName(), ctx.OtherModuleName(module)))
+		}
 		if dep, ok := module.(PythonDependency); ok {
 			srcs := dep.GetSrcsPathMappings()
 			for _, path := range srcs {
@@ -428,9 +526,7 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
 				}
 				// binary needs the Python runfiles paths from all its
 				// dependencies to fill __init__.py in each runfiles dir.
-				if sub, ok := p.subModule.(*pythonBinaryBase); ok {
-					sub.depsPyRunfiles = append(sub.depsPyRunfiles, path.dest)
-				}
+				p.depsPyRunfiles = append(p.depsPyRunfiles, path.dest)
 			}
 			data := dep.GetDataPathMappings()
 			for _, path := range data {
@@ -440,9 +536,7 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
 			}
 			// binary needs the soong_zip arguments from all its
 			// dependencies to generate executable par file.
-			if sub, ok := p.subModule.(*pythonBinaryBase); ok {
-				sub.depsParSpecs = append(sub.depsParSpecs, dep.GetParSpec())
-			}
+			p.depsParSpecs = append(p.depsParSpecs, dep.GetParSpec())
 		}
 	})
 }