<Hermetic> Replace Soong Python bootstrap process with embedded

launcher. For Python2, we bundle embedded launcher as bootstrapper within every .par file. This feature is only enabled for linux_x86_64 for now. We provide a user flag: hermetic_enabled within bp file. By default, Pyhon2 still use classic bootstrapping way to construct .par file and relys on host interpreter. Once embedded_launcher is enabled, launcher will be used to bootstrap .par file and execute user program. For Python3, the launcher will be ready soon, and for now it still relys on classic bootstrapping. Test: Real example is used to test. Bug: b/63018041 Change-Id: I28deba413d8ad3af407595e46f77d663e79a3705
2017-07-12 12:55:28 -07:00
parent ff2abe56da
commit d4e641b6e9
8 changed files with 374 additions and 236 deletions
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -1173,6 +1173,10 @@ func (c *Module) HostToolPath() android.OptionalPath {
 	return c.installer.hostToolPath()
 }

+func (c *Module) IntermPathForModuleOut() android.OptionalPath {
+	return c.outputFile
+}
+
 //
 // Defaults
 //
--- a/python/androidmk.go
+++ b/python/androidmk.go
@@ -23,10 +23,10 @@ import (
 )

 type subAndroidMkProvider interface {
-	AndroidMk(*pythonBaseModule, *android.AndroidMkData)
+	AndroidMk(*Module, *android.AndroidMkData)
 }

-func (p *pythonBaseModule) subAndroidMk(data *android.AndroidMkData, obj interface{}) {
+func (p *Module) subAndroidMk(data *android.AndroidMkData, obj interface{}) {
 	if p.subAndroidMkOnce == nil {
 		p.subAndroidMkOnce = make(map[subAndroidMkProvider]bool)
 	}
@@ -38,7 +38,7 @@ func (p *pythonBaseModule) subAndroidMk(data *android.AndroidMkData, obj interfa
 	}
 }

-func (p *pythonBaseModule) AndroidMk() android.AndroidMkData {
+func (p *Module) AndroidMk() android.AndroidMkData {
 	ret := android.AndroidMkData{}

 	p.subAndroidMk(&ret, p.installer)
@@ -46,17 +46,17 @@ func (p *pythonBaseModule) AndroidMk() android.AndroidMkData {
 	return ret
 }

-func (p *pythonBinaryHostDecorator) AndroidMk(base *pythonBaseModule, ret *android.AndroidMkData) {
+func (p *binaryDecorator) AndroidMk(base *Module, ret *android.AndroidMkData) {
 	ret.Class = "EXECUTABLES"
-	base.subAndroidMk(ret, p.pythonDecorator.baseInstaller)
+	base.subAndroidMk(ret, p.baseInstaller)
 }

-func (p *pythonTestHostDecorator) AndroidMk(base *pythonBaseModule, ret *android.AndroidMkData) {
+func (p *testDecorator) AndroidMk(base *Module, ret *android.AndroidMkData) {
 	ret.Class = "NATIVE_TESTS"
-	base.subAndroidMk(ret, p.pythonDecorator.baseInstaller)
+	base.subAndroidMk(ret, p.binaryDecorator.baseInstaller)
 }

-func (installer *pythonInstaller) AndroidMk(base *pythonBaseModule, ret *android.AndroidMkData) {
+func (installer *pythonInstaller) AndroidMk(base *Module, ret *android.AndroidMkData) {
 	// Soong installation is only supported for host modules. Have Make
 	// installation trigger Soong installation.
 	if base.Target().Os.Class == android.Host {
--- a/python/binary.go
+++ b/python/binary.go
@@ -21,6 +21,8 @@ import (
 	"path/filepath"
 	"strings"

+	"github.com/google/blueprint"
+
 	"android/soong/android"
 )

@@ -28,67 +30,63 @@ func init() {
 	android.RegisterModuleType("python_binary_host", PythonBinaryHostFactory)
 }

-type PythonBinaryBaseProperties struct {
+type BinaryProperties struct {
 	// the name of the source file that is the main entry point of the program.
 	// this file must also be listed in srcs.
 	// If left unspecified, module name is used instead.
 	// If name doesn’t match any filename in srcs, main must be specified.
-	Main string
+	Main string `android:"arch_variant"`

 	// set the name of the output binary.
-	Stem string
+	Stem string `android:"arch_variant"`

 	// append to the name of the output binary.
-	Suffix string
+	Suffix string `android:"arch_variant"`
 }

-type pythonBinaryBase struct {
-	pythonBaseModule
+type binaryDecorator struct {
+	binaryProperties BinaryProperties

-	binaryProperties PythonBinaryBaseProperties
-
-	// soong_zip arguments from all its dependencies.
-	depsParSpecs []parSpec
-
-	// Python runfiles paths from all its dependencies.
-	depsPyRunfiles []string
+	baseInstaller *pythonInstaller
 }

-type PythonBinaryHost struct {
-	pythonBinaryBase
+type IntermPathProvider interface {
+	IntermPathForModuleOut() android.OptionalPath
 }

-var _ PythonSubModule = (*PythonBinaryHost)(nil)
-
-type pythonBinaryHostDecorator struct {
-	pythonDecorator
-}
-
-func (p *pythonBinaryHostDecorator) install(ctx android.ModuleContext, file android.Path) {
-	p.pythonDecorator.baseInstaller.install(ctx, file)
+func (binary *binaryDecorator) install(ctx android.ModuleContext, file android.Path) {
+	binary.baseInstaller.install(ctx, file)
 }

 var (
 	stubTemplateHost = "build/soong/python/scripts/stub_template_host.txt"
 )

-func PythonBinaryHostFactory() android.Module {
-	decorator := &pythonBinaryHostDecorator{
-		pythonDecorator: pythonDecorator{baseInstaller: NewPythonInstaller("bin")}}
+func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) {
+	module := newModule(hod, android.MultilibFirst)
+	decorator := &binaryDecorator{baseInstaller: NewPythonInstaller("bin")}

-	module := &PythonBinaryHost{}
-	module.pythonBaseModule.installer = decorator
-	module.AddProperties(&module.binaryProperties)
+	module.bootstrapper = decorator
+	module.installer = decorator

-	return InitPythonBaseModule(&module.pythonBinaryBase.pythonBaseModule,
-		&module.pythonBinaryBase, android.HostSupportedNoCross)
+	return module, decorator
 }

-func (p *pythonBinaryBase) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath {
-	p.pythonBaseModule.GeneratePythonBuildActions(ctx)
+func PythonBinaryHostFactory() android.Module {
+	module, _ := NewBinary(android.HostSupportedNoCross)

-	// no Python source file for compiling par file.
-	if len(p.pythonBaseModule.srcsPathMappings) == 0 && len(p.depsPyRunfiles) == 0 {
+	return module.Init()
+}
+
+func (binary *binaryDecorator) bootstrapperProps() []interface{} {
+	return []interface{}{&binary.binaryProperties}
+}
+
+func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actual_version string,
+	embedded_launcher bool, srcsPathMappings []pathMapping, parSpec parSpec,
+	depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath {
+	// no Python source file for compiling .par file.
+	if len(srcsPathMappings) == 0 {
 		return android.OptionalPath{}
 	}

@@ -100,10 +98,10 @@ func (p *pythonBinaryBase) GeneratePythonBuildActions(ctx android.ModuleContext)
 	existingPyPkgSet := make(map[string]bool)

 	wholePyRunfiles := []string{}
-	for _, path := range p.pythonBaseModule.srcsPathMappings {
+	for _, path := range srcsPathMappings {
 		wholePyRunfiles = append(wholePyRunfiles, path.dest)
 	}
-	wholePyRunfiles = append(wholePyRunfiles, p.depsPyRunfiles...)
+	wholePyRunfiles = append(wholePyRunfiles, depsPyRunfiles...)

 	// find all the runfiles dirs which have been treated as packages.
 	for _, path := range wholePyRunfiles {
@@ -130,50 +128,62 @@ func (p *pythonBinaryBase) GeneratePythonBuildActions(ctx android.ModuleContext)
 		populateNewPyPkgs(parentPath, existingPyPkgSet, newPyPkgSet, &newPyPkgs)
 	}

-	main := p.getPyMainFile(ctx)
+	main := binary.getPyMainFile(ctx, srcsPathMappings)
 	if main == "" {
 		return android.OptionalPath{}
 	}
-	interp := p.getInterpreter(ctx)
-	if interp == "" {
-		return android.OptionalPath{}
+
+	var launcher_path android.Path
+	if embedded_launcher {
+		ctx.VisitDirectDeps(func(m blueprint.Module) {
+			if ctx.OtherModuleDependencyTag(m) != launcherTag {
+				return
+			}
+			if provider, ok := m.(IntermPathProvider); ok {
+				if launcher_path != nil {
+					panic(fmt.Errorf("launcher path was found before: %q",
+						launcher_path))
+				}
+				launcher_path = provider.IntermPathForModuleOut().Path()
+			}
+		})
 	}

-	// we need remove "runfiles/" suffix since stub script starts
-	// searching for main file in each sub-dir of "runfiles" directory tree.
-	binFile := registerBuildActionForParFile(ctx, p.getInterpreter(ctx),
-		strings.TrimPrefix(main, runFiles+"/"), p.getStem(ctx),
-		newPyPkgs, append(p.depsParSpecs, p.pythonBaseModule.parSpec))
+	binFile := registerBuildActionForParFile(ctx, embedded_launcher, launcher_path,
+		binary.getHostInterpreterName(ctx, actual_version),
+		main, binary.getStem(ctx), newPyPkgs, append(depsParSpecs, parSpec))

 	return android.OptionalPathForPath(binFile)
 }

-// get interpreter path.
-func (p *pythonBinaryBase) getInterpreter(ctx android.ModuleContext) string {
+// get host interpreter name.
+func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext,
+	actual_version string) string {
 	var interp string
-	switch p.pythonBaseModule.properties.ActualVersion {
+	switch actual_version {
 	case pyVersion2:
 		interp = "python2"
 	case pyVersion3:
 		interp = "python3"
 	default:
 		panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
-			p.properties.ActualVersion, ctx.ModuleName()))
+			actual_version, ctx.ModuleName()))
 	}

 	return interp
 }

 // find main program path within runfiles tree.
-func (p *pythonBinaryBase) getPyMainFile(ctx android.ModuleContext) string {
+func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext,
+	srcsPathMappings []pathMapping) string {
 	var main string
-	if p.binaryProperties.Main == "" {
-		main = p.BaseModuleName() + pyExt
+	if binary.binaryProperties.Main == "" {
+		main = ctx.ModuleName() + pyExt
 	} else {
-		main = p.binaryProperties.Main
+		main = binary.binaryProperties.Main
 	}

-	for _, path := range p.pythonBaseModule.srcsPathMappings {
+	for _, path := range srcsPathMappings {
 		if main == path.src.Rel() {
 			return path.dest
 		}
@@ -183,13 +193,13 @@ func (p *pythonBinaryBase) getPyMainFile(ctx android.ModuleContext) string {
 	return ""
 }

-func (p *pythonBinaryBase) getStem(ctx android.ModuleContext) string {
+func (binary *binaryDecorator) getStem(ctx android.ModuleContext) string {
 	stem := ctx.ModuleName()
-	if p.binaryProperties.Stem != "" {
-		stem = p.binaryProperties.Stem
+	if binary.binaryProperties.Stem != "" {
+		stem = binary.binaryProperties.Stem
 	}

-	return stem + p.binaryProperties.Suffix
+	return stem + binary.binaryProperties.Suffix
 }

 // Sets the given directory and all its ancestor directories as Python packages.
--- a/python/builder.go
+++ b/python/builder.go
@@ -17,6 +17,7 @@ package python
 // This file contains Ninja build actions for building Python program.

 import (
+	"fmt"
 	"strings"

 	"android/soong/android"
@@ -28,7 +29,7 @@ import (
 var (
 	pctx = android.NewPackageContext("android/soong/python")

-	par = pctx.AndroidStaticRule("par",
+	host_par = pctx.AndroidStaticRule("host_par",
 		blueprint.RuleParams{
 			Command: `touch $initFile && ` +
 				`sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` +
@@ -37,6 +38,16 @@ var (
 			CommandDeps: []string{"$parCmd", "$template"},
 		},
 		"initFile", "interp", "main", "template", "stub", "parCmd", "parFile", "parArgs")
+
+	embedded_par = pctx.AndroidStaticRule("embedded_par",
+		blueprint.RuleParams{
+			Command: `touch $initFile && ` +
+				`echo '$main' > $entry_point && ` +
+				`$parCmd -o $parFile $parArgs && cat $launcher | cat - $parFile > $out && ` +
+				`chmod +x $out && (rm -f $initFile; rm -f $entry_point; rm -f $parFile)`,
+			CommandDeps: []string{"$parCmd"},
+		},
+		"initFile", "main", "entry_point", "parCmd", "parFile", "parArgs", "launcher")
 )

 func init() {
@@ -58,10 +69,10 @@ type parSpec struct {
 }

 func (p parSpec) soongParArgs() string {
-	ret := "-P " + p.rootPrefix
+	ret := `-P ` + p.rootPrefix

 	for _, spec := range p.fileListSpecs {
-		ret += " -C " + spec.relativeRoot + " -l " + spec.fileList.String()
+		ret += ` -C ` + spec.relativeRoot + ` -l ` + spec.fileList.String()
 	}

 	return ret
@@ -89,22 +100,17 @@ func registerBuildActionForModuleFileList(ctx android.ModuleContext,
 	return fileList
 }

-func registerBuildActionForParFile(ctx android.ModuleContext,
-	interpreter, main, binName string, newPyPkgs []string, parSpecs []parSpec) android.Path {
+func registerBuildActionForParFile(ctx android.ModuleContext, embedded_launcher bool,
+	launcher_path android.Path, interpreter, main, binName string,
+	newPyPkgs []string, parSpecs []parSpec) android.Path {

-	// intermediate output path for __init__.py
+	// .intermediate output path for __init__.py
 	initFile := android.PathForModuleOut(ctx, initFileName).String()

-	// the path of stub_template_host.txt from source tree.
-	template := android.PathForSource(ctx, stubTemplateHost)
-
-	// intermediate output path for __main__.py
-	stub := android.PathForModuleOut(ctx, mainFileName).String()
-
-	// intermediate output path for par file.
+	// .intermediate output path for par file.
 	parFile := android.PathForModuleOut(ctx, binName+parFileExt)

-	// intermediate output path for bin executable.
+	// .intermediate output path for bin executable.
 	binFile := android.PathForModuleOut(ctx, binName)

 	// implicit dependency for parFile build action.
@@ -116,32 +122,68 @@ func registerBuildActionForParFile(ctx android.ModuleContext,
 	}

 	parArgs := []string{}
-	parArgs = append(parArgs, "-C "+strings.TrimSuffix(stub, mainFileName)+" -f "+stub)
-	parArgs = append(parArgs, "-C "+strings.TrimSuffix(initFile, initFileName)+" -f "+initFile)
+	parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(initFile, initFileName)+` -f `+initFile)
 	for _, pkg := range newPyPkgs {
-		parArgs = append(parArgs, "-P "+pkg+" -f "+initFile)
+		parArgs = append(parArgs, `-P `+pkg+` -f `+initFile)
 	}
 	for _, p := range parSpecs {
 		parArgs = append(parArgs, p.soongParArgs())
 	}

-	ctx.ModuleBuild(pctx, android.ModuleBuildParams{
-		Rule:        par,
-		Description: "python archive",
-		Output:      binFile,
-		Implicits:   implicits,
-		Args: map[string]string{
-			"initFile": initFile,
-			// the "\" isn't being interpreted by regex parser, it's being
-			// interpreted in the string literal.
-			"interp":   strings.Replace(interpreter, "/", `\/`, -1),
-			"main":     strings.Replace(main, "/", `\/`, -1),
-			"template": template.String(),
-			"stub":     stub,
-			"parFile":  parFile.String(),
-			"parArgs":  strings.Join(parArgs, " "),
-		},
-	})
+	if !embedded_launcher {
+		// the path of stub_template_host.txt from source tree.
+		template := android.PathForSource(ctx, stubTemplateHost)
+
+		// intermediate output path for __main__.py
+		stub := android.PathForModuleOut(ctx, mainFileName).String()
+
+		// added stub file to the soong_zip args.
+		parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(stub, mainFileName)+` -f `+stub)
+
+		ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+			Rule:        host_par,
+			Description: "host python archive",
+			Output:      binFile,
+			Implicits:   implicits,
+			Args: map[string]string{
+				"initFile": initFile,
+				"interp":   strings.Replace(interpreter, "/", `\/`, -1),
+				// we need remove "runfiles/" suffix since stub script starts
+				// searching for main file in each sub-dir of "runfiles" directory tree.
+				"main": strings.Replace(strings.TrimPrefix(main, runFiles+"/"),
+					"/", `\/`, -1),
+				"template": template.String(),
+				"stub":     stub,
+				"parFile":  parFile.String(),
+				"parArgs":  strings.Join(parArgs, " "),
+			},
+		})
+	} else {
+		// added launcher_path to the implicits Ninja dependencies.
+		implicits = append(implicits, launcher_path)
+
+		// .intermediate output path for entry_point.txt
+		entryPoint := android.PathForModuleOut(ctx, entryPointFile).String()
+
+		// added entry_point file to the soong_zip args.
+		parArgs = append(parArgs, `-P "" `+`-C `+fmt.Sprintf(
+			"%q", strings.TrimSuffix(entryPoint, entryPointFile))+` -f `+entryPoint)
+
+		ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+			Rule:        embedded_par,
+			Description: "embedded python archive",
+			Output:      binFile,
+			Implicits:   implicits,
+			Args: map[string]string{
+				"initFile":    initFile,
+				"main":        main,
+				"entry_point": entryPoint,
+				"parFile":     parFile.String(),
+				"parArgs":     strings.Join(parArgs, " "),
+				"launcher":    launcher_path.String(),
+			},
+		})
+	}

 	return binFile
 }
--- a/python/library.go
+++ b/python/library.go
@@ -24,14 +24,8 @@ func init() {
 	android.RegisterModuleType("python_library_host", PythonLibraryHostFactory)
 }

-type PythonLibrary struct {
-	pythonBaseModule
-}
-
-var _ PythonSubModule = (*PythonLibrary)(nil)
-
 func PythonLibraryHostFactory() android.Module {
-	module := &PythonLibrary{}
+	module := newModule(android.HostSupportedNoCross, android.MultilibFirst)

-	return InitPythonBaseModule(&module.pythonBaseModule, module, android.HostSupportedNoCross)
+	return module.Init()
 }
--- a/python/python.go
+++ b/python/python.go
@@ -24,6 +24,7 @@ import (
 	"strings"

 	"github.com/google/blueprint"
+	"github.com/google/blueprint/proptools"

 	"android/soong/android"
 )
@@ -35,58 +36,70 @@ func init() {
 }

 // the version properties that apply to python libraries and binaries.
-type PythonVersionProperties struct {
+type VersionProperties struct {
 	// true, if the module is required to be built with this version.
-	Enabled *bool
-
-	// if specified, common src files are converted to specific version with converter tool.
-	// Converter bool
+	Enabled *bool `android:"arch_variant"`

 	// non-empty list of .py files under this strict Python version.
 	// srcs may reference the outputs of other modules that produce source files like genrule
 	// or filegroup using the syntax ":module".
-	Srcs []string
+	Srcs []string `android:"arch_variant"`
+
+	// list of source files that should not be used to build the Python module.
+	// This is most useful in the arch/multilib variants to remove non-common files
+	Exclude_srcs []string `android:"arch_variant"`

 	// list of the Python libraries under this Python version.
-	Libs []string
+	Libs []string `android:"arch_variant"`
+
+	// true, if the binary is required to be built with embedded launcher.
+	// TODO(nanzhang): Remove this flag when embedded Python3 is supported later.
+	Embedded_launcher *bool `android:"arch_variant"`
 }

 // properties that apply to python libraries and binaries.
-type PythonBaseModuleProperties struct {
+type BaseProperties struct {
 	// the package path prefix within the output artifact at which to place the source/data
 	// files of the current module.
 	// eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
 	// (from a.b.c import ...) statement.
 	// if left unspecified, all the source/data files of current module are copied to
 	// "runfiles/" tree directory directly.
-	Pkg_path string
+	Pkg_path string `android:"arch_variant"`
+
+	// true, if the Python module is used internally, eg, Python std libs.
+	Is_internal *bool `android:"arch_variant"`

 	// list of source (.py) files compatible both with Python2 and Python3 used to compile the
 	// Python module.
 	// srcs may reference the outputs of other modules that produce source files like genrule
 	// or filegroup using the syntax ":module".
 	// Srcs has to be non-empty.
-	Srcs []string
+	Srcs []string `android:"arch_variant"`
+
+	// list of source files that should not be used to build the C/C++ module.
+	// This is most useful in the arch/multilib variants to remove non-common files
+	Exclude_srcs []string `android:"arch_variant"`

 	// list of files or filegroup modules that provide data that should be installed alongside
 	// the test. the file extension can be arbitrary except for (.py).
-	Data []string
+	Data []string `android:"arch_variant"`

 	// list of the Python libraries compatible both with Python2 and Python3.
-	Libs []string
+	Libs []string `android:"arch_variant"`

 	Version struct {
 		// all the "srcs" or Python dependencies that are to be used only for Python2.
-		Py2 PythonVersionProperties
+		Py2 VersionProperties `android:"arch_variant"`

 		// all the "srcs" or Python dependencies that are to be used only for Python3.
-		Py3 PythonVersionProperties
-	}
+		Py3 VersionProperties `android:"arch_variant"`
+	} `android:"arch_variant"`

 	// the actual version each module uses after variations created.
 	// this property name is hidden from users' perspectives, and soong will populate it during
 	// runtime.
-	ActualVersion string `blueprint:"mutated"`
+	Actual_version string `blueprint:"mutated"`
 }

 type pathMapping struct {
@@ -94,11 +107,21 @@ type pathMapping struct {
 	src  android.Path
 }

-type pythonBaseModule struct {
+type Module struct {
 	android.ModuleBase
-	subModule PythonSubModule

-	properties PythonBaseModuleProperties
+	properties BaseProperties
+
+	// initialize before calling Init
+	hod      android.HostOrDeviceSupported
+	multilib android.Multilib
+
+	// the bootstrapper is used to bootstrap .par executable.
+	// bootstrapper might be nil (Python library module).
+	bootstrapper bootstrapper
+
+	// the installer might be nil.
+	installer installer

 	// the Python files of current module after expanding source dependencies.
 	// pathMapping: <dest: runfile_path, src: source_path>
@@ -108,17 +131,37 @@ type pythonBaseModule struct {
 	// pathMapping: <dest: runfile_path, src: source_path>
 	dataPathMappings []pathMapping

+	// soong_zip arguments of all its dependencies.
+	depsParSpecs []parSpec
+
+	// Python runfiles paths of all its dependencies.
+	depsPyRunfiles []string
+
+	// (.intermediate) module output path as installation source.
+	installSource android.OptionalPath
+
 	// the soong_zip arguments for zipping current module source/data files.
 	parSpec parSpec

-	// the installer might be nil.
-	installer installer
-
 	subAndroidMkOnce map[subAndroidMkProvider]bool
 }

-type PythonSubModule interface {
-	GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath
+func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
+	return &Module{
+		hod:      hod,
+		multilib: multilib,
+	}
+}
+
+type bootstrapper interface {
+	bootstrapperProps() []interface{}
+	bootstrap(ctx android.ModuleContext, Actual_version string, embedded_launcher bool,
+		srcsPathMappings []pathMapping, parSpec parSpec,
+		depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath
+}
+
+type installer interface {
+	install(ctx android.ModuleContext, path android.Path)
 }

 type PythonDependency interface {
@@ -127,64 +170,58 @@ type PythonDependency interface {
 	GetParSpec() parSpec
 }

-type pythonDecorator struct {
-	baseInstaller *pythonInstaller
-}
-
-type installer interface {
-	install(ctx android.ModuleContext, path android.Path)
-}
-
-func (p *pythonBaseModule) GetSrcsPathMappings() []pathMapping {
+func (p *Module) GetSrcsPathMappings() []pathMapping {
 	return p.srcsPathMappings
 }

-func (p *pythonBaseModule) GetDataPathMappings() []pathMapping {
+func (p *Module) GetDataPathMappings() []pathMapping {
 	return p.dataPathMappings
 }

-func (p *pythonBaseModule) GetParSpec() parSpec {
+func (p *Module) GetParSpec() parSpec {
 	return p.parSpec
 }

-var _ PythonDependency = (*pythonBaseModule)(nil)
+var _ PythonDependency = (*Module)(nil)

-var _ android.AndroidMkDataProvider = (*pythonBaseModule)(nil)
+var _ android.AndroidMkDataProvider = (*Module)(nil)

-func InitPythonBaseModule(baseModule *pythonBaseModule, subModule PythonSubModule,
-	hod android.HostOrDeviceSupported) android.Module {
+func (p *Module) Init() android.Module {

-	baseModule.subModule = subModule
+	p.AddProperties(&p.properties)
+	if p.bootstrapper != nil {
+		p.AddProperties(p.bootstrapper.bootstrapperProps()...)
+	}

-	baseModule.AddProperties(&baseModule.properties)
+	android.InitAndroidArchModule(p, p.hod, p.multilib)

-	android.InitAndroidArchModule(baseModule, hod, android.MultilibCommon)
-
-	return baseModule
+	return p
 }

-// the tag used to mark dependencies within "py_libs" attribute.
-type pythonDependencyTag struct {
+type dependencyTag struct {
 	blueprint.BaseDependencyTag
+	name string
 }

-var pyDependencyTag pythonDependencyTag
-
 var (
+	pythonLibTag       = dependencyTag{name: "pythonLib"}
+	launcherTag        = dependencyTag{name: "launcher"}
 	pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`)
 	pyExt              = ".py"
 	pyVersion2         = "PY2"
 	pyVersion3         = "PY3"
 	initFileName       = "__init__.py"
 	mainFileName       = "__main__.py"
+	entryPointFile     = "entry_point.txt"
 	parFileExt         = ".zip"
 	runFiles           = "runfiles"
+	internal           = "internal"
 )

 // create version variants for modules.
 func versionSplitMutator() func(android.BottomUpMutatorContext) {
 	return func(mctx android.BottomUpMutatorContext) {
-		if base, ok := mctx.Module().(*pythonBaseModule); ok {
+		if base, ok := mctx.Module().(*Module); ok {
 			versionNames := []string{}
 			if base.properties.Version.Py2.Enabled != nil &&
 				*(base.properties.Version.Py2.Enabled) == true {
@@ -197,36 +234,61 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) {
 			modules := mctx.CreateVariations(versionNames...)
 			for i, v := range versionNames {
 				// set the actual version for Python module.
-				modules[i].(*pythonBaseModule).properties.ActualVersion = v
+				modules[i].(*Module).properties.Actual_version = v
 			}
 		}
 	}
 }

-func (p *pythonBaseModule) DepsMutator(ctx android.BottomUpMutatorContext) {
+func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool {
+	switch actual_version {
+	case pyVersion2:
+		return proptools.Bool(p.properties.Version.Py2.Embedded_launcher)
+	case pyVersion3:
+		return proptools.Bool(p.properties.Version.Py3.Embedded_launcher)
+	}
+
+	return false
+}
+
+func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
 	// deps from "data".
 	android.ExtractSourcesDeps(ctx, p.properties.Data)
 	// deps from "srcs".
 	android.ExtractSourcesDeps(ctx, p.properties.Srcs)

-	switch p.properties.ActualVersion {
+	switch p.properties.Actual_version {
 	case pyVersion2:
 		// deps from "version.py2.srcs" property.
 		android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs)

-		ctx.AddVariationDependencies(nil, pyDependencyTag,
+		ctx.AddVariationDependencies(nil, pythonLibTag,
 			uniqueLibs(ctx, p.properties.Libs, "version.py2.libs",
 				p.properties.Version.Py2.Libs)...)
+
+		if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) {
+			ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib")
+			ctx.AddFarVariationDependencies([]blueprint.Variation{
+				{"arch", ctx.Target().String()},
+			}, launcherTag, "py2-launcher")
+		}
+
 	case pyVersion3:
 		// deps from "version.py3.srcs" property.
 		android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs)

-		ctx.AddVariationDependencies(nil, pyDependencyTag,
+		ctx.AddVariationDependencies(nil, pythonLibTag,
 			uniqueLibs(ctx, p.properties.Libs, "version.py3.libs",
 				p.properties.Version.Py3.Libs)...)
+
+		if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion3) {
+			//TODO(nanzhang): Add embedded launcher for Python3.
+			ctx.PropertyErrorf("version.py3.embedded_launcher",
+				"is not supported yet for Python3.")
+		}
 	default:
-		panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
-			p.properties.ActualVersion, ctx.ModuleName()))
+		panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+			p.properties.Actual_version, ctx.ModuleName()))
 	}
 }

@@ -258,27 +320,43 @@ func uniqueLibs(ctx android.BottomUpMutatorContext,
 	return ret
 }

-func (p *pythonBaseModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
-	installSource := p.subModule.GeneratePythonBuildActions(ctx)
+func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+	p.GeneratePythonBuildActions(ctx)

-	if p.installer != nil && installSource.Valid() {
-		p.installer.install(ctx, installSource.Path())
+	if p.bootstrapper != nil {
+		// TODO(nanzhang): Since embedded launcher is not supported for Python3 for now,
+		// so we initialize "embedded_launcher" to false.
+		embedded_launcher := false
+		if p.properties.Actual_version == pyVersion2 {
+			embedded_launcher = p.isEmbeddedLauncherEnabled(pyVersion2)
+		}
+		p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
+			embedded_launcher, p.srcsPathMappings, p.parSpec, p.depsPyRunfiles,
+			p.depsParSpecs)
 	}
+
+	if p.installer != nil && p.installSource.Valid() {
+		p.installer.install(ctx, p.installSource.Path())
+	}
+
 }

-func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath {
+func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) {
 	// expand python files from "srcs" property.
 	srcs := p.properties.Srcs
-	switch p.properties.ActualVersion {
+	exclude_srcs := p.properties.Exclude_srcs
+	switch p.properties.Actual_version {
 	case pyVersion2:
 		srcs = append(srcs, p.properties.Version.Py2.Srcs...)
+		exclude_srcs = append(exclude_srcs, p.properties.Version.Py2.Exclude_srcs...)
 	case pyVersion3:
 		srcs = append(srcs, p.properties.Version.Py3.Srcs...)
+		exclude_srcs = append(exclude_srcs, p.properties.Version.Py3.Exclude_srcs...)
 	default:
-		panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
-			p.properties.ActualVersion, ctx.ModuleName()))
+		panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+			p.properties.Actual_version, ctx.ModuleName()))
 	}
-	expandedSrcs := ctx.ExpandSources(srcs, nil)
+	expandedSrcs := ctx.ExpandSources(srcs, exclude_srcs)
 	if len(expandedSrcs) == 0 {
 		ctx.ModuleErrorf("doesn't have any source files!")
 	}
@@ -292,15 +370,26 @@ func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext)
 		pkg_path = filepath.Clean(p.properties.Pkg_path)
 		if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") ||
 			strings.HasPrefix(pkg_path, "/") {
-			ctx.PropertyErrorf("pkg_path", "%q is not a valid format.",
+			ctx.PropertyErrorf("pkg_path",
+				"%q must be a relative path contained in par file.",
 				p.properties.Pkg_path)
-			return android.OptionalPath{}
+			return
+		}
+		if p.properties.Is_internal != nil && *p.properties.Is_internal {
+			// pkg_path starts from "internal/" implicitly.
+			pkg_path = filepath.Join(internal, pkg_path)
+		} else {
+			// pkg_path starts from "runfiles/" implicitly.
+			pkg_path = filepath.Join(runFiles, pkg_path)
 		}
-		// pkg_path starts from "runfiles/" implicitly.
-		pkg_path = filepath.Join(runFiles, pkg_path)
 	} else {
-		// pkg_path starts from "runfiles/" implicitly.
-		pkg_path = runFiles
+		if p.properties.Is_internal != nil && *p.properties.Is_internal {
+			// pkg_path starts from "runfiles/" implicitly.
+			pkg_path = internal
+		} else {
+			// pkg_path starts from "runfiles/" implicitly.
+			pkg_path = runFiles
+		}
 	}

 	p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData)
@@ -308,13 +397,11 @@ func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext)
 	p.parSpec = p.dumpFileList(ctx, pkg_path)

 	p.uniqWholeRunfilesTree(ctx)
-
-	return android.OptionalPath{}
 }

 // generate current module unique pathMappings: <dest: runfiles_path, src: source_path>
 // for python/data files.
-func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
+func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
 	expandedSrcs, expandedData android.Paths) {
 	// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
 	// check duplicates.
@@ -355,7 +442,7 @@ func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_
 }

 // register build actions to dump filelist to disk.
-func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
+func (p *Module) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
 	relativeRootMap := make(map[string]android.Paths)
 	// the soong_zip params in order to pack current module's Python/data files.
 	ret := parSpec{rootPrefix: pkg_path}
@@ -365,7 +452,8 @@ func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path stri
 	// "srcs" or "data" properties may have filegroup so it might happen that
 	// the relative root for each source path is different.
 	for _, path := range pathMappings {
-		relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
+		var relativeRoot string
+		relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
 		if v, found := relativeRootMap[relativeRoot]; found {
 			relativeRootMap[relativeRoot] = append(v, path.src)
 		} else {
@@ -392,8 +480,19 @@ func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path stri
 	return ret
 }

-// check Python/data files duplicates from current module and its whole dependencies.
-func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
+func isPythonLibModule(module blueprint.Module) bool {
+	if m, ok := module.(*Module); ok {
+		// Python library has no bootstrapper or installer.
+		if m.bootstrapper != nil || m.installer != nil {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// check Python source/data files duplicates from current module and its whole dependencies.
+func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) {
 	// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
 	// check duplicates.
 	destToPySrcs := make(map[string]string)
@@ -408,16 +507,15 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {

 	// visit all its dependencies in depth first.
 	ctx.VisitDepsDepthFirst(func(module blueprint.Module) {
-		// module can only depend on Python library.
-		if base, ok := module.(*pythonBaseModule); ok {
-			if _, ok := base.subModule.(*PythonLibrary); !ok {
-				panic(fmt.Errorf(
-					"the dependency %q of module %q is not Python library!",
-					ctx.ModuleName(), ctx.OtherModuleName(module)))
-			}
-		} else {
+		if ctx.OtherModuleDependencyTag(module) != pythonLibTag {
 			return
 		}
+		// Python module cannot depend on modules, except for Python library.
+		if !isPythonLibModule(module) {
+			panic(fmt.Errorf(
+				"the dependency %q of module %q is not Python library!",
+				ctx.ModuleName(), ctx.OtherModuleName(module)))
+		}
 		if dep, ok := module.(PythonDependency); ok {
 			srcs := dep.GetSrcsPathMappings()
 			for _, path := range srcs {
@@ -428,9 +526,7 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
 				}
 				// binary needs the Python runfiles paths from all its
 				// dependencies to fill __init__.py in each runfiles dir.
-				if sub, ok := p.subModule.(*pythonBinaryBase); ok {
-					sub.depsPyRunfiles = append(sub.depsPyRunfiles, path.dest)
-				}
+				p.depsPyRunfiles = append(p.depsPyRunfiles, path.dest)
 			}
 			data := dep.GetDataPathMappings()
 			for _, path := range data {
@@ -440,9 +536,7 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
 			}
 			// binary needs the soong_zip arguments from all its
 			// dependencies to generate executable par file.
-			if sub, ok := p.subModule.(*pythonBinaryBase); ok {
-				sub.depsParSpecs = append(sub.depsParSpecs, dep.GetParSpec())
-			}
+			p.depsParSpecs = append(p.depsParSpecs, dep.GetParSpec())
 		}
 	})
 }
--- a/python/python_test.go
+++ b/python/python_test.go
@@ -28,7 +28,7 @@ import (
 	"android/soong/android"
 )

-type pyBinary struct {
+type pyModule struct {
 	name           string
 	actualVersion  string
 	pyRunfiles     []string
@@ -41,7 +41,7 @@ var (
 	buildNamePrefix          = "soong_python_test"
 	moduleVariantErrTemplate = "%s: module %q variant %q: "
 	pkgPathErrTemplate       = moduleVariantErrTemplate +
-		"pkg_path: %q is not a valid format."
+		"pkg_path: %q must be a relative path contained in par file."
 	badIdentifierErrTemplate = moduleVariantErrTemplate +
 		"srcs: the path %q contains invalid token %q."
 	dupRunfileErrTemplate = moduleVariantErrTemplate +
@@ -58,7 +58,7 @@ var (
 		mockFiles map[string][]byte

 		errors           []string
-		expectedBinaries []pyBinary
+		expectedBinaries []pyModule
 	}{
 		{
 			desc: "module without any src files",
@@ -278,7 +278,7 @@ var (
 				stubTemplateHost: []byte(`PYTHON_BINARY = '%interpreter%'
 				MAIN_FILE = '%main%'`),
 			},
-			expectedBinaries: []pyBinary{
+			expectedBinaries: []pyModule{
 				{
 					name:          "bin",
 					actualVersion: "PY3",
@@ -363,14 +363,10 @@ func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, varian
 	expParSpec string, expDepsParSpecs []string) (testErrs []error) {
 	module := ctx.ModuleForTests(name, variant)

-	base, baseOk := module.Module().(*pythonBaseModule)
+	base, baseOk := module.Module().(*Module)
 	if !baseOk {
 		t.Fatalf("%s is not Python module!", name)
 	}
-	sub, subOk := base.subModule.(*pythonBinaryBase)
-	if !subOk {
-		t.Fatalf("%s is not Python binary!", name)
-	}

 	actPyRunfiles := []string{}
 	for _, path := range base.srcsPathMappings {
@@ -381,28 +377,28 @@ func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, varian
 		testErrs = append(testErrs, errors.New(fmt.Sprintf(
 			`binary "%s" variant "%s" has unexpected pyRunfiles: %q!`,
 			base.Name(),
-			base.properties.ActualVersion,
+			base.properties.Actual_version,
 			actPyRunfiles)))
 	}

-	if !reflect.DeepEqual(sub.depsPyRunfiles, expDepsPyRunfiles) {
+	if !reflect.DeepEqual(base.depsPyRunfiles, expDepsPyRunfiles) {
 		testErrs = append(testErrs, errors.New(fmt.Sprintf(
 			`binary "%s" variant "%s" has unexpected depsPyRunfiles: %q!`,
 			base.Name(),
-			base.properties.ActualVersion,
-			sub.depsPyRunfiles)))
+			base.properties.Actual_version,
+			base.depsPyRunfiles)))
 	}

 	if base.parSpec.soongParArgs() != strings.Replace(expParSpec, "@prefix@", buildDir, 1) {
 		testErrs = append(testErrs, errors.New(fmt.Sprintf(
 			`binary "%s" variant "%s" has unexpected parSpec: %q!`,
 			base.Name(),
-			base.properties.ActualVersion,
+			base.properties.Actual_version,
 			base.parSpec.soongParArgs())))
 	}

 	actDepsParSpecs := []string{}
-	for i, p := range sub.depsParSpecs {
+	for i, p := range base.depsParSpecs {
 		actDepsParSpecs = append(actDepsParSpecs, p.soongParArgs())
 		expDepsParSpecs[i] = strings.Replace(expDepsParSpecs[i], "@prefix@", buildDir, 1)
 	}
@@ -411,7 +407,7 @@ func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, varian
 		testErrs = append(testErrs, errors.New(fmt.Sprintf(
 			`binary "%s" variant "%s" has unexpected depsParSpecs: %q!`,
 			base.Name(),
-			base.properties.ActualVersion,
+			base.properties.Actual_version,
 			actDepsParSpecs)))
 	}

--- a/python/test.go
+++ b/python/test.go
@@ -16,7 +16,6 @@ package python

 import (
 	"android/soong/android"
-	"path/filepath"
 )

 // This file contains the module types for building Python test.
@@ -25,30 +24,29 @@ func init() {
 	android.RegisterModuleType("python_test_host", PythonTestHostFactory)
 }

-type PythonTestHost struct {
-	pythonBinaryBase
+type testDecorator struct {
+	*binaryDecorator
 }

-var _ PythonSubModule = (*PythonTestHost)(nil)
-
-type pythonTestHostDecorator struct {
-	pythonDecorator
+func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) {
+	test.binaryDecorator.baseInstaller.install(ctx, file)
 }

-func (p *pythonTestHostDecorator) install(ctx android.ModuleContext, file android.Path) {
-	p.pythonDecorator.baseInstaller.dir = filepath.Join("nativetest", ctx.ModuleName())
-	p.pythonDecorator.baseInstaller.install(ctx, file)
+func NewTest(hod android.HostOrDeviceSupported) *Module {
+	module, binary := NewBinary(hod)
+
+	binary.baseInstaller = NewPythonInstaller("nativetest")
+
+	test := &testDecorator{binaryDecorator: binary}
+
+	module.bootstrapper = test
+	module.installer = test
+
+	return module
 }

 func PythonTestHostFactory() android.Module {
-	decorator := &pythonTestHostDecorator{
-		pythonDecorator: pythonDecorator{baseInstaller: NewPythonInstaller("nativetest")}}
+	module := NewTest(android.HostSupportedNoCross)

-	module := &PythonBinaryHost{}
-	module.pythonBaseModule.installer = decorator
-
-	module.AddProperties(&module.binaryProperties)
-
-	return InitPythonBaseModule(&module.pythonBinaryBase.pythonBaseModule,
-		&module.pythonBinaryBase, android.HostSupportedNoCross)
+	return module.Init()
 }