diff --git a/apex/apex.go b/apex/apex.go index 9485a4b0b..ad7da2716 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1657,7 +1657,7 @@ func apexFileForRustLibrary(ctx android.BaseModuleContext, rustm *rust.Module) a return newApexFile(ctx, fileToCopy, androidMkModuleName, dirInApex, nativeSharedLib, rustm) } -func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.Module) apexFile { +func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.PythonBinaryModule) apexFile { dirInApex := "bin" fileToCopy := py.HostToolPath().Path() return newApexFile(ctx, fileToCopy, py.BaseModuleName(), dirInApex, pyBinary, py) @@ -2147,7 +2147,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, case *cc.Module: vctx.filesInfo = append(vctx.filesInfo, apexFileForExecutable(ctx, ch)) return true // track transitive dependencies - case *python.Module: + case *python.PythonBinaryModule: if ch.HostToolPath().Valid() { vctx.filesInfo = append(vctx.filesInfo, apexFileForPyBinary(ctx, ch)) } diff --git a/python/Android.bp b/python/Android.bp index e49fa6a3c..4584f1e83 100644 --- a/python/Android.bp +++ b/python/Android.bp @@ -11,11 +11,10 @@ bootstrap_go_package { "soong-tradefed", ], srcs: [ - "androidmk.go", "binary.go", + "bp2build.go", "builder.go", "defaults.go", - "installer.go", "library.go", "proto.go", "python.go", diff --git a/python/androidmk.go b/python/androidmk.go deleted file mode 100644 index 7dc471397..000000000 --- a/python/androidmk.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "path/filepath" - "strings" - - "android/soong/android" -) - -type subAndroidMkProvider interface { - AndroidMk(*Module, *android.AndroidMkEntries) -} - -func (p *Module) subAndroidMk(entries *android.AndroidMkEntries, obj interface{}) { - if p.subAndroidMkOnce == nil { - p.subAndroidMkOnce = make(map[subAndroidMkProvider]bool) - } - if androidmk, ok := obj.(subAndroidMkProvider); ok { - if !p.subAndroidMkOnce[androidmk] { - p.subAndroidMkOnce[androidmk] = true - androidmk.AndroidMk(p, entries) - } - } -} - -func (p *Module) AndroidMkEntries() []android.AndroidMkEntries { - entries := android.AndroidMkEntries{OutputFile: p.installSource} - - p.subAndroidMk(&entries, p.installer) - - return []android.AndroidMkEntries{entries} -} - -func (p *binaryDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) { - entries.Class = "EXECUTABLES" - - entries.ExtraEntries = append(entries.ExtraEntries, - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) - }) - base.subAndroidMk(entries, p.pythonInstaller) -} - -func (p *testDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) { - entries.Class = "NATIVE_TESTS" - - entries.ExtraEntries = append(entries.ExtraEntries, - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.AddCompatibilityTestSuites(p.binaryDecorator.binaryProperties.Test_suites...) - if p.testConfig != nil { - entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) - } - - entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) - - entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...) - - p.testProperties.Test_options.SetAndroidMkEntries(entries) - }) - base.subAndroidMk(entries, p.binaryDecorator.pythonInstaller) -} - -func (installer *pythonInstaller) AndroidMk(base *Module, entries *android.AndroidMkEntries) { - entries.Required = append(entries.Required, "libc++") - entries.ExtraEntries = append(entries.ExtraEntries, - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - path, file := filepath.Split(installer.path.String()) - stem := strings.TrimSuffix(file, filepath.Ext(file)) - - entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file)) - entries.SetString("LOCAL_MODULE_PATH", path) - entries.SetString("LOCAL_MODULE_STEM", stem) - entries.AddStrings("LOCAL_SHARED_LIBRARIES", installer.androidMkSharedLibs...) - entries.SetBool("LOCAL_CHECK_ELF_FILES", false) - }) -} diff --git a/python/binary.go b/python/binary.go index 670e0d313..95eb2c66c 100644 --- a/python/binary.go +++ b/python/binary.go @@ -18,11 +18,12 @@ package python import ( "fmt" + "path/filepath" + "strings" + + "github.com/google/blueprint" "android/soong/android" - "android/soong/bazel" - - "github.com/google/blueprint/proptools" ) func init() { @@ -33,63 +34,6 @@ func registerPythonBinaryComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) } -type bazelPythonBinaryAttributes struct { - Main *bazel.Label - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Python_version *string - Imports bazel.StringListAttribute -} - -func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *Module) { - // TODO(b/182306917): this doesn't fully handle all nested props versioned - // by the python version, which would have been handled by the version split - // mutator. This is sufficient for very simple python_binary_host modules - // under Bionic. - py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false) - py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) - var python_version *string - if py3Enabled && py2Enabled { - panic(fmt.Errorf( - "error for '%s' module: bp2build's python_binary_host converter does not support "+ - "converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name())) - } else if py2Enabled { - python_version = &pyVersion2 - } else { - // do nothing, since python_version defaults to PY3. - } - - baseAttrs := m.makeArchVariantBaseAttributes(ctx) - attrs := &bazelPythonBinaryAttributes{ - Main: nil, - Srcs: baseAttrs.Srcs, - Deps: baseAttrs.Deps, - Python_version: python_version, - Imports: baseAttrs.Imports, - } - - for _, propIntf := range m.GetProperties() { - if props, ok := propIntf.(*BinaryProperties); ok { - // main is optional. - if props.Main != nil { - main := android.BazelLabelForModuleSrcSingle(ctx, *props.Main) - attrs.Main = &main - break - } - } - } - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_binary rule. - Rule_class: "py_binary", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: m.Name(), - Data: baseAttrs.Data, - }, attrs) -} - type BinaryProperties struct { // the name of the source file that is the main entry point of the program. // this file must also be listed in srcs. @@ -118,52 +62,61 @@ type BinaryProperties struct { Auto_gen_config *bool } -type binaryDecorator struct { +type PythonBinaryModule struct { + PythonLibraryModule binaryProperties BinaryProperties - *pythonInstaller + // (.intermediate) module output path as installation source. + installSource android.Path + + // Final installation path. + installedDest android.Path + + androidMkSharedLibs []string } +var _ android.AndroidMkEntriesProvider = (*PythonBinaryModule)(nil) +var _ android.Module = (*PythonBinaryModule)(nil) + type IntermPathProvider interface { IntermPathForModuleOut() android.OptionalPath } -func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { - module := newModule(hod, android.MultilibFirst) - decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "")} - - module.bootstrapper = decorator - module.installer = decorator - - return module, decorator +func NewBinary(hod android.HostOrDeviceSupported) *PythonBinaryModule { + return &PythonBinaryModule{ + PythonLibraryModule: *newModule(hod, android.MultilibFirst), + } } func PythonBinaryHostFactory() android.Module { - module, _ := NewBinary(android.HostSupported) - - android.InitBazelModule(module) - - return module.init() + return NewBinary(android.HostSupported).init() } -func (binary *binaryDecorator) autorun() bool { - return BoolDefault(binary.binaryProperties.Autorun, true) +func (p *PythonBinaryModule) init() android.Module { + p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.binaryProperties) + android.InitAndroidArchModule(p, p.hod, p.multilib) + android.InitDefaultableModule(p) + android.InitBazelModule(p) + return p } -func (binary *binaryDecorator) bootstrapperProps() []interface{} { - return []interface{}{&binary.binaryProperties} +func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + p.PythonLibraryModule.GenerateAndroidBuildActions(ctx) + p.buildBinary(ctx) + p.installedDest = ctx.InstallFile(installDir(ctx, "bin", "", ""), + p.installSource.Base(), p.installSource) } -func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersion string, - embeddedLauncher bool, srcsPathMappings []pathMapping, srcsZip android.Path, - depsSrcsZips android.Paths) android.OptionalPath { - +func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { + depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx) main := "" - if binary.autorun() { - main = binary.getPyMainFile(ctx, srcsPathMappings) + if p.autorun() { + main = p.getPyMainFile(ctx, p.srcsPathMappings) } var launcherPath android.OptionalPath + embeddedLauncher := p.isEmbeddedLauncherEnabled() if embeddedLauncher { ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) { if provider, ok := m.(IntermPathProvider); ok { @@ -175,15 +128,137 @@ func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersio } }) } - binFile := registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, - binary.getHostInterpreterName(ctx, actualVersion), - main, binary.getStem(ctx), append(android.Paths{srcsZip}, depsSrcsZips...)) + p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, + p.getHostInterpreterName(ctx, p.properties.Actual_version), + main, p.getStem(ctx), append(android.Paths{p.srcsZip}, depsSrcsZips...)) - return android.OptionalPathForPath(binFile) + var sharedLibs []string + // if embedded launcher is enabled, we need to collect the shared library dependencies of the + // launcher + for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { + sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) + } + p.androidMkSharedLibs = sharedLibs +} + +func (p *PythonBinaryModule) AndroidMkEntries() []android.AndroidMkEntries { + entries := android.AndroidMkEntries{OutputFile: android.OptionalPathForPath(p.installSource)} + + entries.Class = "EXECUTABLES" + + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) + }) + + entries.Required = append(entries.Required, "libc++") + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + path, file := filepath.Split(p.installedDest.String()) + stem := strings.TrimSuffix(file, filepath.Ext(file)) + + entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file)) + entries.SetString("LOCAL_MODULE_PATH", path) + entries.SetString("LOCAL_MODULE_STEM", stem) + entries.AddStrings("LOCAL_SHARED_LIBRARIES", p.androidMkSharedLibs...) + entries.SetBool("LOCAL_CHECK_ELF_FILES", false) + }) + + return []android.AndroidMkEntries{entries} +} + +func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) { + p.PythonLibraryModule.DepsMutator(ctx) + + versionVariation := []blueprint.Variation{ + {"python_version", p.properties.Actual_version}, + } + + // If this module will be installed and has an embedded launcher, we need to add dependencies for: + // * standard library + // * launcher + // * shared dependencies of the launcher + if p.isEmbeddedLauncherEnabled() { + var stdLib string + var launcherModule string + // Add launcher shared lib dependencies. Ideally, these should be + // derived from the `shared_libs` property of the launcher. However, we + // cannot read the property at this stage and it will be too late to add + // dependencies later. + launcherSharedLibDeps := []string{ + "libsqlite", + } + // Add launcher-specific dependencies for bionic + if ctx.Target().Os.Bionic() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") + } + if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") + } + + switch p.properties.Actual_version { + case pyVersion2: + stdLib = "py2-stdlib" + + launcherModule = "py2-launcher" + if p.autorun() { + launcherModule = "py2-launcher-autorun" + } + + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") + + case pyVersion3: + stdLib = "py3-stdlib" + + launcherModule = "py3-launcher" + if p.autorun() { + launcherModule = "py3-launcher-autorun" + } + if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl { + launcherModule += "-static" + } + + if ctx.Device() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") + } + default: + panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", + p.properties.Actual_version, ctx.ModuleName())) + } + ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...) + } +} + +// HostToolPath returns a path if appropriate such that this module can be used as a host tool, +// fulfilling the android.HostToolProvider interface. +func (p *PythonBinaryModule) HostToolPath() android.OptionalPath { + // TODO: This should only be set when building host binaries -- tests built for device would be + // setting this incorrectly. + return android.OptionalPathForPath(p.installedDest) +} + +// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. +func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{p.installSource}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool { + return Bool(p.properties.Embedded_launcher) +} + +func (b *PythonBinaryModule) autorun() bool { + return BoolDefault(b.binaryProperties.Autorun, true) } // get host interpreter name. -func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext, +func (p *PythonBinaryModule) getHostInterpreterName(ctx android.ModuleContext, actualVersion string) string { var interp string switch actualVersion { @@ -200,13 +275,13 @@ func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext, } // find main program path within runfiles tree. -func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext, +func (p *PythonBinaryModule) getPyMainFile(ctx android.ModuleContext, srcsPathMappings []pathMapping) string { var main string - if String(binary.binaryProperties.Main) == "" { + if String(p.binaryProperties.Main) == "" { main = ctx.ModuleName() + pyExt } else { - main = String(binary.binaryProperties.Main) + main = String(p.binaryProperties.Main) } for _, path := range srcsPathMappings { @@ -219,11 +294,21 @@ func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext, return "" } -func (binary *binaryDecorator) getStem(ctx android.ModuleContext) string { +func (p *PythonBinaryModule) getStem(ctx android.ModuleContext) string { stem := ctx.ModuleName() - if String(binary.binaryProperties.Stem) != "" { - stem = String(binary.binaryProperties.Stem) + if String(p.binaryProperties.Stem) != "" { + stem = String(p.binaryProperties.Stem) } - return stem + String(binary.binaryProperties.Suffix) + return stem + String(p.binaryProperties.Suffix) +} + +func installDir(ctx android.ModuleContext, dir, dir64, relative string) android.InstallPath { + if ctx.Arch().ArchType.Multilib == "lib64" && dir64 != "" { + dir = dir64 + } + if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { + dir = filepath.Join(dir, ctx.Arch().ArchType.String()) + } + return android.PathForModuleInstall(ctx, dir, relative) } diff --git a/python/bp2build.go b/python/bp2build.go new file mode 100644 index 000000000..bdac2dc38 --- /dev/null +++ b/python/bp2build.go @@ -0,0 +1,226 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/bazel" +) + +type bazelPythonLibraryAttributes struct { + Srcs bazel.LabelListAttribute + Deps bazel.LabelListAttribute + Imports bazel.StringListAttribute + Srcs_version *string +} + +type bazelPythonProtoLibraryAttributes struct { + Deps bazel.LabelListAttribute +} + +type baseAttributes struct { + // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv + //Pkg_path bazel.StringAttribute + // TODO: Related to Pkg_bath and similarLy gated + //Is_internal bazel.BoolAttribute + // Combines Srcs and Exclude_srcs + Srcs bazel.LabelListAttribute + Deps bazel.LabelListAttribute + // Combines Data and Java_data (invariant) + Data bazel.LabelListAttribute + Imports bazel.StringListAttribute +} + +func (m *PythonLibraryModule) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes { + var attrs baseAttributes + archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{}) + for axis, configToProps := range archVariantBaseProps { + for config, props := range configToProps { + if baseProps, ok := props.(*BaseProperties); ok { + attrs.Srcs.SetSelectValue(axis, config, + android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs)) + attrs.Deps.SetSelectValue(axis, config, + android.BazelLabelForModuleDeps(ctx, baseProps.Libs)) + data := android.BazelLabelForModuleSrc(ctx, baseProps.Data) + data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data)) + attrs.Data.SetSelectValue(axis, config, data) + } + } + } + + partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ + "proto": android.ProtoSrcLabelPartition, + "py": bazel.LabelPartition{Keep_remainder: true}, + }) + attrs.Srcs = partitionedSrcs["py"] + + if !partitionedSrcs["proto"].IsEmpty() { + protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) + protoLabel := bazel.Label{Label: ":" + protoInfo.Name} + + pyProtoLibraryName := m.Name() + "_py_proto" + ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ + Rule_class: "py_proto_library", + Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", + }, android.CommonAttributes{ + Name: pyProtoLibraryName, + }, &bazelPythonProtoLibraryAttributes{ + Deps: bazel.MakeSingleLabelListAttribute(protoLabel), + }) + + attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) + } + + // Bazel normally requires `import path.from.top.of.tree` statements in + // python code, but with soong you can directly import modules from libraries. + // Add "imports" attributes to the bazel library so it matches soong's behavior. + imports := "." + if m.properties.Pkg_path != nil { + // TODO(b/215119317) This is a hack to handle the fact that we don't convert + // pkg_path properly right now. If the folder structure that contains this + // Android.bp file matches pkg_path, we can set imports to an appropriate + // number of ../..s to emulate moving the files under a pkg_path folder. + pkg_path := filepath.Clean(*m.properties.Pkg_path) + if strings.HasPrefix(pkg_path, "/") { + ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) + } + + if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { + ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) + } + numFolders := strings.Count(pkg_path, "/") + 1 + dots := make([]string, numFolders) + for i := 0; i < numFolders; i++ { + dots[i] = ".." + } + imports = strings.Join(dots, "/") + } + attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) + + return attrs +} + +func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *PythonLibraryModule) { + // TODO(b/182306917): this doesn't fully handle all nested props versioned + // by the python version, which would have been handled by the version split + // mutator. This is sufficient for very simple python_library modules under + // Bionic. + py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true) + py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) + var python_version *string + if py2Enabled && !py3Enabled { + python_version = &pyVersion2 + } else if !py2Enabled && py3Enabled { + python_version = &pyVersion3 + } else if !py2Enabled && !py3Enabled { + ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled") + } else { + // do nothing, since python_version defaults to PY2ANDPY3 + } + + baseAttrs := m.makeArchVariantBaseAttributes(ctx) + + attrs := &bazelPythonLibraryAttributes{ + Srcs: baseAttrs.Srcs, + Deps: baseAttrs.Deps, + Srcs_version: python_version, + Imports: baseAttrs.Imports, + } + + props := bazel.BazelTargetModuleProperties{ + // Use the native py_library rule. + Rule_class: "py_library", + } + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{ + Name: m.Name(), + Data: baseAttrs.Data, + }, attrs) +} + +type bazelPythonBinaryAttributes struct { + Main *bazel.Label + Srcs bazel.LabelListAttribute + Deps bazel.LabelListAttribute + Python_version *string + Imports bazel.StringListAttribute +} + +func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *PythonBinaryModule) { + // TODO(b/182306917): this doesn't fully handle all nested props versioned + // by the python version, which would have been handled by the version split + // mutator. This is sufficient for very simple python_binary_host modules + // under Bionic. + py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false) + py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) + var python_version *string + if py3Enabled && py2Enabled { + panic(fmt.Errorf( + "error for '%s' module: bp2build's python_binary_host converter does not support "+ + "converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name())) + } else if py2Enabled { + python_version = &pyVersion2 + } else { + // do nothing, since python_version defaults to PY3. + } + + baseAttrs := m.makeArchVariantBaseAttributes(ctx) + attrs := &bazelPythonBinaryAttributes{ + Main: nil, + Srcs: baseAttrs.Srcs, + Deps: baseAttrs.Deps, + Python_version: python_version, + Imports: baseAttrs.Imports, + } + + for _, propIntf := range m.GetProperties() { + if props, ok := propIntf.(*BinaryProperties); ok { + // main is optional. + if props.Main != nil { + main := android.BazelLabelForModuleSrcSingle(ctx, *props.Main) + attrs.Main = &main + break + } + } + } + + props := bazel.BazelTargetModuleProperties{ + // Use the native py_binary rule. + Rule_class: "py_binary", + } + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{ + Name: m.Name(), + Data: baseAttrs.Data, + }, attrs) +} + +func (p *PythonLibraryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + pythonLibBp2Build(ctx, p) +} + +func (p *PythonBinaryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + pythonBinaryBp2Build(ctx, p) +} + +func (p *PythonTestModule) ConvertWithBp2build(_ android.TopDownMutatorContext) { + // Tests are currently unsupported +} diff --git a/python/installer.go b/python/installer.go deleted file mode 100644 index 396f03667..000000000 --- a/python/installer.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "path/filepath" - - "android/soong/android" -) - -// This file handles installing python executables into their final location - -type installLocation int - -const ( - InstallInData installLocation = iota -) - -type pythonInstaller struct { - dir string - dir64 string - relative string - - path android.InstallPath - - androidMkSharedLibs []string -} - -func NewPythonInstaller(dir, dir64 string) *pythonInstaller { - return &pythonInstaller{ - dir: dir, - dir64: dir64, - } -} - -var _ installer = (*pythonInstaller)(nil) - -func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.InstallPath { - dir := installer.dir - if ctx.Arch().ArchType.Multilib == "lib64" && installer.dir64 != "" { - dir = installer.dir64 - } - if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { - dir = filepath.Join(dir, ctx.Arch().ArchType.String()) - } - return android.PathForModuleInstall(ctx, dir, installer.relative) -} - -func (installer *pythonInstaller) install(ctx android.ModuleContext, file android.Path) { - installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file) -} - -func (installer *pythonInstaller) setAndroidMkSharedLibs(sharedLibs []string) { - installer.androidMkSharedLibs = sharedLibs -} diff --git a/python/library.go b/python/library.go index df92df42b..7cdb80b87 100644 --- a/python/library.go +++ b/python/library.go @@ -18,9 +18,6 @@ package python import ( "android/soong/android" - "android/soong/bazel" - - "github.com/google/blueprint/proptools" ) func init() { @@ -33,66 +30,9 @@ func registerPythonLibraryComponents(ctx android.RegistrationContext) { } func PythonLibraryHostFactory() android.Module { - module := newModule(android.HostSupported, android.MultilibFirst) - - android.InitBazelModule(module) - - return module.init() -} - -type bazelPythonLibraryAttributes struct { - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Imports bazel.StringListAttribute - Srcs_version *string -} - -type bazelPythonProtoLibraryAttributes struct { - Deps bazel.LabelListAttribute -} - -func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *Module) { - // TODO(b/182306917): this doesn't fully handle all nested props versioned - // by the python version, which would have been handled by the version split - // mutator. This is sufficient for very simple python_library modules under - // Bionic. - py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true) - py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) - var python_version *string - if py2Enabled && !py3Enabled { - python_version = &pyVersion2 - } else if !py2Enabled && py3Enabled { - python_version = &pyVersion3 - } else if !py2Enabled && !py3Enabled { - ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled") - } else { - // do nothing, since python_version defaults to PY2ANDPY3 - } - - baseAttrs := m.makeArchVariantBaseAttributes(ctx) - - attrs := &bazelPythonLibraryAttributes{ - Srcs: baseAttrs.Srcs, - Deps: baseAttrs.Deps, - Srcs_version: python_version, - Imports: baseAttrs.Imports, - } - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_library rule. - Rule_class: "py_library", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: m.Name(), - Data: baseAttrs.Data, - }, attrs) + return newModule(android.HostSupported, android.MultilibFirst).init() } func PythonLibraryFactory() android.Module { - module := newModule(android.HostAndDeviceSupported, android.MultilibBoth) - - android.InitBazelModule(module) - - return module.init() + return newModule(android.HostAndDeviceSupported, android.MultilibBoth).init() } diff --git a/python/python.go b/python/python.go index 24e1bb2ec..2b71e83fe 100644 --- a/python/python.go +++ b/python/python.go @@ -22,8 +22,6 @@ import ( "regexp" "strings" - "android/soong/bazel" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -122,26 +120,13 @@ type BaseProperties struct { Embedded_launcher *bool `blueprint:"mutated"` } -type baseAttributes struct { - // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv - //Pkg_path bazel.StringAttribute - // TODO: Related to Pkg_bath and similarLy gated - //Is_internal bazel.BoolAttribute - // Combines Srcs and Exclude_srcs - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - // Combines Data and Java_data (invariant) - Data bazel.LabelListAttribute - Imports bazel.StringListAttribute -} - // Used to store files of current module after expanding dependencies type pathMapping struct { dest string src android.Path } -type Module struct { +type PythonLibraryModule struct { android.ModuleBase android.DefaultableModuleBase android.BazelModuleBase @@ -153,16 +138,6 @@ type Module struct { hod android.HostOrDeviceSupported multilib android.Multilib - // interface used to bootstrap .par executable when embedded_launcher is true - // this should be set by Python modules which are runnable, e.g. binaries and tests - // bootstrapper might be nil (e.g. Python library module). - bootstrapper bootstrapper - - // interface that implements functions required for installation - // this should be set by Python modules which are runnable, e.g. binaries and tests - // installer might be nil (e.g. Python library module). - installer installer - // the Python files of current module after expanding source dependencies. // pathMapping: srcsPathMappings []pathMapping @@ -173,110 +148,16 @@ type Module struct { // the zip filepath for zipping current module source/data files. srcsZip android.Path - - // dependency modules' zip filepath for zipping current module source/data files. - depsSrcsZips android.Paths - - // (.intermediate) module output path as installation source. - installSource android.OptionalPath - - // Map to ensure sub-part of the AndroidMk for this module is only added once - subAndroidMkOnce map[subAndroidMkProvider]bool } // newModule generates new Python base module -func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { - return &Module{ +func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *PythonLibraryModule { + return &PythonLibraryModule{ hod: hod, multilib: multilib, } } -func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes { - var attrs baseAttributes - archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{}) - for axis, configToProps := range archVariantBaseProps { - for config, props := range configToProps { - if baseProps, ok := props.(*BaseProperties); ok { - attrs.Srcs.SetSelectValue(axis, config, - android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs)) - attrs.Deps.SetSelectValue(axis, config, - android.BazelLabelForModuleDeps(ctx, baseProps.Libs)) - data := android.BazelLabelForModuleSrc(ctx, baseProps.Data) - data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data)) - attrs.Data.SetSelectValue(axis, config, data) - } - } - } - - partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ - "proto": android.ProtoSrcLabelPartition, - "py": bazel.LabelPartition{Keep_remainder: true}, - }) - attrs.Srcs = partitionedSrcs["py"] - - if !partitionedSrcs["proto"].IsEmpty() { - protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) - protoLabel := bazel.Label{Label: ":" + protoInfo.Name} - - pyProtoLibraryName := m.Name() + "_py_proto" - ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ - Rule_class: "py_proto_library", - Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", - }, android.CommonAttributes{ - Name: pyProtoLibraryName, - }, &bazelPythonProtoLibraryAttributes{ - Deps: bazel.MakeSingleLabelListAttribute(protoLabel), - }) - - attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) - } - - // Bazel normally requires `import path.from.top.of.tree` statements in - // python code, but with soong you can directly import modules from libraries. - // Add "imports" attributes to the bazel library so it matches soong's behavior. - imports := "." - if m.properties.Pkg_path != nil { - // TODO(b/215119317) This is a hack to handle the fact that we don't convert - // pkg_path properly right now. If the folder structure that contains this - // Android.bp file matches pkg_path, we can set imports to an appropriate - // number of ../..s to emulate moving the files under a pkg_path folder. - pkg_path := filepath.Clean(*m.properties.Pkg_path) - if strings.HasPrefix(pkg_path, "/") { - ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) - } - - if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { - ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) - } - numFolders := strings.Count(pkg_path, "/") + 1 - dots := make([]string, numFolders) - for i := 0; i < numFolders; i++ { - dots[i] = ".." - } - imports = strings.Join(dots, "/") - } - attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) - - return attrs -} - -// bootstrapper interface should be implemented for runnable modules, e.g. binary and test -type bootstrapper interface { - bootstrapperProps() []interface{} - bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool, - srcsPathMappings []pathMapping, srcsZip android.Path, - depsSrcsZips android.Paths) android.OptionalPath - - autorun() bool -} - -// installer interface should be implemented for installable modules, e.g. binary and test -type installer interface { - install(ctx android.ModuleContext, path android.Path) - setAndroidMkSharedLibs(sharedLibs []string) -} - // interface implemented by Python modules to provide source and data mappings and zip to python // modules that depend on it type pythonDependency interface { @@ -286,37 +167,31 @@ type pythonDependency interface { } // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination -func (p *Module) getSrcsPathMappings() []pathMapping { +func (p *PythonLibraryModule) getSrcsPathMappings() []pathMapping { return p.srcsPathMappings } // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination -func (p *Module) getDataPathMappings() []pathMapping { +func (p *PythonLibraryModule) getDataPathMappings() []pathMapping { return p.dataPathMappings } // getSrcsZip returns the filepath where the current module's source/data files are zipped. -func (p *Module) getSrcsZip() android.Path { +func (p *PythonLibraryModule) getSrcsZip() android.Path { return p.srcsZip } -var _ pythonDependency = (*Module)(nil) +func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { + return &p.properties +} -var _ android.AndroidMkEntriesProvider = (*Module)(nil) +var _ pythonDependency = (*PythonLibraryModule)(nil) -func (p *Module) init(additionalProps ...interface{}) android.Module { +func (p *PythonLibraryModule) init() android.Module { p.AddProperties(&p.properties, &p.protoProperties) - - // Add additional properties for bootstrapping/installation - // This is currently tied to the bootstrapper interface; - // however, these are a combination of properties for the installation and bootstrapping of a module - if p.bootstrapper != nil { - p.AddProperties(p.bootstrapper.bootstrapperProps()...) - } - android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) - + android.InitBazelModule(p) return p } @@ -350,24 +225,29 @@ var ( internalPath = "internal" ) +type basePropertiesProvider interface { + getBaseProperties() *BaseProperties +} + // versionSplitMutator creates version variants for modules and appends the version-specific // properties for a given variant to the properties in the variant module func versionSplitMutator() func(android.BottomUpMutatorContext) { return func(mctx android.BottomUpMutatorContext) { - if base, ok := mctx.Module().(*Module); ok { - versionNames := []string{} + if base, ok := mctx.Module().(basePropertiesProvider); ok { + props := base.getBaseProperties() + var versionNames []string // collect version specific properties, so that we can merge version-specific properties // into the module's overall properties - versionProps := []VersionProperties{} + var versionProps []VersionProperties // PY3 is first so that we alias the PY3 variant rather than PY2 if both // are available - if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) { + if proptools.BoolDefault(props.Version.Py3.Enabled, true) { versionNames = append(versionNames, pyVersion3) - versionProps = append(versionProps, base.properties.Version.Py3) + versionProps = append(versionProps, props.Version.Py3) } - if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) { + if proptools.BoolDefault(props.Version.Py2.Enabled, false) { versionNames = append(versionNames, pyVersion2) - versionProps = append(versionProps, base.properties.Version.Py2) + versionProps = append(versionProps, props.Version.Py2) } modules := mctx.CreateLocalVariations(versionNames...) // Alias module to the first variant @@ -376,9 +256,10 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } for i, v := range versionNames { // set the actual version for Python module. - modules[i].(*Module).properties.Actual_version = v + newProps := modules[i].(basePropertiesProvider).getBaseProperties() + newProps.Actual_version = v // append versioned properties for the Python module to the overall properties - err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil) + err := proptools.AppendMatchingProperties([]interface{}{newProps}, &versionProps[i], nil) if err != nil { panic(err) } @@ -387,38 +268,6 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } } -// HostToolPath returns a path if appropriate such that this module can be used as a host tool, -// fulfilling HostToolProvider interface. -func (p *Module) HostToolPath() android.OptionalPath { - if p.installer != nil { - if bin, ok := p.installer.(*binaryDecorator); ok { - // TODO: This should only be set when building host binaries -- tests built for device would be - // setting this incorrectly. - return android.OptionalPathForPath(bin.path) - } - } - - return android.OptionalPath{} - -} - -// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. -func (p *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if outputFile := p.installSource; outputFile.Valid() { - return android.Paths{outputFile.Path()}, nil - } - return android.Paths{}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - -func (p *Module) isEmbeddedLauncherEnabled() bool { - return p.installer != nil && Bool(p.properties.Embedded_launcher) -} - func anyHasExt(paths []string, ext string) bool { for _, p := range paths { if filepath.Ext(p) == ext { @@ -429,7 +278,7 @@ func anyHasExt(paths []string, ext string) bool { return false } -func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { +func (p *PythonLibraryModule) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { return anyHasExt(p.properties.Srcs, ext) } @@ -437,7 +286,7 @@ func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bo // - handles proto dependencies, // - if required, specifies launcher and adds launcher dependencies, // - applies python version mutations to Python dependencies -func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { +func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) { android.ProtoDeps(ctx, &p.protoProperties) versionVariation := []blueprint.Variation{ @@ -452,111 +301,15 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { // Add python library dependencies for this python version variation ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...) - // If this module will be installed and has an embedded launcher, we need to add dependencies for: - // * standard library - // * launcher - // * shared dependencies of the launcher - if p.installer != nil && p.isEmbeddedLauncherEnabled() { - var stdLib string - var launcherModule string - // Add launcher shared lib dependencies. Ideally, these should be - // derived from the `shared_libs` property of the launcher. However, we - // cannot read the property at this stage and it will be too late to add - // dependencies later. - launcherSharedLibDeps := []string{ - "libsqlite", - } - // Add launcher-specific dependencies for bionic - if ctx.Target().Os.Bionic() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") - } - if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") - } - - switch p.properties.Actual_version { - case pyVersion2: - stdLib = "py2-stdlib" - - launcherModule = "py2-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py2-launcher-autorun" - } - - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") - - case pyVersion3: - stdLib = "py3-stdlib" - - launcherModule = "py3-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py3-launcher-autorun" - } - if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl { - launcherModule += "-static" - } - - if ctx.Device() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") - } - default: - panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", - p.properties.Actual_version, ctx.ModuleName())) - } - ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib) - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...) - } - // Emulate the data property for java_data but with the arch variation overridden to "common" // so that it can point to java modules. javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}} ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...) } -func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { - p.generatePythonBuildActions(ctx) - - // Only Python binary and test modules have non-empty bootstrapper. - if p.bootstrapper != nil { - // if the module is being installed, we need to collect all transitive dependencies to embed in - // the final par - p.collectPathsFromTransitiveDeps(ctx) - // bootstrap the module, including resolving main file, getting launcher path, and - // registering actions to build the par file - // bootstrap returns the binary output path - p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version, - p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) - } - - // Only Python binary and test modules have non-empty installer. - if p.installer != nil { - var sharedLibs []string - // if embedded launcher is enabled, we need to collect the shared library depenendencies of the - // launcher - for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { - sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) - } - - p.installer.setAndroidMkSharedLibs(sharedLibs) - - // Install the par file from installSource - if p.installSource.Valid() { - p.installer.install(ctx, p.installSource.Path()) - } - } -} - -// generatePythonBuildActions performs build actions common to all Python modules -func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) { +// GenerateAndroidBuildActions performs build actions common to all Python modules +func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) - requiresSrcs := true - if p.bootstrapper != nil && !p.bootstrapper.autorun() { - requiresSrcs = false - } - if len(expandedSrcs) == 0 && requiresSrcs { - ctx.ModuleErrorf("doesn't have any source files!") - } // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) @@ -607,7 +360,7 @@ func isValidPythonPath(path string) error { // For this module, generate unique pathMappings: // for python/data files expanded from properties. -func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string, +func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, pkgPath string, expandedSrcs, expandedData android.Paths) { // fetch pairs from "src" and "data" properties to // check current module duplicates. @@ -642,7 +395,7 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string } // createSrcsZip registers build actions to zip current module's sources and data. -func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { +func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { relativeRootMap := make(map[string]android.Paths) pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) @@ -654,13 +407,8 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi if path.src.Ext() == protoExt { protoSrcs = append(protoSrcs, path.src) } else { - var relativeRoot string - relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel()) - if v, found := relativeRootMap[relativeRoot]; found { - relativeRootMap[relativeRoot] = append(v, path.src) - } else { - relativeRootMap[relativeRoot] = android.Paths{path.src} - } + relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel()) + relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src) } } var zips android.Paths @@ -736,30 +484,20 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi } } -// isPythonLibModule returns whether the given module is a Python library Module or not +// isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not func isPythonLibModule(module blueprint.Module) bool { - if m, ok := module.(*Module); ok { - return m.isLibrary() + if _, ok := module.(*PythonLibraryModule); ok { + if _, ok := module.(*PythonBinaryModule); !ok { + return true + } } return false } -// This is distinguished by the fact that Python libraries are not installable, while other Python -// modules are. -func (p *Module) isLibrary() bool { - // Python library has no bootstrapper or installer - return p.bootstrapper == nil && p.installer == nil -} - -func (p *Module) isBinary() bool { - _, ok := p.bootstrapper.(*binaryDecorator) - return ok -} - // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths // for module and its transitive dependencies and collects list of data/source file // zips for transitive dependencies. -func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { +func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext) android.Paths { // fetch pairs from "src" and "data" properties to // check duplicates. destToPySrcs := make(map[string]string) @@ -773,6 +511,8 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { seen := make(map[android.Module]bool) + var result android.Paths + // visit all its dependencies in depth first. ctx.WalkDeps(func(child, parent android.Module) bool { // we only collect dependencies tagged as python library deps @@ -801,10 +541,11 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { checkForDuplicateOutputPath(ctx, destToPyData, path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) } - p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip()) + result = append(result, dep.getSrcsZip()) } return true }) + return result } // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which @@ -825,18 +566,10 @@ func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, } // InstallInData returns true as Python is not supported in the system partition -func (p *Module) InstallInData() bool { +func (p *PythonLibraryModule) InstallInData() bool { return true } -func (p *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - if p.isLibrary() { - pythonLibBp2Build(ctx, p) - } else if p.isBinary() { - pythonBinaryBp2Build(ctx, p) - } -} - var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String diff --git a/python/python_test.go b/python/python_test.go index 42a1ffb2c..6f4223a7c 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -312,10 +312,6 @@ var ( "e/file4.py", }, srcsZip: "out/soong/.intermediates/dir/bin/PY3/bin.py.srcszip", - depsSrcsZips: []string{ - "out/soong/.intermediates/dir/lib5/PY3/lib5.py.srcszip", - "out/soong/.intermediates/dir/lib6/PY3/lib6.py.srcszip", - }, }, }, }, @@ -346,17 +342,17 @@ func TestPythonModule(t *testing.T) { for _, e := range d.expectedBinaries { t.Run(e.name, func(t *testing.T) { - expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles, e.depsSrcsZips) + expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles) }) } }) } } -func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles, expectedDepsSrcsZips []string) { +func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles []string) { module := ctx.ModuleForTests(name, variant) - base, baseOk := module.Module().(*Module) + base, baseOk := module.Module().(*PythonLibraryModule) if !baseOk { t.Fatalf("%s is not Python module!", name) } @@ -369,8 +365,6 @@ func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expecte android.AssertDeepEquals(t, "pyRunfiles", expectedPyRunfiles, actualPyRunfiles) android.AssertPathRelativeToTopEquals(t, "srcsZip", expectedSrcsZip, base.srcsZip) - - android.AssertPathsRelativeToTopEquals(t, "depsSrcsZips", expectedDepsSrcsZips, base.depsSrcsZips) } func TestMain(m *testing.M) { diff --git a/python/test.go b/python/test.go index fc5c2112f..fb8e91806 100644 --- a/python/test.go +++ b/python/test.go @@ -32,6 +32,20 @@ func registerPythonTestComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("python_test", PythonTestFactory) } +func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule { + return &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} +} + +func PythonTestHostFactory() android.Module { + return NewTest(android.HostSupportedNoCross).init() +} + +func PythonTestFactory() android.Module { + module := NewTest(android.HostAndDeviceSupported) + module.multilib = android.MultilibBoth + return module.init() +} + type TestProperties struct { // the name of the test configuration (for example "AndroidTest.xml") that should be // installed with the module. @@ -52,76 +66,79 @@ type TestProperties struct { Test_options android.CommonTestOptions } -type testDecorator struct { - *binaryDecorator +type PythonTestModule struct { + PythonBinaryModule testProperties TestProperties - - testConfig android.Path - - data []android.DataPath + testConfig android.Path + data []android.DataPath } -func (test *testDecorator) bootstrapperProps() []interface{} { - return append(test.binaryDecorator.bootstrapperProps(), &test.testProperties) +func (p *PythonTestModule) init() android.Module { + p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.binaryProperties) + p.AddProperties(&p.testProperties) + android.InitAndroidArchModule(p, p.hod, p.multilib) + android.InitDefaultableModule(p) + android.InitBazelModule(p) + if p.hod == android.HostSupportedNoCross && p.testProperties.Test_options.Unit_test == nil { + p.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) + } + return p } -func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) { - test.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ - TestConfigProp: test.testProperties.Test_config, - TestConfigTemplateProp: test.testProperties.Test_config_template, - TestSuites: test.binaryDecorator.binaryProperties.Test_suites, - AutoGenConfig: test.binaryDecorator.binaryProperties.Auto_gen_config, +func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // We inherit from only the library's GenerateAndroidBuildActions, and then + // just use buildBinary() so that the binary is not installed into the location + // it would be for regular binaries. + p.PythonLibraryModule.GenerateAndroidBuildActions(ctx) + p.buildBinary(ctx) + + p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ + TestConfigProp: p.testProperties.Test_config, + TestConfigTemplateProp: p.testProperties.Test_config_template, + TestSuites: p.binaryProperties.Test_suites, + AutoGenConfig: p.binaryProperties.Auto_gen_config, DeviceTemplate: "${PythonBinaryHostTestConfigTemplate}", HostTemplate: "${PythonBinaryHostTestConfigTemplate}", }) - test.binaryDecorator.pythonInstaller.dir = "nativetest" - test.binaryDecorator.pythonInstaller.dir64 = "nativetest64" + p.installedDest = ctx.InstallFile(installDir(ctx, "nativetest", "nativetest64", ctx.ModuleName()), p.installSource.Base(), p.installSource) - test.binaryDecorator.pythonInstaller.relative = ctx.ModuleName() - - test.binaryDecorator.pythonInstaller.install(ctx, file) - - dataSrcPaths := android.PathsForModuleSrc(ctx, test.testProperties.Data) - - for _, dataSrcPath := range dataSrcPaths { - test.data = append(test.data, android.DataPath{SrcPath: dataSrcPath}) + for _, dataSrcPath := range android.PathsForModuleSrc(ctx, p.testProperties.Data) { + p.data = append(p.data, android.DataPath{SrcPath: dataSrcPath}) } // Emulate the data property for java_data dependencies. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") { - test.data = append(test.data, android.DataPath{SrcPath: javaDataSrcPath}) + p.data = append(p.data, android.DataPath{SrcPath: javaDataSrcPath}) } } } -func NewTest(hod android.HostOrDeviceSupported) *Module { - module, binary := NewBinary(hod) - - binary.pythonInstaller = NewPythonInstaller("nativetest", "nativetest64") - - test := &testDecorator{binaryDecorator: binary} - if hod == android.HostSupportedNoCross && test.testProperties.Test_options.Unit_test == nil { - test.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) +func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := p.PythonBinaryModule.AndroidMkEntries() + if len(entriesList) != 1 { + panic("Expected 1 entry") } + entries := &entriesList[0] - module.bootstrapper = test - module.installer = test + entries.Class = "NATIVE_TESTS" - return module -} - -func PythonTestHostFactory() android.Module { - module := NewTest(android.HostSupportedNoCross) - - return module.init() -} - -func PythonTestFactory() android.Module { - module := NewTest(android.HostAndDeviceSupported) - module.multilib = android.MultilibBoth - - return module.init() + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + //entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) + if p.testConfig != nil { + entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) + } + + entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) + + entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...) + + p.testProperties.Test_options.SetAndroidMkEntries(entries) + }) + + return entriesList }