Export Soong modules and build a database from metadata from Make and Soong.

Bug: 324465531
Test: CIs
Test: m compliance-metadata.db
Change-Id: Ia1c9ab0ae874dd47969555ddbfb93405b57a651f
This commit is contained in:
Wei Li
2024-06-21 13:08:51 -07:00
parent b651aaaf9e
commit a1aa297518
6 changed files with 414 additions and 0 deletions

View File

@@ -40,6 +40,7 @@ bootstrap_go_package {
"base_module_context.go",
"build_prop.go",
"buildinfo_prop.go",
"compliance_metadata.go",
"config.go",
"test_config.go",
"configurable_properties.go",

View File

@@ -0,0 +1,314 @@
// Copyright 2024 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"bytes"
"encoding/csv"
"fmt"
"slices"
"strconv"
"strings"
"github.com/google/blueprint"
)
var (
// Constants of property names used in compliance metadata of modules
ComplianceMetadataProp = struct {
NAME string
PACKAGE string
MODULE_TYPE string
OS string
ARCH string
IS_PRIMARY_ARCH string
VARIANT string
IS_STATIC_LIB string
INSTALLED_FILES string
BUILT_FILES string
STATIC_DEPS string
STATIC_DEP_FILES string
WHOLE_STATIC_DEPS string
WHOLE_STATIC_DEP_FILES string
LICENSES string
// module_type=package
PKG_DEFAULT_APPLICABLE_LICENSES string
// module_type=license
LIC_LICENSE_KINDS string
LIC_LICENSE_TEXT string
LIC_PACKAGE_NAME string
// module_type=license_kind
LK_CONDITIONS string
LK_URL string
}{
"name",
"package",
"module_type",
"os",
"arch",
"is_primary_arch",
"variant",
"is_static_lib",
"installed_files",
"built_files",
"static_deps",
"static_dep_files",
"whole_static_deps",
"whole_static_dep_files",
"licenses",
"pkg_default_applicable_licenses",
"lic_license_kinds",
"lic_license_text",
"lic_package_name",
"lk_conditions",
"lk_url",
}
// A constant list of all property names in compliance metadata
// Order of properties here is the order of columns in the exported CSV file.
COMPLIANCE_METADATA_PROPS = []string{
ComplianceMetadataProp.NAME,
ComplianceMetadataProp.PACKAGE,
ComplianceMetadataProp.MODULE_TYPE,
ComplianceMetadataProp.OS,
ComplianceMetadataProp.ARCH,
ComplianceMetadataProp.VARIANT,
ComplianceMetadataProp.IS_STATIC_LIB,
ComplianceMetadataProp.IS_PRIMARY_ARCH,
// Space separated installed files
ComplianceMetadataProp.INSTALLED_FILES,
// Space separated built files
ComplianceMetadataProp.BUILT_FILES,
// Space separated module names of static dependencies
ComplianceMetadataProp.STATIC_DEPS,
// Space separated file paths of static dependencies
ComplianceMetadataProp.STATIC_DEP_FILES,
// Space separated module names of whole static dependencies
ComplianceMetadataProp.WHOLE_STATIC_DEPS,
// Space separated file paths of whole static dependencies
ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES,
ComplianceMetadataProp.LICENSES,
// module_type=package
ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES,
// module_type=license
ComplianceMetadataProp.LIC_LICENSE_KINDS,
ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths
ComplianceMetadataProp.LIC_PACKAGE_NAME,
// module_type=license_kind
ComplianceMetadataProp.LK_CONDITIONS,
ComplianceMetadataProp.LK_URL,
}
)
// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license,
// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility
// methods to get/set properties' values.
type ComplianceMetadataInfo struct {
properties map[string]string
}
func NewComplianceMetadataInfo() *ComplianceMetadataInfo {
return &ComplianceMetadataInfo{
properties: map[string]string{},
}
}
func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) {
if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
}
c.properties[propertyName] = value
}
func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) {
c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " ")))
}
func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string {
if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
}
return c.properties[propertyName]
}
func (c *ComplianceMetadataInfo) getAllValues() map[string]string {
return c.properties
}
var (
ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]()
)
// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata
// for different module types without accessing their private fields but through android.Module interface
// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider.
func buildComplianceMetadataProvider(ctx ModuleContext, m *ModuleBase) {
complianceMetadataInfo := ctx.ComplianceMetadataInfo()
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name())
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir())
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType())
switch ctx.ModuleType() {
case "license":
licenseModule := m.module.(*licenseModule)
complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds)
complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings())
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name))
case "license_kind":
licenseKindModule := m.module.(*licenseKindModule)
complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions)
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url)
default:
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String())
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String())
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch()))
complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir())
if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" {
complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings())
}
var installed InstallPaths
installed = append(installed, m.module.FilesToInstall()...)
installed = append(installed, m.katiInstalls.InstallPaths()...)
installed = append(installed, m.katiSymlinks.InstallPaths()...)
installed = append(installed, m.katiInitRcInstalls.InstallPaths()...)
installed = append(installed, m.katiVintfInstalls.InstallPaths()...)
complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings()))
}
ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo)
}
func init() {
RegisterComplianceMetadataSingleton(InitRegistrationContext)
}
func RegisterComplianceMetadataSingleton(ctx RegistrationContext) {
ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory)
}
var (
// sqlite3 command line tool
sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3")
// Command to import .csv files to sqlite3 database
importCsv = pctx.AndroidStaticRule("importCsv",
blueprint.RuleParams{
Command: `rm -rf $out && ` +
`${sqlite3} $out ".import --csv $in modules" && ` +
`${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` +
`${sqlite3} $out ".import --csv ${make_modules} make_modules"`,
CommandDeps: []string{"${sqlite3}"},
}, "make_metadata", "make_modules")
)
func complianceMetadataSingletonFactory() Singleton {
return &complianceMetadataSingleton{}
}
type complianceMetadataSingleton struct {
}
func writerToCsv(csvWriter *csv.Writer, row []string) {
err := csvWriter.Write(row)
if err != nil {
panic(err)
}
}
// Collect compliance metadata from all Soong modules, write to a CSV file and
// import compliance metadata from Make and Soong to a sqlite3 database.
func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) {
if !ctx.Config().HasDeviceProduct() {
return
}
var buffer bytes.Buffer
csvWriter := csv.NewWriter(&buffer)
// Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file.
columnNames := []string{"id"}
columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...)
writerToCsv(csvWriter, columnNames)
rowId := -1
ctx.VisitAllModules(func(module Module) {
if !module.Enabled(ctx) {
return
}
moduleType := ctx.ModuleType(module)
if moduleType == "package" {
metadataMap := map[string]string{
ComplianceMetadataProp.NAME: ctx.ModuleName(module),
ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module),
ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "),
}
rowId = rowId + 1
metadata := []string{strconv.Itoa(rowId)}
for _, propertyName := range COMPLIANCE_METADATA_PROPS {
metadata = append(metadata, metadataMap[propertyName])
}
writerToCsv(csvWriter, metadata)
return
}
if provider, ok := ctx.moduleProvider(module, ComplianceMetadataProvider); ok {
metadataInfo := provider.(*ComplianceMetadataInfo)
rowId = rowId + 1
metadata := []string{strconv.Itoa(rowId)}
for _, propertyName := range COMPLIANCE_METADATA_PROPS {
metadata = append(metadata, metadataInfo.getStringValue(propertyName))
}
writerToCsv(csvWriter, metadata)
return
}
})
csvWriter.Flush()
deviceProduct := ctx.Config().DeviceProduct()
modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv")
WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String())
// Metadata generated in Make
makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv")
makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv")
// Import metadata from Make and Soong to sqlite3 database
complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db")
ctx.Build(pctx, BuildParams{
Rule: importCsv,
Input: modulesCsv,
Implicits: []Path{
makeMetadataCsv,
makeModulesCsv,
},
Output: complianceMetadataDb,
Args: map[string]string{
"make_metadata": makeMetadataCsv.String(),
"make_modules": makeModulesCsv.String(),
},
})
// Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database.
ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
Inputs: []Path{complianceMetadataDb},
Output: PathForPhony(ctx, "compliance-metadata.db"),
})
}

View File

@@ -919,6 +919,10 @@ type ModuleBase struct {
// outputFiles stores the output of a module by tag and is used to set
// the OutputFilesProvider in GenerateBuildActions
outputFiles OutputFilesInfo
// complianceMetadataInfo is for different module types to dump metadata.
// See android.ModuleContext interface.
complianceMetadataInfo *ComplianceMetadataInfo
}
func (m *ModuleBase) AddJSONData(d *map[string]interface{}) {
@@ -2049,6 +2053,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
if m.outputFiles.DefaultOutputFiles != nil || m.outputFiles.TaggedOutputFiles != nil {
SetProvider(ctx, OutputFilesProvider, m.outputFiles)
}
buildComplianceMetadataProvider(ctx, m)
}
func SetJarJarPrefixHandler(handler func(ModuleContext)) {

View File

@@ -216,6 +216,11 @@ type ModuleContext interface {
// SetOutputFiles stores the outputFiles to outputFiles property, which is used
// to set the OutputFilesProvider later.
SetOutputFiles(outputFiles Paths, tag string)
// ComplianceMetadataInfo returns a ComplianceMetadataInfo instance for different module types to dump metadata,
// which usually happens in GenerateAndroidBuildActions() of a module type.
// See android.ModuleBase.complianceMetadataInfo
ComplianceMetadataInfo() *ComplianceMetadataInfo
}
type moduleContext struct {
@@ -729,6 +734,15 @@ func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) {
}
}
func (m *moduleContext) ComplianceMetadataInfo() *ComplianceMetadataInfo {
if complianceMetadataInfo := m.module.base().complianceMetadataInfo; complianceMetadataInfo != nil {
return complianceMetadataInfo
}
complianceMetadataInfo := NewComplianceMetadataInfo()
m.module.base().complianceMetadataInfo = complianceMetadataInfo
return complianceMetadataInfo
}
// Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must
// be tagged with `android:"path" to support automatic source module dependency resolution.
//

View File

@@ -2121,6 +2121,43 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
}
}
buildComplianceMetadataInfo(ctx, c, deps)
}
func buildComplianceMetadataInfo(ctx ModuleContext, c *Module, deps PathDeps) {
// Dump metadata that can not be done in android/compliance-metadata.go
complianceMetadataInfo := ctx.ComplianceMetadataInfo()
complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(ctx.static()))
complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, c.outputFile.String())
// Static deps
staticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(false))
staticDepNames := make([]string, 0, len(staticDeps))
for _, dep := range staticDeps {
staticDepNames = append(staticDepNames, dep.Name())
}
staticDepPaths := make([]string, 0, len(deps.StaticLibs))
for _, dep := range deps.StaticLibs {
staticDepPaths = append(staticDepPaths, dep.String())
}
complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames))
complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths))
// Whole static deps
wholeStaticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(true))
wholeStaticDepNames := make([]string, 0, len(wholeStaticDeps))
for _, dep := range wholeStaticDeps {
wholeStaticDepNames = append(wholeStaticDepNames, dep.Name())
}
wholeStaticDepPaths := make([]string, 0, len(deps.WholeStaticLibs))
for _, dep := range deps.WholeStaticLibs {
wholeStaticDepPaths = append(wholeStaticDepPaths, dep.String())
}
complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEPS, android.FirstUniqueStrings(wholeStaticDepNames))
complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, android.FirstUniqueStrings(wholeStaticDepPaths))
}
func (c *Module) maybeUnhideFromMake() {

View File

@@ -16,6 +16,7 @@ package rust
import (
"fmt"
"strconv"
"strings"
"android/soong/bloaty"
@@ -967,7 +968,10 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
if mod.testModule {
android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{})
}
mod.setOutputFiles(ctx)
buildComplianceMetadataInfo(ctx, mod, deps)
}
func (mod *Module) setOutputFiles(ctx ModuleContext) {
@@ -983,6 +987,44 @@ func (mod *Module) setOutputFiles(ctx ModuleContext) {
}
}
func buildComplianceMetadataInfo(ctx *moduleContext, mod *Module, deps PathDeps) {
// Dump metadata that can not be done in android/compliance-metadata.go
metadataInfo := ctx.ComplianceMetadataInfo()
metadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(mod.Static()))
metadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, mod.outputFile.String())
// Static libs
staticDeps := ctx.GetDirectDepsWithTag(rlibDepTag)
staticDepNames := make([]string, 0, len(staticDeps))
for _, dep := range staticDeps {
staticDepNames = append(staticDepNames, dep.Name())
}
ccStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(false))
for _, dep := range ccStaticDeps {
staticDepNames = append(staticDepNames, dep.Name())
}
staticDepPaths := make([]string, 0, len(deps.StaticLibs)+len(deps.RLibs))
// C static libraries
for _, dep := range deps.StaticLibs {
staticDepPaths = append(staticDepPaths, dep.String())
}
// Rust static libraries
for _, dep := range deps.RLibs {
staticDepPaths = append(staticDepPaths, dep.Path.String())
}
metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames))
metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths))
// C Whole static libs
ccWholeStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(true))
wholeStaticDepNames := make([]string, 0, len(ccWholeStaticDeps))
for _, dep := range ccStaticDeps {
wholeStaticDepNames = append(wholeStaticDepNames, dep.Name())
}
metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames))
}
func (mod *Module) deps(ctx DepsContext) Deps {
deps := Deps{}