From a1aa29751835c3d8949356e27f1cce92b00970ca Mon Sep 17 00:00:00 2001 From: Wei Li Date: Fri, 21 Jun 2024 13:08:51 -0700 Subject: [PATCH] Export Soong modules and build a database from metadata from Make and Soong. Bug: 324465531 Test: CIs Test: m compliance-metadata.db Change-Id: Ia1c9ab0ae874dd47969555ddbfb93405b57a651f --- android/Android.bp | 1 + android/compliance_metadata.go | 314 +++++++++++++++++++++++++++++++++ android/module.go | 6 + android/module_context.go | 14 ++ cc/cc.go | 37 ++++ rust/rust.go | 42 +++++ 6 files changed, 414 insertions(+) create mode 100644 android/compliance_metadata.go diff --git a/android/Android.bp b/android/Android.bp index ce8c9b0b3..985ffa9f1 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -40,6 +40,7 @@ bootstrap_go_package { "base_module_context.go", "build_prop.go", "buildinfo_prop.go", + "compliance_metadata.go", "config.go", "test_config.go", "configurable_properties.go", diff --git a/android/compliance_metadata.go b/android/compliance_metadata.go new file mode 100644 index 000000000..6ea66541a --- /dev/null +++ b/android/compliance_metadata.go @@ -0,0 +1,314 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "bytes" + "encoding/csv" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/google/blueprint" +) + +var ( + // Constants of property names used in compliance metadata of modules + ComplianceMetadataProp = struct { + NAME string + PACKAGE string + MODULE_TYPE string + OS string + ARCH string + IS_PRIMARY_ARCH string + VARIANT string + IS_STATIC_LIB string + INSTALLED_FILES string + BUILT_FILES string + STATIC_DEPS string + STATIC_DEP_FILES string + WHOLE_STATIC_DEPS string + WHOLE_STATIC_DEP_FILES string + LICENSES string + + // module_type=package + PKG_DEFAULT_APPLICABLE_LICENSES string + + // module_type=license + LIC_LICENSE_KINDS string + LIC_LICENSE_TEXT string + LIC_PACKAGE_NAME string + + // module_type=license_kind + LK_CONDITIONS string + LK_URL string + }{ + "name", + "package", + "module_type", + "os", + "arch", + "is_primary_arch", + "variant", + "is_static_lib", + "installed_files", + "built_files", + "static_deps", + "static_dep_files", + "whole_static_deps", + "whole_static_dep_files", + "licenses", + + "pkg_default_applicable_licenses", + + "lic_license_kinds", + "lic_license_text", + "lic_package_name", + + "lk_conditions", + "lk_url", + } + + // A constant list of all property names in compliance metadata + // Order of properties here is the order of columns in the exported CSV file. + COMPLIANCE_METADATA_PROPS = []string{ + ComplianceMetadataProp.NAME, + ComplianceMetadataProp.PACKAGE, + ComplianceMetadataProp.MODULE_TYPE, + ComplianceMetadataProp.OS, + ComplianceMetadataProp.ARCH, + ComplianceMetadataProp.VARIANT, + ComplianceMetadataProp.IS_STATIC_LIB, + ComplianceMetadataProp.IS_PRIMARY_ARCH, + // Space separated installed files + ComplianceMetadataProp.INSTALLED_FILES, + // Space separated built files + ComplianceMetadataProp.BUILT_FILES, + // Space separated module names of static dependencies + ComplianceMetadataProp.STATIC_DEPS, + // Space separated file paths of static dependencies + ComplianceMetadataProp.STATIC_DEP_FILES, + // Space separated module names of whole static dependencies + ComplianceMetadataProp.WHOLE_STATIC_DEPS, + // Space separated file paths of whole static dependencies + ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, + ComplianceMetadataProp.LICENSES, + // module_type=package + ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES, + // module_type=license + ComplianceMetadataProp.LIC_LICENSE_KINDS, + ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths + ComplianceMetadataProp.LIC_PACKAGE_NAME, + // module_type=license_kind + ComplianceMetadataProp.LK_CONDITIONS, + ComplianceMetadataProp.LK_URL, + } +) + +// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license, +// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility +// methods to get/set properties' values. +type ComplianceMetadataInfo struct { + properties map[string]string +} + +func NewComplianceMetadataInfo() *ComplianceMetadataInfo { + return &ComplianceMetadataInfo{ + properties: map[string]string{}, + } +} + +func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) { + if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { + panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) + } + c.properties[propertyName] = value +} + +func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) { + c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " "))) +} + +func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string { + if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { + panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) + } + return c.properties[propertyName] +} + +func (c *ComplianceMetadataInfo) getAllValues() map[string]string { + return c.properties +} + +var ( + ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]() +) + +// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata +// for different module types without accessing their private fields but through android.Module interface +// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider. +func buildComplianceMetadataProvider(ctx ModuleContext, m *ModuleBase) { + complianceMetadataInfo := ctx.ComplianceMetadataInfo() + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType()) + + switch ctx.ModuleType() { + case "license": + licenseModule := m.module.(*licenseModule) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name)) + case "license_kind": + licenseKindModule := m.module.(*licenseKindModule) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url) + default: + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch())) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir()) + if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" { + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings()) + } + + var installed InstallPaths + installed = append(installed, m.module.FilesToInstall()...) + installed = append(installed, m.katiInstalls.InstallPaths()...) + installed = append(installed, m.katiSymlinks.InstallPaths()...) + installed = append(installed, m.katiInitRcInstalls.InstallPaths()...) + installed = append(installed, m.katiVintfInstalls.InstallPaths()...) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings())) + } + ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo) +} + +func init() { + RegisterComplianceMetadataSingleton(InitRegistrationContext) +} + +func RegisterComplianceMetadataSingleton(ctx RegistrationContext) { + ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory) +} + +var ( + // sqlite3 command line tool + sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3") + + // Command to import .csv files to sqlite3 database + importCsv = pctx.AndroidStaticRule("importCsv", + blueprint.RuleParams{ + Command: `rm -rf $out && ` + + `${sqlite3} $out ".import --csv $in modules" && ` + + `${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` + + `${sqlite3} $out ".import --csv ${make_modules} make_modules"`, + CommandDeps: []string{"${sqlite3}"}, + }, "make_metadata", "make_modules") +) + +func complianceMetadataSingletonFactory() Singleton { + return &complianceMetadataSingleton{} +} + +type complianceMetadataSingleton struct { +} + +func writerToCsv(csvWriter *csv.Writer, row []string) { + err := csvWriter.Write(row) + if err != nil { + panic(err) + } +} + +// Collect compliance metadata from all Soong modules, write to a CSV file and +// import compliance metadata from Make and Soong to a sqlite3 database. +func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) { + if !ctx.Config().HasDeviceProduct() { + return + } + var buffer bytes.Buffer + csvWriter := csv.NewWriter(&buffer) + + // Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata//soong-modules.csv file. + columnNames := []string{"id"} + columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...) + writerToCsv(csvWriter, columnNames) + + rowId := -1 + ctx.VisitAllModules(func(module Module) { + if !module.Enabled(ctx) { + return + } + moduleType := ctx.ModuleType(module) + if moduleType == "package" { + metadataMap := map[string]string{ + ComplianceMetadataProp.NAME: ctx.ModuleName(module), + ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module), + ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "), + } + rowId = rowId + 1 + metadata := []string{strconv.Itoa(rowId)} + for _, propertyName := range COMPLIANCE_METADATA_PROPS { + metadata = append(metadata, metadataMap[propertyName]) + } + writerToCsv(csvWriter, metadata) + return + } + if provider, ok := ctx.moduleProvider(module, ComplianceMetadataProvider); ok { + metadataInfo := provider.(*ComplianceMetadataInfo) + rowId = rowId + 1 + metadata := []string{strconv.Itoa(rowId)} + for _, propertyName := range COMPLIANCE_METADATA_PROPS { + metadata = append(metadata, metadataInfo.getStringValue(propertyName)) + } + writerToCsv(csvWriter, metadata) + return + } + }) + csvWriter.Flush() + + deviceProduct := ctx.Config().DeviceProduct() + modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv") + WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String()) + + // Metadata generated in Make + makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv") + makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv") + + // Import metadata from Make and Soong to sqlite3 database + complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db") + ctx.Build(pctx, BuildParams{ + Rule: importCsv, + Input: modulesCsv, + Implicits: []Path{ + makeMetadataCsv, + makeModulesCsv, + }, + Output: complianceMetadataDb, + Args: map[string]string{ + "make_metadata": makeMetadataCsv.String(), + "make_modules": makeModulesCsv.String(), + }, + }) + + // Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database. + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Inputs: []Path{complianceMetadataDb}, + Output: PathForPhony(ctx, "compliance-metadata.db"), + }) + +} diff --git a/android/module.go b/android/module.go index a316a1b4e..eb949e484 100644 --- a/android/module.go +++ b/android/module.go @@ -919,6 +919,10 @@ type ModuleBase struct { // outputFiles stores the output of a module by tag and is used to set // the OutputFilesProvider in GenerateBuildActions outputFiles OutputFilesInfo + + // complianceMetadataInfo is for different module types to dump metadata. + // See android.ModuleContext interface. + complianceMetadataInfo *ComplianceMetadataInfo } func (m *ModuleBase) AddJSONData(d *map[string]interface{}) { @@ -2049,6 +2053,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) if m.outputFiles.DefaultOutputFiles != nil || m.outputFiles.TaggedOutputFiles != nil { SetProvider(ctx, OutputFilesProvider, m.outputFiles) } + + buildComplianceMetadataProvider(ctx, m) } func SetJarJarPrefixHandler(handler func(ModuleContext)) { diff --git a/android/module_context.go b/android/module_context.go index e2677a4f6..2e16a2478 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -216,6 +216,11 @@ type ModuleContext interface { // SetOutputFiles stores the outputFiles to outputFiles property, which is used // to set the OutputFilesProvider later. SetOutputFiles(outputFiles Paths, tag string) + + // ComplianceMetadataInfo returns a ComplianceMetadataInfo instance for different module types to dump metadata, + // which usually happens in GenerateAndroidBuildActions() of a module type. + // See android.ModuleBase.complianceMetadataInfo + ComplianceMetadataInfo() *ComplianceMetadataInfo } type moduleContext struct { @@ -729,6 +734,15 @@ func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) { } } +func (m *moduleContext) ComplianceMetadataInfo() *ComplianceMetadataInfo { + if complianceMetadataInfo := m.module.base().complianceMetadataInfo; complianceMetadataInfo != nil { + return complianceMetadataInfo + } + complianceMetadataInfo := NewComplianceMetadataInfo() + m.module.base().complianceMetadataInfo = complianceMetadataInfo + return complianceMetadataInfo +} + // Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must // be tagged with `android:"path" to support automatic source module dependency resolution. // diff --git a/cc/cc.go b/cc/cc.go index 2dfb23a19..d8fe3199b 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -2121,6 +2121,43 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { } } + + buildComplianceMetadataInfo(ctx, c, deps) +} + +func buildComplianceMetadataInfo(ctx ModuleContext, c *Module, deps PathDeps) { + // Dump metadata that can not be done in android/compliance-metadata.go + complianceMetadataInfo := ctx.ComplianceMetadataInfo() + complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(ctx.static())) + complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, c.outputFile.String()) + + // Static deps + staticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(false)) + staticDepNames := make([]string, 0, len(staticDeps)) + for _, dep := range staticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + + staticDepPaths := make([]string, 0, len(deps.StaticLibs)) + for _, dep := range deps.StaticLibs { + staticDepPaths = append(staticDepPaths, dep.String()) + } + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths)) + + // Whole static deps + wholeStaticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(true)) + wholeStaticDepNames := make([]string, 0, len(wholeStaticDeps)) + for _, dep := range wholeStaticDeps { + wholeStaticDepNames = append(wholeStaticDepNames, dep.Name()) + } + + wholeStaticDepPaths := make([]string, 0, len(deps.WholeStaticLibs)) + for _, dep := range deps.WholeStaticLibs { + wholeStaticDepPaths = append(wholeStaticDepPaths, dep.String()) + } + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEPS, android.FirstUniqueStrings(wholeStaticDepNames)) + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, android.FirstUniqueStrings(wholeStaticDepPaths)) } func (c *Module) maybeUnhideFromMake() { diff --git a/rust/rust.go b/rust/rust.go index 8dc52f070..3a3ca4d33 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -16,6 +16,7 @@ package rust import ( "fmt" + "strconv" "strings" "android/soong/bloaty" @@ -967,7 +968,10 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if mod.testModule { android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } + mod.setOutputFiles(ctx) + + buildComplianceMetadataInfo(ctx, mod, deps) } func (mod *Module) setOutputFiles(ctx ModuleContext) { @@ -983,6 +987,44 @@ func (mod *Module) setOutputFiles(ctx ModuleContext) { } } +func buildComplianceMetadataInfo(ctx *moduleContext, mod *Module, deps PathDeps) { + // Dump metadata that can not be done in android/compliance-metadata.go + metadataInfo := ctx.ComplianceMetadataInfo() + metadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(mod.Static())) + metadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, mod.outputFile.String()) + + // Static libs + staticDeps := ctx.GetDirectDepsWithTag(rlibDepTag) + staticDepNames := make([]string, 0, len(staticDeps)) + for _, dep := range staticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + ccStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(false)) + for _, dep := range ccStaticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + + staticDepPaths := make([]string, 0, len(deps.StaticLibs)+len(deps.RLibs)) + // C static libraries + for _, dep := range deps.StaticLibs { + staticDepPaths = append(staticDepPaths, dep.String()) + } + // Rust static libraries + for _, dep := range deps.RLibs { + staticDepPaths = append(staticDepPaths, dep.Path.String()) + } + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths)) + + // C Whole static libs + ccWholeStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(true)) + wholeStaticDepNames := make([]string, 0, len(ccWholeStaticDeps)) + for _, dep := range ccStaticDeps { + wholeStaticDepNames = append(wholeStaticDepNames, dep.Name()) + } + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) +} + func (mod *Module) deps(ctx DepsContext) Deps { deps := Deps{}