Add gen_notice module.

Refactor notices to support notices for multiple modules.

Enforce visibility and handle missing dependencies.

Test: m cts dist reportmissinglicenses

Test: m droid dist

Bug: 213388645
Bug: 240342946

Change-Id: Id6a81987f087419ad37d0cce57a71e8a7c4cd6e0
Merged-in: Id6a81987f087419ad37d0cce57a71e8a7c4cd6e0
Merged-in: I07963c83eaddfe363a04871e813b56fe7f1465ad
Merged-in: Ib2f8d0ad46ffe795d392166a74a7c6309b1246e2
Merged-in: I1d630c3d14d27b7605ab13d204d34b6851a24d33

android/Android.bp

@@ -49,6 +49,7 @@ bootstrap_go_package {
         "expand.go",
         "filegroup.go",
         "fixture.go",
+        "gen_notice.go",
         "hooks.go",
         "image.go",
         "license.go",
@@ -106,6 +107,7 @@ bootstrap_go_package {
         "deptag_test.go",
         "expand_test.go",
         "fixture_test.go",
+        "gen_notice_test.go",
         "license_kind_test.go",
         "license_test.go",
         "licenses_test.go",

android/gen_notice.go

@@ -0,0 +1,212 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/google/blueprint/proptools"
+)
+
+func init() {
+	RegisterGenNoticeBuildComponents(InitRegistrationContext)
+}
+
+// Register the gen_notice module type.
+func RegisterGenNoticeBuildComponents(ctx RegistrationContext) {
+	ctx.RegisterSingletonType("gen_notice_build_rules", GenNoticeBuildRulesFactory)
+	ctx.RegisterModuleType("gen_notice", GenNoticeFactory)
+}
+
+type genNoticeBuildRules struct{}
+
+func (s *genNoticeBuildRules) GenerateBuildActions(ctx SingletonContext) {
+	ctx.VisitAllModules(func(m Module) {
+		gm, ok := m.(*genNoticeModule)
+		if !ok {
+			return
+		}
+		if len(gm.missing) > 0 {
+			missingReferencesRule(ctx, gm)
+			return
+		}
+		out := BuildNoticeTextOutputFromLicenseMetadata
+		if proptools.Bool(gm.properties.Xml) {
+			out = BuildNoticeXmlOutputFromLicenseMetadata
+		} else if proptools.Bool(gm.properties.Html) {
+			out = BuildNoticeHtmlOutputFromLicenseMetadata
+		}
+		defaultName := ""
+		if len(gm.properties.For) > 0 {
+			defaultName = gm.properties.For[0]
+		}
+
+		modules := make([]Module, 0)
+		for _, name := range gm.properties.For {
+			mods := ctx.ModuleVariantsFromName(gm, name)
+			for _, mod := range mods {
+				if mod == nil {
+					continue
+				}
+				modules = append(modules, mod)
+			}
+		}
+		if ctx.Failed() {
+			return
+		}
+		out(ctx, gm.output, ctx.ModuleName(gm),
+			proptools.StringDefault(gm.properties.ArtifactName, defaultName),
+			[]string{
+				ctx.Config().OutDir() + "/",
+				ctx.Config().SoongOutDir() + "/",
+			}, modules...)
+	})
+}
+
+func GenNoticeBuildRulesFactory() Singleton {
+	return &genNoticeBuildRules{}
+}
+
+type genNoticeProperties struct {
+	// For specifies the modules for which to generate a notice file.
+	For []string
+	// ArtifactName specifies the internal name to use for the notice file.
+	// It appears in the "used by:" list for targets whose entire name is stripped by --strip_prefix.
+	ArtifactName *string
+	// Stem specifies the base name of the output file.
+	Stem *string `android:"arch_variant"`
+	// Html indicates an html-format file is needed. The default is text. Can be Html or Xml but not both.
+	Html *bool
+	// Xml indicates an xml-format file is needed. The default is text. Can be Html or Xml but not both.
+	Xml *bool
+	// Gzipped indicates the output file must be compressed with gzip. Will append .gz to suffix if not there.
+	Gzipped *bool
+	// Suffix specifies the file extension to use. Defaults to .html for html, .xml for xml, or no extension for text.
+	Suffix *string
+	// Visibility specifies where this license can be used
+	Visibility []string
+}
+
+type genNoticeModule struct {
+	ModuleBase
+	DefaultableModuleBase
+
+	properties genNoticeProperties
+
+	output  OutputPath
+	missing []string
+}
+
+func (m *genNoticeModule) DepsMutator(ctx BottomUpMutatorContext) {
+	if proptools.Bool(m.properties.Html) && proptools.Bool(m.properties.Xml) {
+		ctx.ModuleErrorf("can be html or xml but not both")
+	}
+	if !ctx.Config().AllowMissingDependencies() {
+		var missing []string
+		// Verify the modules for which to generate notices exist.
+		for _, otherMod := range m.properties.For {
+			if !ctx.OtherModuleExists(otherMod) {
+				missing = append(missing, otherMod)
+			}
+		}
+		if len(missing) == 1 {
+			ctx.PropertyErrorf("for", "no %q module exists", missing[0])
+		} else if len(missing) > 1 {
+			ctx.PropertyErrorf("for", "modules \"%s\" do not exist", strings.Join(missing, "\", \""))
+		}
+	}
+}
+
+func (m *genNoticeModule) getStem() string {
+	stem := m.base().BaseModuleName()
+	if m.properties.Stem != nil {
+		stem = proptools.String(m.properties.Stem)
+	}
+	return stem
+}
+
+func (m *genNoticeModule) getSuffix() string {
+	suffix := ""
+	if m.properties.Suffix == nil {
+		if proptools.Bool(m.properties.Html) {
+			suffix = ".html"
+		} else if proptools.Bool(m.properties.Xml) {
+			suffix = ".xml"
+		}
+	} else {
+		suffix = proptools.String(m.properties.Suffix)
+	}
+	if proptools.Bool(m.properties.Gzipped) && !strings.HasSuffix(suffix, ".gz") {
+		suffix += ".gz"
+	}
+	return suffix
+}
+
+func (m *genNoticeModule) GenerateAndroidBuildActions(ctx ModuleContext) {
+	if ctx.Config().AllowMissingDependencies() {
+		// Verify the modules for which to generate notices exist.
+		for _, otherMod := range m.properties.For {
+			if !ctx.OtherModuleExists(otherMod) {
+				m.missing = append(m.missing, otherMod)
+			}
+		}
+		m.missing = append(m.missing, ctx.GetMissingDependencies()...)
+		m.missing = FirstUniqueStrings(m.missing)
+	}
+	out := m.getStem() + m.getSuffix()
+	m.output = PathForModuleOut(ctx, out).OutputPath
+}
+
+func GenNoticeFactory() Module {
+	module := &genNoticeModule{}
+
+	base := module.base()
+	module.AddProperties(&base.nameProperties, &module.properties)
+
+	// The visibility property needs to be checked and parsed by the visibility module.
+	setPrimaryVisibilityProperty(module, "visibility", &module.properties.Visibility)
+
+	initAndroidModuleBase(module)
+	InitDefaultableModule(module)
+
+	return module
+}
+
+var _ OutputFileProducer = (*genNoticeModule)(nil)
+
+// Implements OutputFileProducer
+func (m *genNoticeModule) OutputFiles(tag string) (Paths, error) {
+	if tag == "" {
+		return Paths{m.output}, nil
+	}
+	return nil, fmt.Errorf("unrecognized tag %q", tag)
+}
+
+// missingReferencesRule emits an ErrorRule for missing module references.
+func missingReferencesRule(ctx BuilderContext, m *genNoticeModule) {
+	if len(m.missing) < 1 {
+		panic(fmt.Errorf("missing references rule requested with no missing references"))
+	}
+
+	ctx.Build(pctx, BuildParams{
+		Rule:        ErrorRule,
+		Output:      m.output,
+		Description: "notice for " + proptools.StringDefault(m.properties.ArtifactName, "container"),
+		Args: map[string]string{
+			"error": m.Name() + " references missing module(s): " + strings.Join(m.missing, ", "),
+		},
+	})
+}

android/gen_notice_test.go

@@ -0,0 +1,164 @@
+package android
+
+import (
+	"testing"
+
+	"github.com/google/blueprint"
+)
+
+var genNoticeTests = []struct {
+	name           string
+	fs             MockFS
+	expectedErrors []string
+}{
+	{
+		name: "gen_notice must not accept licenses property",
+		fs: map[string][]byte{
+			"top/Android.bp": []byte(`
+				gen_notice {
+					name: "top_license",
+					licenses: ["other_license"],
+				}`),
+		},
+		expectedErrors: []string{
+			`unrecognized property "licenses"`,
+		},
+	},
+	{
+		name: "bad gen_notice",
+		fs: map[string][]byte{
+			"top/Android.bp": []byte(`
+				gen_notice {
+					name: "top_notice",
+					for: ["top_rule"],
+				}`),
+			"other/Android.bp": []byte(`
+				mock_genrule {
+					name: "other_rule",
+					dep: ["top_notice"],
+				}`),
+		},
+		expectedErrors: []string{
+			`module "top_notice": for: no "top_rule" module exists`,
+		},
+	},
+	{
+		name: "doubly bad gen_notice",
+		fs: map[string][]byte{
+			"top/Android.bp": []byte(`
+				gen_notice {
+					name: "top_notice",
+					for: ["top_rule", "other_rule"],
+				}`),
+		},
+		expectedErrors: []string{
+			`module "top_notice": for: modules "top_rule", "other_rule" do not exist`,
+		},
+	},
+	{
+		name: "good gen_notice",
+		fs: map[string][]byte{
+			"top/Android.bp": []byte(`
+				gen_notice {
+					name: "top_notice",
+					for: ["top_rule"],
+				}
+
+				mock_genrule {
+					name: "top_rule",
+					dep: ["top_notice"],
+				}`),
+			"other/Android.bp": []byte(`
+				mock_genrule {
+					name: "other_rule",
+					dep: ["top_notice"],
+				}`),
+		},
+	},
+	{
+		name: "multiple license kinds",
+		fs: map[string][]byte{
+			"top/Android.bp": []byte(`
+				gen_notice {
+					name: "top_notice",
+					for: ["top_rule"],
+				}
+
+				gen_notice {
+					name: "top_html_notice",
+					html: true,
+					for: ["top_rule"],
+				}
+
+				gen_notice {
+					name: "top_xml_notice",
+					xml: true,
+					for: ["top_notice"],
+				}
+
+				mock_genrule {
+					name: "top_rule",
+					dep: [
+						"top_notice",
+						"top_html_notice",
+						"top_xml_notice",
+					],
+				}`),
+			"other/Android.bp": []byte(`
+				mock_genrule {
+					name: "other_rule",
+					dep: ["top_xml_notice"],
+				}`),
+		},
+	},
+}
+
+func TestGenNotice(t *testing.T) {
+	for _, test := range genNoticeTests {
+		t.Run(test.name, func(t *testing.T) {
+			GroupFixturePreparers(
+				PrepareForTestWithGenNotice,
+				FixtureRegisterWithContext(func(ctx RegistrationContext) {
+					ctx.RegisterModuleType("mock_genrule", newMockGenruleModule)
+				}),
+				test.fs.AddToFixture(),
+			).
+				ExtendWithErrorHandler(FixtureExpectsAllErrorsToMatchAPattern(test.expectedErrors)).
+				RunTest(t)
+		})
+	}
+}
+
+type mockGenruleProperties struct {
+	Dep []string
+}
+
+type mockGenruleModule struct {
+	ModuleBase
+	DefaultableModuleBase
+
+	properties mockGenruleProperties
+}
+
+func newMockGenruleModule() Module {
+	m := &mockGenruleModule{}
+	m.AddProperties(&m.properties)
+	InitAndroidArchModule(m, HostAndDeviceSupported, MultilibCommon)
+	InitDefaultableModule(m)
+	return m
+}
+
+type genruleDepTag struct {
+	blueprint.BaseDependencyTag
+}
+
+func (j *mockGenruleModule) DepsMutator(ctx BottomUpMutatorContext) {
+	m, ok := ctx.Module().(Module)
+	if !ok {
+		return
+	}
+	ctx.AddDependency(m, genruleDepTag{}, j.properties.Dep...)
+}
+
+func (p *mockGenruleModule) GenerateAndroidBuildActions(ModuleContext) {
+}

android/licenses.go

@@ -303,6 +303,7 @@ func exemptFromRequiredApplicableLicensesProperty(module Module) bool {
 	switch reflect.TypeOf(module).String() {
 	case "*android.licenseModule": // is a license, doesn't need one
 	case "*android.licenseKindModule": // is a license, doesn't need one
+	case "*android.genNoticeModule": // contains license texts as data
 	case "*android.NamespaceModule": // just partitions things, doesn't add anything
 	case "*android.soongConfigModuleTypeModule": // creates aliases for modules with licenses
 	case "*android.soongConfigModuleTypeImport": // creates aliases for modules with licenses

android/singleton.go

@@ -29,6 +29,10 @@ type SingletonContext interface {
 	ModuleType(module blueprint.Module) string
 	BlueprintFile(module blueprint.Module) string
 
+	// ModuleVariantsFromName returns the list of module variants named `name` in the same namespace as `referer` enforcing visibility rules.
+	// Allows generating build actions for `referer` based on the metadata for `name` deferred until the singleton context.
+	ModuleVariantsFromName(referer Module, name string) []Module
+
 	// ModuleProvider returns the value, if any, for the provider for a module.  If the value for the
 	// provider was not set it returns the zero value of the type of the provider, which means the
 	// return value can always be type-asserted to the type of the provider.  The return value should
@@ -251,3 +255,30 @@ func (s *singletonContextAdaptor) PrimaryModule(module Module) Module {
 func (s *singletonContextAdaptor) FinalModule(module Module) Module {
 	return s.SingletonContext.FinalModule(module).(Module)
 }
+
+func (s *singletonContextAdaptor) ModuleVariantsFromName(referer Module, name string) []Module {
+	// get qualified module name for visibility enforcement
+	qualified := createQualifiedModuleName(s.ModuleName(referer), s.ModuleDir(referer))
+
+	modules := s.SingletonContext.ModuleVariantsFromName(referer, name)
+	result := make([]Module, 0, len(modules))
+	for _, m := range modules {
+		if module, ok := m.(Module); ok {
+			// enforce visibility
+			depName := s.ModuleName(module)
+			depDir := s.ModuleDir(module)
+			depQualified := qualifiedModuleName{depDir, depName}
+			// Targets are always visible to other targets in their own package.
+			if depQualified.pkg != qualified.pkg {
+				rule := effectiveVisibilityRules(s.Config(), depQualified)
+				if !rule.matches(qualified) {
+					s.ModuleErrorf(referer, "module %q references %q which is not visible to this module\nYou may need to add %q to its visibility",
+						referer.Name(), depQualified, "//"+s.ModuleDir(referer))
+					continue
+				}
+			}
+			result = append(result, module)
+		}
+	}
+	return result
+}

android/testing.go

@@ -83,6 +83,8 @@ var PrepareForTestWithLicenses = GroupFixturePreparers(
 	FixtureRegisterWithContext(registerLicenseMutators),
 )
 
+var PrepareForTestWithGenNotice = FixtureRegisterWithContext(RegisterGenNoticeBuildComponents)
+
 func registerLicenseMutators(ctx RegistrationContext) {
 	ctx.PreArchMutators(RegisterLicensesPackageMapper)
 	ctx.PreArchMutators(RegisterLicensesPropertyGatherer)

android/visibility.go

@@ -234,7 +234,7 @@ func RegisterVisibilityRuleEnforcer(ctx RegisterMutatorsContext) {
 
 // Checks the per-module visibility rule lists before defaults expansion.
 func visibilityRuleChecker(ctx BottomUpMutatorContext) {
-	qualified := createQualifiedModuleName(ctx)
+	qualified := createQualifiedModuleName(ctx.ModuleName(), ctx.ModuleDir())
 	if m, ok := ctx.Module().(Module); ok {
 		visibilityProperties := m.visibilityProperties()
 		for _, p := range visibilityProperties {
@@ -435,7 +435,7 @@ func visibilityRuleEnforcer(ctx TopDownMutatorContext) {
 		return
 	}
 
-	qualified := createQualifiedModuleName(ctx)
+	qualified := createQualifiedModuleName(ctx.ModuleName(), ctx.ModuleDir())
 
 	// Visit all the dependencies making sure that this module has access to them all.
 	ctx.VisitDirectDeps(func(dep Module) {
@@ -486,9 +486,7 @@ func effectiveVisibilityRules(config Config, qualified qualifiedModuleName) comp
 	return rule
 }
 
-func createQualifiedModuleName(ctx BaseModuleContext) qualifiedModuleName {
+func createQualifiedModuleName(moduleName, dir string) qualifiedModuleName {
-	moduleName := ctx.ModuleName()
-	dir := ctx.ModuleDir()
 	qualified := qualifiedModuleName{dir, moduleName}
 	return qualified
 }

java/app.go

@@ -589,16 +589,6 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) {
 
 	a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx)
 
-	var noticeAssetPath android.WritablePath
-	if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") {
-		// The rule to create the notice file can't be generated yet, as the final output path
-		// for the apk isn't known yet.  Add the path where the notice file will be generated to the
-		// aapt rules now before calling aaptBuildActions, the rule to create the notice file will
-		// be generated later.
-		noticeAssetPath = android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz")
-		a.aapt.noticeFile = android.OptionalPathForPath(noticeAssetPath)
-	}
-
 	// Process all building blocks, from AAPT to certificates.
 	a.aaptBuildActions(ctx)
 
@@ -673,8 +663,7 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) {
 		a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile)
 	}
 
-	if a.aapt.noticeFile.Valid() {
+	if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") {
-		// Generating the notice file rule has to be here after a.outputFile is known.
 		noticeFile := android.PathForModuleOut(ctx, "NOTICE.html.gz")
 		android.BuildNoticeHtmlOutputFromLicenseMetadata(
 			ctx, noticeFile, "", "",
@@ -683,11 +672,13 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) {
 				android.PathForModuleInstall(ctx).String() + "/",
 				a.outputFile.String(),
 			})
+		noticeAssetPath := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz")
 		builder := android.NewRuleBuilder(pctx, ctx)
 		builder.Command().Text("cp").
 			Input(noticeFile).
 			Output(noticeAssetPath)
 		builder.Build("notice_dir", "Building notice dir")
+		a.aapt.noticeFile = android.OptionalPathForPath(noticeAssetPath)
 	}
 
 	for _, split := range a.aapt.splits {