Merge changes from topic "sdk_repo_license"

* changes:
  Generate sdk_repo notice files from license metadata
  Annotate more java dependencies for licensing
  Track transitive packaged license deps of containers

android/license_metadata.go

@@ -34,7 +34,7 @@ var (
 	}, "args")
 )
 
-func buildLicenseMetadata(ctx ModuleContext) {
+func buildLicenseMetadata(ctx ModuleContext, licenseMetadataFile WritablePath) {
 	base := ctx.Module().base()
 
 	if !base.Enabled() {
@@ -45,9 +45,18 @@ func buildLicenseMetadata(ctx ModuleContext) {
 		return
 	}
 
+	var outputFiles Paths
+	if outputFileProducer, ok := ctx.Module().(OutputFileProducer); ok {
+		outputFiles, _ = outputFileProducer.OutputFiles("")
+		outputFiles = PathsIfNonNil(outputFiles...)
+	}
+
+	isContainer := isContainerFromFileExtensions(base.installFiles, outputFiles)
+
 	var allDepMetadataFiles Paths
 	var allDepMetadataArgs []string
 	var allDepOutputFiles Paths
+	var allDepMetadataDepSets []*PathsDepSet
 
 	ctx.VisitDirectDepsBlueprint(func(bpdep blueprint.Module) {
 		dep, _ := bpdep.(Module)
@@ -61,6 +70,9 @@ func buildLicenseMetadata(ctx ModuleContext) {
 		if ctx.OtherModuleHasProvider(dep, LicenseMetadataProvider) {
 			info := ctx.OtherModuleProvider(dep, LicenseMetadataProvider).(*LicenseMetadataInfo)
 			allDepMetadataFiles = append(allDepMetadataFiles, info.LicenseMetadataPath)
+			if isContainer || IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) {
+				allDepMetadataDepSets = append(allDepMetadataDepSets, info.LicenseMetadataDepSet)
+			}
 
 			depAnnotations := licenseAnnotationsFromTag(ctx.OtherModuleDependencyTag(dep))
 
@@ -105,9 +117,16 @@ func buildLicenseMetadata(ctx ModuleContext) {
 	args = append(args,
 		JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(base.commonProperties.Effective_license_text.Strings()), "-n "))
 
-	args = append(args,
+	if isContainer {
-		JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(allDepMetadataArgs), "-d "))
+		transitiveDeps := newPathsDepSet(nil, allDepMetadataDepSets).ToList()
-	orderOnlyDeps = append(orderOnlyDeps, allDepMetadataFiles...)
+		args = append(args,
+			JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(transitiveDeps.Strings()), "-d "))
+		orderOnlyDeps = append(orderOnlyDeps, transitiveDeps...)
+	} else {
+		args = append(args,
+			JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(allDepMetadataArgs), "-d "))
+		orderOnlyDeps = append(orderOnlyDeps, allDepMetadataFiles...)
+	}
 
 	args = append(args,
 		JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(allDepOutputFiles.Strings()), "-s "))
@@ -117,12 +136,6 @@ func buildLicenseMetadata(ctx ModuleContext) {
 		JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(base.licenseInstallMap), "-m "))
 
 	// Built files
-	var outputFiles Paths
-	if outputFileProducer, ok := ctx.Module().(OutputFileProducer); ok {
-		outputFiles, _ = outputFileProducer.OutputFiles("")
-		outputFiles = PathsIfNonNil(outputFiles...)
-	}
-
 	if len(outputFiles) > 0 {
 		args = append(args,
 			JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(outputFiles.Strings()), "-t "))
@@ -134,13 +147,10 @@ func buildLicenseMetadata(ctx ModuleContext) {
 	args = append(args,
 		JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(base.installFiles.Strings()), "-i "))
 
-	isContainer := isContainerFromFileExtensions(base.installFiles, outputFiles)
 	if isContainer {
 		args = append(args, "--is_container")
 	}
 
-	licenseMetadataFile := PathForModuleOut(ctx, "meta_lic")
-
 	ctx.Build(pctx, BuildParams{
 		Rule:        licenseMetadataRule,
 		Output:      licenseMetadataFile,
@@ -152,7 +162,8 @@ func buildLicenseMetadata(ctx ModuleContext) {
 	})
 
 	ctx.SetProvider(LicenseMetadataProvider, &LicenseMetadataInfo{
-		LicenseMetadataPath: licenseMetadataFile,
+		LicenseMetadataPath:   licenseMetadataFile,
+		LicenseMetadataDepSet: newPathsDepSet(Paths{licenseMetadataFile}, allDepMetadataDepSets),
 	})
 }
 
@@ -179,7 +190,8 @@ var LicenseMetadataProvider = blueprint.NewProvider(&LicenseMetadataInfo{})
 
 // LicenseMetadataInfo stores the license metadata path for a module.
 type LicenseMetadataInfo struct {
-	LicenseMetadataPath Path
+	LicenseMetadataPath   Path
+	LicenseMetadataDepSet *PathsDepSet
 }
 
 // licenseAnnotationsFromTag returns the LicenseAnnotations for a tag (if any) converted into
@@ -212,6 +224,9 @@ const (
 	// LicenseAnnotationSharedDependency should be returned by LicenseAnnotations implementations
 	// of dependency tags when the usage of the dependency is dynamic, for example a shared library
 	// linkage for native modules or as a classpath library for java modules.
+	//
+	// Dependency tags that need to always return LicenseAnnotationSharedDependency
+	// can embed LicenseAnnotationSharedDependencyTag to implement LicenseAnnotations.
 	LicenseAnnotationSharedDependency LicenseAnnotation = "dynamic"
 
 	// LicenseAnnotationToolchain should be returned by LicenseAnnotations implementations of
@@ -222,6 +237,14 @@ const (
 	LicenseAnnotationToolchain LicenseAnnotation = "toolchain"
 )
 
+// LicenseAnnotationSharedDependencyTag can be embedded in a dependency tag to implement
+// LicenseAnnotations that always returns LicenseAnnotationSharedDependency.
+type LicenseAnnotationSharedDependencyTag struct{}
+
+func (LicenseAnnotationSharedDependencyTag) LicenseAnnotations() []LicenseAnnotation {
+	return []LicenseAnnotation{LicenseAnnotationSharedDependency}
+}
+
 // LicenseAnnotationToolchainDependencyTag can be embedded in a dependency tag to implement
 // LicenseAnnotations that always returns LicenseAnnotationToolchain.
 type LicenseAnnotationToolchainDependencyTag struct{}

android/module.go

@@ -1321,6 +1321,9 @@ type ModuleBase struct {
 	// set of dependency module:location mappings used to populate the license metadata for
 	// apex containers.
 	licenseInstallMap []string
+
+	// The path to the generated license metadata file for the module.
+	licenseMetadataFile WritablePath
 }
 
 // A struct containing all relevant information about a Bazel target converted via bp2build.
@@ -2076,6 +2079,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
 		variables:         make(map[string]string),
 	}
 
+	m.licenseMetadataFile = PathForModuleOut(ctx, "meta_lic")
+
 	dependencyInstallFiles, dependencyPackagingSpecs := m.computeInstallDeps(ctx)
 	// set m.installFilesDepSet to only the transitive dependencies to be used as the dependencies
 	// of installed files of this module.  It will be replaced by a depset including the installed
@@ -2207,7 +2212,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
 	m.installFilesDepSet = newInstallPathsDepSet(m.installFiles, dependencyInstallFiles)
 	m.packagingSpecsDepSet = newPackagingSpecsDepSet(m.packagingSpecs, dependencyPackagingSpecs)
 
-	buildLicenseMetadata(ctx)
+	buildLicenseMetadata(ctx, m.licenseMetadataFile)
 
 	m.buildParams = ctx.buildParams
 	m.ruleParams = ctx.ruleParams

android/notices.go

@@ -16,6 +16,7 @@ package android
 
 import (
 	"path/filepath"
+	"strings"
 
 	"github.com/google/blueprint"
 )
@@ -101,55 +102,15 @@ func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilena
 	}
 }
 
-// BuildNotices merges the supplied NOTICE files into a single file that lists notices
+// BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's
-// for every key in noticeMap (which would normally be installed files).
+// generated license metadata file.
-func BuildNotices(ctx ModuleContext, noticeMap map[string]Paths) NoticeOutputs {
+func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) {
-	// TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass.
+	depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", "."))
-	//
+	rule := NewRuleBuilder(pctx, ctx)
-	// generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules
+	rule.Command().
-	// about input NOTICE file paths.
+		BuiltTool("textnotice").
-	// 1. Their relative paths to the src root become their NOTICE index titles. We want to use
+		FlagWithOutput("-o ", outputFile).
-	// on-device paths as titles, and so output the merged NOTICE file the corresponding location.
+		FlagWithDepFile("-d ", depsFile).
-	// 2. They must end with .txt extension. Otherwise, they're ignored.
+		Input(ctx.Module().base().licenseMetadataFile)
-
+	rule.Build("container_notice", "container notice file")
-	mergeTool := PathForSource(ctx, "build/soong/scripts/mergenotice.py")
-	generateNoticeTool := PathForSource(ctx, "build/soong/scripts/generate-notice-files.py")
-
-	outputDir := PathForModuleOut(ctx, "notices")
-	builder := NewRuleBuilder(pctx, ctx).
-		Sbox(outputDir, PathForModuleOut(ctx, "notices.sbox.textproto"))
-	for _, installPath := range SortedStringKeys(noticeMap) {
-		noticePath := outputDir.Join(ctx, installPath+".txt")
-		// It would be nice if sbox created directories for temporaries, but until then
-		// this is simple enough.
-		builder.Command().
-			Text("(cd").OutputDir().Text("&&").
-			Text("mkdir -p").Text(filepath.Dir(installPath)).Text(")")
-		builder.Temporary(noticePath)
-		builder.Command().
-			Tool(mergeTool).
-			Flag("--output").Output(noticePath).
-			Inputs(noticeMap[installPath])
-	}
-
-	// Transform the merged NOTICE file into a gzipped HTML file.
-	txtOutput := outputDir.Join(ctx, "NOTICE.txt")
-	htmlOutput := outputDir.Join(ctx, "NOTICE.html")
-	htmlGzOutput := outputDir.Join(ctx, "NOTICE.html.gz")
-	title := "\"Notices for " + ctx.ModuleName() + "\""
-	builder.Command().Tool(generateNoticeTool).
-		FlagWithOutput("--text-output ", txtOutput).
-		FlagWithOutput("--html-output ", htmlOutput).
-		FlagWithArg("-t ", title).
-		Flag("-s").OutputDir()
-	builder.Command().BuiltTool("minigzip").
-		FlagWithInput("-c ", htmlOutput).
-		FlagWithOutput("> ", htmlGzOutput)
-	builder.Build("build_notices", "generate notice output")
-
-	return NoticeOutputs{
-		TxtOutput:    OptionalPathForPath(txtOutput),
-		HtmlOutput:   OptionalPathForPath(htmlOutput),
-		HtmlGzOutput: OptionalPathForPath(htmlGzOutput),
-	}
 }

android/paths.go

@@ -2149,3 +2149,23 @@ func IsThirdPartyPath(path string) bool {
 	}
 	return false
 }
+
+// PathsDepSet is a thin type-safe wrapper around the generic depSet.  It always uses
+// topological order.
+type PathsDepSet struct {
+	depSet
+}
+
+// newPathsDepSet returns an immutable PathsDepSet with the given direct and
+// transitive contents.
+func newPathsDepSet(direct Paths, transitive []*PathsDepSet) *PathsDepSet {
+	return &PathsDepSet{*newDepSet(TOPOLOGICAL, direct, transitive)}
+}
+
+// ToList returns the PathsDepSet flattened to a list in topological order.
+func (d *PathsDepSet) ToList() Paths {
+	if d == nil {
+		return nil
+	}
+	return d.depSet.ToList().(Paths)
+}

android_sdk/sdk_repo_host.go

@@ -122,17 +122,10 @@ func (s *sdkRepoHost) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 
 	s.CopySpecsToDir(ctx, builder, packageSpecs, dir)
 
-	// Collect licenses to write into NOTICE.txt
+	noticeFile := android.PathForModuleOut(ctx, "NOTICES.txt")
-	noticeMap := map[string]android.Paths{}
+	android.BuildNoticeTextOutputFromLicenseMetadata(ctx, noticeFile)
-	for path, pkgSpec := range packageSpecs {
-		licenseFiles := pkgSpec.EffectiveLicenseFiles()
-		if len(licenseFiles) > 0 {
-			noticeMap[path] = pkgSpec.EffectiveLicenseFiles()
-		}
-	}
-	notices := android.BuildNotices(ctx, noticeMap)
 	builder.Command().Text("cp").
-		Input(notices.TxtOutput.Path()).
+		Input(noticeFile).
 		Text(filepath.Join(dir.String(), "NOTICE.txt"))
 
 	// Handle `merge_zips` by extracting their contents into our tmpdir

dexpreopt/config.go

@@ -408,6 +408,7 @@ func dex2oatModuleName(config android.Config) string {
 
 type dex2oatDependencyTag struct {
 	blueprint.BaseDependencyTag
+	android.LicenseAnnotationToolchainDependencyTag
 }
 
 func (d dex2oatDependencyTag) ExcludeFromVisibilityEnforcement() {

java/hiddenapi.go

@@ -297,6 +297,7 @@ func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Pa
 
 type hiddenApiAnnotationsDependencyTag struct {
 	blueprint.BaseDependencyTag
+	android.LicenseAnnotationSharedDependencyTag
 }
 
 // Tag used to mark dependencies on java_library instances that contains Java source files whose

java/java.go

@@ -274,6 +274,9 @@ type dependencyTag struct {
 
 	// True if the dependency is relinked at runtime.
 	runtimeLinked bool
+
+	// True if the dependency is a toolchain, for example an annotation processor.
+	toolchain bool
 }
 
 // installDependencyTag is a dependency tag that is annotated to cause the installed files of the
@@ -287,6 +290,8 @@ type installDependencyTag struct {
 func (d dependencyTag) LicenseAnnotations() []android.LicenseAnnotation {
 	if d.runtimeLinked {
 		return []android.LicenseAnnotation{android.LicenseAnnotationSharedDependency}
+	} else if d.toolchain {
+		return []android.LicenseAnnotation{android.LicenseAnnotationToolchain}
 	}
 	return nil
 }
@@ -329,19 +334,19 @@ var (
 	staticLibTag            = dependencyTag{name: "staticlib"}
 	libTag                  = dependencyTag{name: "javalib", runtimeLinked: true}
 	java9LibTag             = dependencyTag{name: "java9lib", runtimeLinked: true}
-	pluginTag               = dependencyTag{name: "plugin"}
+	pluginTag               = dependencyTag{name: "plugin", toolchain: true}
-	errorpronePluginTag     = dependencyTag{name: "errorprone-plugin"}
+	errorpronePluginTag     = dependencyTag{name: "errorprone-plugin", toolchain: true}
-	exportedPluginTag       = dependencyTag{name: "exported-plugin"}
+	exportedPluginTag       = dependencyTag{name: "exported-plugin", toolchain: true}
 	bootClasspathTag        = dependencyTag{name: "bootclasspath", runtimeLinked: true}
 	systemModulesTag        = dependencyTag{name: "system modules", runtimeLinked: true}
 	frameworkResTag         = dependencyTag{name: "framework-res"}
 	kotlinStdlibTag         = dependencyTag{name: "kotlin-stdlib", runtimeLinked: true}
 	kotlinAnnotationsTag    = dependencyTag{name: "kotlin-annotations", runtimeLinked: true}
-	kotlinPluginTag         = dependencyTag{name: "kotlin-plugin"}
+	kotlinPluginTag         = dependencyTag{name: "kotlin-plugin", toolchain: true}
 	proguardRaiseTag        = dependencyTag{name: "proguard-raise"}
 	certificateTag          = dependencyTag{name: "certificate"}
 	instrumentationForTag   = dependencyTag{name: "instrumentation_for"}
-	extraLintCheckTag       = dependencyTag{name: "extra-lint-check"}
+	extraLintCheckTag       = dependencyTag{name: "extra-lint-check", toolchain: true}
 	jniLibTag               = dependencyTag{name: "jnilib", runtimeLinked: true}
 	syspropPublicStubDepTag = dependencyTag{name: "sysprop public stub"}
 	jniInstallTag           = installDependencyTag{name: "jni install"}