Merge "Add some documentation and simplify data model." am: aff9f3a77c

Original change: https://android-review.googlesource.com/c/platform/build/+/2277208

Change-Id: I85e46720b805af3e755c649964389b1fa091198e
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

tools/compliance/README.md

@@ -0,0 +1,101 @@
+# Compliance
+
+<!-- Much of this content appears too in doc.go
+When changing this file consider whether the change also applies to doc.go -->
+
+Package compliance provides an approved means for reading, consuming, and
+analyzing license metadata graphs.
+
+Assuming the license metadata and dependencies are fully and accurately
+recorded in the build system, any discrepancy between the official policy for
+open source license compliance and this code is **a bug in this code.**
+
+## Naming
+
+All of the code that directly reflects a policy decision belongs in a file with
+a name begninning `policy_`. Changes to these files need to be authored or
+reviewed by someone in OSPO or whichever successor group governs policy.
+
+The files with names not beginning `policy_` describe data types, and general,
+reusable algorithms.
+
+The source code for binary tools and utilities appears under the `cmd/`
+subdirectory. Other subdirectories contain reusable components that are not
+`compliance` per se.
+
+## Data Types
+
+A few principal types to understand are LicenseGraph, LicenseCondition, and
+ResolutionSet.
+
+### LicenseGraph
+
+A LicenseGraph is an immutable graph of the targets and dependencies reachable
+from a specific set of root targets. In general, the root targets will be the
+artifacts in a release or distribution. While conceptually immutable, parts of
+the graph may be loaded or evaluated lazily.
+
+Conceptually, the graph itself will always be a directed acyclic graph. One
+representation is a set of directed edges. Another is a set of nodes with
+directed edges to their dependencies.
+
+The edges have annotations, which can distinguish between build tools, runtime
+dependencies, and dependencies like 'contains' that make a derivative work.
+
+### LicenseCondition
+
+A LicenseCondition is an immutable tuple pairing a condition name with an
+originating target. e.g. Per current policy, a static library licensed under an
+MIT license would pair a "notice" condition with the static library target, and
+a dynamic license licensed under GPL would pair a "restricted" condition with
+the dynamic library target.
+
+### ResolutionSet
+
+A ResolutionSet is an immutable set of `AttachesTo`, `ActsOn`, `Resolves`
+tuples describing how license conditions apply to targets.
+
+`AttachesTo` is the trigger for acting. Distribution of the target invokes
+the policy.
+
+`ActsOn` is the target to share, give notice for, hide etc.
+
+`Resolves` is the set of conditions that the action resolves.
+
+For most condition types, `ActsOn` will be the target where the condition
+originated. For example, a notice condition policy means attribution or notice
+must be given for the target where the condition originates. Likewise, a
+proprietary condition policy means the privacy of the target where the
+condition originates must be respected. i.e. The thing acted on is the origin.
+
+Restricted conditions are different. The infectious nature of restricted often
+means sharing code that is not the target where the restricted condition
+originates. Linking an MIT library to a GPL library implies a policy to share
+the MIT library despite the MIT license having no source sharing requirement.
+
+In this case, one or more resolution tuples will have the MIT license module in
+`ActsOn` and the restricted condition originating at the GPL library module in
+`Resolves`. These tuples will `AttachTo` every target that depends on the GPL
+library because shipping any of those targets trigger the policy to share the
+code.
+
+## Processes
+
+### ReadLicenseGraph
+
+The principal means to ingest license metadata. Given the distribution targets,
+ReadLicenseGraph populates the LicenseGraph for those root targets.
+
+### NoticeIndex.IndexLicenseTexts
+
+IndexLicenseTexts reads, deduplicates and caches license texts for notice
+files. Also reads and caches project metadata for deriving library names.
+
+The algorithm for deriving library names has not been dictated by OSPO policy,
+but reflects a pragmatic attempt to comply with Android policy regarding
+unreleased product names, proprietary partner names etc.
+
+### projectmetadata.Index.MetadataForProjects
+
+MetadataForProjects reads, deduplicates and caches project METADATA files used
+for notice library names, and various properties appearing in SBOMs.

tools/compliance/cmd/testdata/proprietary/bin/bin3.meta_lic

@@ -2,7 +2,7 @@ package_name:  "Compiler"
 module_classes: "EXECUTABLES"
 projects:  "standalone/binary"
 license_kinds:  "SPDX-license-identifier-LGPL-2.0"
-license_conditions:  "restricted"
+license_conditions:  "restricted_allows_dynamic_linking"
 license_texts:  "testdata/restricted/RESTRICTED_LICENSE"
 is_container:  false
 built:  "out/target/product/fictional/obj/EXECUTABLES/bin_intermediates/bin3"

tools/compliance/cmd/testdata/restricted/bin/bin3.meta_lic

@@ -2,7 +2,7 @@ package_name:  "Compiler"
 module_classes: "EXECUTABLES"
 projects:  "standalone/binary"
 license_kinds:  "SPDX-license-identifier-LGPL-2.0"
-license_conditions:  "restricted"
+license_conditions:  "restricted_allows_dynamic_linking"
 license_texts:  "testdata/restricted/RESTRICTED_LICENSE"
 is_container:  false
 built:  "out/target/product/fictional/obj/EXECUTABLES/bin_intermediates/bin3"

tools/compliance/cmd/testdata/restricted/lib/liba.so.meta_lic

@@ -1,7 +1,7 @@
 package_name:  "Device"
 projects:  "device/library"
 license_kinds:  "SPDX-license-identifier-LGPL-2.0"
-license_conditions:  "restricted"
+license_conditions:  "restricted_allows_dynamic_linking"
 license_texts:  "testdata/restricted/RESTRICTED_LICENSE"
 is_container:  false
 built:  "out/target/product/fictional/obj/SHARED_LIBRARIES/lib_intermediates/liba.so"

tools/compliance/doc.go

@@ -11,6 +11,10 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+// Much of this content appears too in README.md
+// When changing this file consider whether the change also applies to README.md
+
 /*
 
 Package compliance provides an approved means for reading, consuming, and
@@ -31,6 +35,13 @@ from a specific set of root targets. In general, the root targets will be the
 artifacts in a release or distribution. While conceptually immutable, parts of
 the graph may be loaded or evaluated lazily.
 
+Conceptually, the graph itself will always be a directed acyclic graph. One
+representation is a set of directed edges. Another is a set of nodes with
+directed edges to their dependencies.
+
+The edges have annotations, which can distinguish between build tools, runtime
+dependencies, and dependencies like 'contains' that make a derivative work.
+
 LicenseCondition
 ----------------
 
@@ -51,17 +62,13 @@ the policy.
 
 `ActsOn` is the target to share, give notice for, hide etc.
 
-`Resolves` is the license condition that the action resolves.
+`Resolves` is the set of condition types that the action resolves.
 
-Remember: Each license condition pairs a condition name with an originating
+For most condition types, `ActsOn` will be the target where the condition
-target so each resolution in a ResolutionSet has two targets it applies to and
+originated. For example, a notice condition policy means attribution or notice
-one target from which it originates, all of which may be the same target.
+must be given for the target where the condition originates. Likewise, a
-
+proprietary condition policy means the privacy of the target where the
-For most condition types, `ActsOn` and `Resolves.Origin` will be the same
+condition originates must be respected. i.e. The thing acted on is the origin.
-target. For example, a notice condition policy means attribution or notice must
-be given for the target where the condition originates. Likewise, a proprietary
-condition policy means the privacy of the target where the condition originates
-must be respected. i.e. The thing acted on is the origin.
 
 Restricted conditions are different. The infectious nature of restricted often
 means sharing code that is not the target where the restricted condition

tools/compliance/graph.go

@@ -300,23 +300,6 @@ func (tn *TargetNode) PackageName() string {
 	return tn.proto.GetPackageName()
 }
 
-// ModuleTypes returns the list of module types implementing the target.
-// (unordered)
-//
-// In an ideal world, only 1 module type would implement each target, but the
-// interactions between Soong and Make for host versus product and for a
-// variety of architectures sometimes causes multiple module types per target
-// (often a regular build target and a prebuilt.)
-func (tn *TargetNode) ModuleTypes() []string {
-	return append([]string{}, tn.proto.ModuleTypes...)
-}
-
-// ModuleClasses returns the list of module classes implementing the target.
-// (unordered)
-func (tn *TargetNode) ModuleClasses() []string {
-	return append([]string{}, tn.proto.ModuleClasses...)
-}
-
 // Projects returns the projects defining the target node. (unordered)
 //
 // In an ideal world, only 1 project defines a target, but the interaction
@@ -326,14 +309,6 @@ func (tn *TargetNode) Projects() []string {
 	return append([]string{}, tn.proto.Projects...)
 }
 
-// LicenseKinds returns the list of license kind names for the module or
-// target. (unordered)
-//
-// e.g. SPDX-license-identifier-MIT or legacy_proprietary
-func (tn *TargetNode) LicenseKinds() []string {
-	return append([]string{}, tn.proto.LicenseKinds...)
-}
-
 // LicenseConditions returns a copy of the set of license conditions
 // originating at the target. The values that appear and how each is resolved
 // is a matter of policy. (unordered)

tools/compliance/policy_policy.go

@@ -117,32 +117,6 @@ func init() {
 func LicenseConditionSetFromNames(tn *TargetNode, names ...string) LicenseConditionSet {
 	cs := NewLicenseConditionSet()
 	for _, name := range names {
-		if name == "restricted" {
-			if 0 == len(tn.LicenseKinds()) {
-				cs = cs.Plus(RestrictedCondition)
-				continue
-			}
-			hasLgpl := false
-			hasGeneric := false
-			for _, kind := range tn.LicenseKinds() {
-				if anyLgpl.MatchString(kind) {
-					cs = cs.Plus(WeaklyRestrictedCondition)
-					hasLgpl = true
-				} else if versionedGpl.MatchString(kind) {
-					cs = cs.Plus(RestrictedCondition)
-				} else if genericGpl.MatchString(kind) {
-					hasGeneric = true
-				} else if kind == "legacy_restricted" || ccBySa.MatchString(kind) {
-					cs = cs.Plus(RestrictedCondition)
-				} else {
-					cs = cs.Plus(RestrictedCondition)
-				}
-			}
-			if hasGeneric && !hasLgpl {
-				cs = cs.Plus(RestrictedCondition)
-			}
-			continue
-		}
 		if lc, ok := RecognizedConditionNames[name]; ok {
 			cs |= LicenseConditionSet(lc)
 		}

tools/compliance/policy_resolvenotices_test.go

@@ -217,10 +217,10 @@ func TestResolveNotices(t *testing.T) {
 			},
 			expectedResolutions: []res{
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"apacheBin.meta_lic", "apacheBin.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheBin.meta_lic", "apacheBin.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
-				{"apacheBin.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheBin.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheBin.meta_lic", "mitLib.meta_lic", "mitLib.meta_lic", "notice"},
-				{"apacheBin.meta_lic", "mitLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheBin.meta_lic", "mitLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -245,7 +245,7 @@ func TestResolveNotices(t *testing.T) {
 			expectedResolutions: []res{
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
 				{"apacheBin.meta_lic", "mitLib.meta_lic", "mitLib.meta_lic", "notice"},
-				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted"},
+				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -258,17 +258,17 @@ func TestResolveNotices(t *testing.T) {
 			},
 			expectedResolutions: []res{
 				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "apacheContainer.meta_lic", "notice"},
-				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheContainer.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"apacheContainer.meta_lic", "apacheBin.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheContainer.meta_lic", "apacheBin.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
-				{"apacheContainer.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheContainer.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheContainer.meta_lic", "mitLib.meta_lic", "mitLib.meta_lic", "notice"},
-				{"apacheContainer.meta_lic", "mitLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheContainer.meta_lic", "mitLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"apacheBin.meta_lic", "apacheBin.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheBin.meta_lic", "apacheBin.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
-				{"apacheBin.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheBin.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheBin.meta_lic", "mitLib.meta_lic", "mitLib.meta_lic", "notice"},
-				{"apacheBin.meta_lic", "mitLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheBin.meta_lic", "mitLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -280,11 +280,11 @@ func TestResolveNotices(t *testing.T) {
 			},
 			expectedResolutions: []res{
 				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "apacheContainer.meta_lic", "notice"},
-				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheContainer.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"apacheContainer.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"apacheContainer.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -309,7 +309,7 @@ func TestResolveNotices(t *testing.T) {
 			expectedResolutions: []res{
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
 				{"apacheBin.meta_lic", "mitLib.meta_lic", "mitLib.meta_lic", "notice"},
-				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -336,7 +336,7 @@ func TestResolveNotices(t *testing.T) {
 				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "apacheContainer.meta_lic", "notice"},
 				{"apacheContainer.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -363,7 +363,7 @@ func TestResolveNotices(t *testing.T) {
 				{"apacheContainer.meta_lic", "apacheContainer.meta_lic", "apacheContainer.meta_lic", "notice"},
 				{"apacheContainer.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
 				{"apacheBin.meta_lic", "apacheBin.meta_lic", "apacheBin.meta_lic", "notice"},
-				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted"},
+				{"lgplLib.meta_lic", "lgplLib.meta_lic", "lgplLib.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{

tools/compliance/policy_resolveshare_test.go

@@ -73,8 +73,8 @@ func TestResolveSourceSharing(t *testing.T) {
 				{"lgplBin.meta_lic", "apacheLib.meta_lic", []string{"static"}},
 			},
 			expectedResolutions: []res{
-				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted"},
+				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted_allows_dynamic_linking"},
-				{"lgplBin.meta_lic", "apacheLib.meta_lic", "lgplBin.meta_lic", "restricted"},
+				{"lgplBin.meta_lic", "apacheLib.meta_lic", "lgplBin.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -84,7 +84,7 @@ func TestResolveSourceSharing(t *testing.T) {
 				{"lgplBin.meta_lic", "apacheLib.meta_lic", []string{"dynamic"}},
 			},
 			expectedResolutions: []res{
-				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted"},
+				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{
@@ -94,7 +94,7 @@ func TestResolveSourceSharing(t *testing.T) {
 				{"lgplBin.meta_lic", "apacheLib.meta_lic", []string{"dynamic"}},
 			},
 			expectedResolutions: []res{
-				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted"},
+				{"lgplBin.meta_lic", "lgplBin.meta_lic", "lgplBin.meta_lic", "restricted_allows_dynamic_linking"},
 			},
 		},
 		{

tools/compliance/test_util.go

@@ -57,7 +57,7 @@ license_conditions: "notice"
 	LGPL = `` +
 		`package_name: "Free Library"
 license_kinds: "SPDX-license-identifier-LGPL-2.0"
-license_conditions: "restricted"
+license_conditions: "restricted_allows_dynamic_linking"
 `
 
 	// MPL starts a test metadata file for a module with MPL 2.0 reciprical licensing.