StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance and scale.

Browse Source 
Defer edge creation.

Don't create edges until the count is known to avoid repeated allocate+
copy operatios.

Limit resolutions.

Allow only a single resolution condition set per target, and overwrite
intermediate results. Reduces memory and obviates allocations.

Propagate fewer conditions.

Instead of propagating notice conditions to parents in graph during
initial resolve, leave them on leaf node, and attach to ancestors in
the final walk. Reduces copies.

Parallelize resolutions.

Use goroutines, mutexes, and waitgroups to resolve branches of the
graph in parallel. Makes better use of available cores.

Don't accumulate resolutions inside non-containers.

During the final resolution walk, only attach actions to ancestors from
the root down until the 1st non-aggregate. Prevents an explosion of
copies in the lower levels of the graph.

Drop origin for scale.

Tracking the origin of every potential origin for every restricted
condition does not scale. By dropping origin, propagating from top
to bottom can prune many redundant paths avoiding an exponential
explosion.

Conditions as bitmask.

Use bit masks for license conditions and condition sets. Reduces maps
and allocations.

Bug: 68860345
Bug: 151177513
Bug: 151953481

Test: m all
Test: m systemlicense
Test: m listshare; out/soong/host/linux-x86/bin/listshare ...
Test: m checkshare; out/soong/host/linux-x86/bin/checkshare ...
Test: m dumpgraph; out/soong/host/linux-x86/dumpgraph ...
Test: m dumpresolutions; out/soong/host/linux-x86/dumpresolutions ...

where ... is the path to the .meta_lic file for the system image. In my
case if

$ export PRODUCT=$(realpath $ANDROID_PRODUCT_OUT --relative-to=$PWD)

... can be expressed as:

${PRODUCT}/gen/META/lic_intermediates/${PRODUCT}/system.img.meta_lic

Change-Id: Ia2ec1b818de6122c239fbd0824754f1d65daffd3
This commit is contained in:
Bob Badour
2022-01-10 13:50:57 -08:00
parent 5446a6f8e1
commit 103eb0f9bc
35 changed files with 3444 additions and 4586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
tools/compliance/readgraph.go
View File

@@ -39,16 +39,13 @@ type result struct {
// target contains the parsed metadata or nil if an error
target *TargetNode
// edges contains the parsed dependencies
edges []*dependencyEdge
// err is nil unless an error occurs
err error
}
// receiver coordinates the tasks for reading and parsing license metadata files.
type receiver struct {
// lg accumulates the read metadata and becomes the final resulting LicensGraph.
// lg accumulates the read metadata and becomes the final resulting LicenseGraph.
lg *LicenseGraph
// rootFS locates the root of the file system from which to read the files.
@@ -138,10 +135,7 @@ func ReadLicenseGraph(rootFS fs.FS, stderr io.Writer, files []string) (*LicenseG
// record the parsed metadata (guarded by mutex)
recv.lg.mu.Lock()
recv.lg.targets[r.file] = r.target
if len(r.edges) > 0 {
recv.lg.edges = append(recv.lg.edges, r.edges...)
}
lg.targets[r.target.name] = r.target
recv.lg.mu.Unlock()
} else {
// finished -- nil the results channel
@@ -150,6 +144,21 @@ func ReadLicenseGraph(rootFS fs.FS, stderr io.Writer, files []string) (*LicenseG
}
}
if lg != nil {
esize := 0
for _, tn := range lg.targets {
esize += len(tn.proto.Deps)
}
lg.edges = make(TargetEdgeList, 0, esize)
for _, tn := range lg.targets {
tn.licenseConditions = LicenseConditionSetFromNames(tn, tn.proto.LicenseConditions...)
err = addDependencies(lg, tn)
if err != nil {
return nil, fmt.Errorf("error indexing dependencies for %q: %w", tn.name, err)
}
tn.proto.Deps = []*license_metadata_proto.AnnotatedDependency{}
}
}
return lg, err
}
@@ -158,34 +167,37 @@ func ReadLicenseGraph(rootFS fs.FS, stderr io.Writer, files []string) (*LicenseG
type targetNode struct {
proto license_metadata_proto.LicenseMetadata
// name is the path to the metadata file
// name is the path to the metadata file.
name string
}
// dependencyEdge describes a single edge in the license graph.
type dependencyEdge struct {
// target identifies the target node being built and/or installed.
target string
// lg is the license graph the node belongs to.
lg *LicenseGraph
// dependency identifies the target node being depended on.
//
// i.e. `dependency` is necessary to build `target`.
dependency string
// edges identifies the dependencies of the target.
edges TargetEdgeList
// annotations are a set of text attributes attached to the edge.
//
// Policy prescribes meaning to a limited set of annotations; others
// are preserved and ignored.
annotations TargetEdgeAnnotations
// licenseConditions identifies the set of license conditions originating at the target node.
licenseConditions LicenseConditionSet
// resolution identifies the set of conditions resolved by acting on the target node.
resolution LicenseConditionSet
}
// addDependencies converts the proto AnnotatedDependencies into `edges`
func addDependencies(edges *[]*dependencyEdge, target string, dependencies []*license_metadata_proto.AnnotatedDependency) error {
for _, ad := range dependencies {
func addDependencies(lg *LicenseGraph, tn *TargetNode) error {
tn.edges = make(TargetEdgeList, 0,len(tn.proto.Deps))
for _, ad := range tn.proto.Deps {
dependency := ad.GetFile()
if len(dependency) == 0 {
return fmt.Errorf("missing dependency name")
}
dtn, ok := lg.targets[dependency]
if !ok {
return fmt.Errorf("unknown dependency name %q", dependency)
}
if dtn == nil {
return fmt.Errorf("nil dependency for name %q", dependency)
}
annotations := newEdgeAnnotations()
for _, a := range ad.Annotations {
// look up a common constant annotation string from a small map
@@ -194,7 +206,9 @@ func addDependencies(edges *[]*dependencyEdge, target string, dependencies []*li
annotations.annotations[ann] = struct{}{}
}
}
*edges = append(*edges, &dependencyEdge{target, dependency, annotations})
edge := &TargetEdge{tn, dtn, annotations}
lg.edges = append(lg.edges, edge)
tn.edges = append(tn.edges, edge)
}
return nil
}
@@ -207,50 +221,44 @@ func readFile(recv *receiver, file string) {
go func() {
f, err := recv.rootFS.Open(file)
if err != nil {
recv.results <- &result{file, nil, nil, fmt.Errorf("error opening license metadata %q: %w", file, err)}
recv.results <- &result{file, nil, fmt.Errorf("error opening license metadata %q: %w", file, err)}
return
}
// read the file
data, err := io.ReadAll(f)
if err != nil {
recv.results <- &result{file, nil, nil, fmt.Errorf("error reading license metadata %q: %w", file, err)}
recv.results <- &result{file, nil, fmt.Errorf("error reading license metadata %q: %w", file, err)}
return
}
f.Close()
tn := &TargetNode{name: file}
tn := &TargetNode{lg: recv.lg, name: file}
err = prototext.Unmarshal(data, &tn.proto)
if err != nil {
recv.results <- &result{file, nil, nil, fmt.Errorf("error license metadata %q: %w", file, err)}
recv.results <- &result{file, nil, fmt.Errorf("error license metadata %q: %w", file, err)}
return
}
edges := []*dependencyEdge{}
err = addDependencies(&edges, file, tn.proto.Deps)
if err != nil {
recv.results <- &result{file, nil, nil, fmt.Errorf("error license metadata dependency %q: %w", file, err)}
return
}
tn.proto.Deps = []*license_metadata_proto.AnnotatedDependency{}
// send result for this file and release task before scheduling dependencies,
// but do not signal done to WaitGroup until dependencies are scheduled.
recv.results <- &result{file, tn, edges, nil}
recv.results <- &result{file, tn, nil}
recv.task <- true
// schedule tasks as necessary to read dependencies
for _, e := range edges {
for _, ad := range tn.proto.Deps {
dependency := ad.GetFile()
// decide, signal and record whether to schedule task in critical section
recv.lg.mu.Lock()
_, alreadyScheduled := recv.lg.targets[e.dependency]
_, alreadyScheduled := recv.lg.targets[dependency]
if !alreadyScheduled {
recv.lg.targets[e.dependency] = nil
recv.lg.targets[dependency] = nil
}
recv.lg.mu.Unlock()
// schedule task to read dependency file outside critical section
if !alreadyScheduled {
readFile(recv, e.dependency)
readFile(recv, dependency)
}
}