diff --git a/tools/compliance/Android.bp b/tools/compliance/Android.bp index 5684f2f63d..0d7cf10472 100644 --- a/tools/compliance/Android.bp +++ b/tools/compliance/Android.bp @@ -45,6 +45,13 @@ blueprint_go_binary { testSrcs: ["cmd/dumpresolutions_test.go"], } +blueprint_go_binary { + name: "htmlnotice", + srcs: ["cmd/htmlnotice.go"], + deps: ["compliance-module"], + testSrcs: ["cmd/htmlnotice_test.go"], +} + blueprint_go_binary { name: "textnotice", srcs: ["cmd/textnotice.go"], diff --git a/tools/compliance/cmd/htmlnotice.go b/tools/compliance/cmd/htmlnotice.go new file mode 100644 index 0000000000..cff1ff84eb --- /dev/null +++ b/tools/compliance/cmd/htmlnotice.go @@ -0,0 +1,216 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "compliance" + "flag" + "fmt" + "html" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +var ( + outputFile = flag.String("o", "-", "Where to write the NOTICE text file. (default stdout)") + includeTOC = flag.Bool("toc", true, "Whether to include a table of contents.") + stripPrefix = flag.String("strip_prefix", "", "Prefix to remove from paths. i.e. path to root") + title = flag.String("title", "", "The title of the notice file.") + + failNoneRequested = fmt.Errorf("\nNo license metadata files requested") + failNoLicenses = fmt.Errorf("No licenses found") +) + +type context struct { + stdout io.Writer + stderr io.Writer + rootFS fs.FS + includeTOC bool + stripPrefix string + title string +} + +func init() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...} + +Outputs an html NOTICE.html file. + +Options: +`, filepath.Base(os.Args[0])) + flag.PrintDefaults() + } +} + +func main() { + flag.Parse() + + // Must specify at least one root target. + if flag.NArg() == 0 { + flag.Usage() + os.Exit(2) + } + + if len(*outputFile) == 0 { + flag.Usage() + fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n") + os.Exit(2) + } else { + dir, err := filepath.Abs(filepath.Dir(*outputFile)) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot determine path to %q: %w\n", *outputFile, err) + os.Exit(1) + } + fi, err := os.Stat(dir) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %w\n", dir, *outputFile, err) + os.Exit(1) + } + if !fi.IsDir() { + fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile) + os.Exit(1) + } + } + + var ofile io.Writer + ofile = os.Stdout + if *outputFile != "-" { + ofile = &bytes.Buffer{} + } + + ctx := &context{ofile, os.Stderr, os.DirFS("."), *includeTOC, *stripPrefix, *title} + + err := htmlNotice(ctx, flag.Args()...) + if err != nil { + if err == failNoneRequested { + flag.Usage() + } + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } + if *outputFile != "-" { + err := os.WriteFile(*outputFile, ofile.(*bytes.Buffer).Bytes(), 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "could not write output to %q: %w\n", *outputFile, err) + os.Exit(1) + } + } + os.Exit(0) +} + +// htmlNotice implements the htmlnotice utility. +func htmlNotice(ctx *context, files ...string) error { + // Must be at least one root file. + if len(files) < 1 { + return failNoneRequested + } + + // Read the license graph from the license metadata files (*.meta_lic). + licenseGraph, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files) + if err != nil { + return fmt.Errorf("Unable to read license metadata file(s) %q: %v\n", files, err) + } + if licenseGraph == nil { + return failNoLicenses + } + + // rs contains all notice resolutions. + rs := compliance.ResolveNotices(licenseGraph) + + ni, err := compliance.IndexLicenseTexts(ctx.rootFS, licenseGraph, rs) + if err != nil { + return fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) + } + + fmt.Fprintln(ctx.stdout, "") + fmt.Fprintln(ctx.stdout, "
\n") + fmt.Fprintln(ctx.stdout, "\n") + if 0 < len(ctx.title) { + fmt.Fprintf(ctx.stdout, "", h.String()) + fmt.Fprintln(ctx.stdout, html.EscapeString(string(ni.HashText(h)))) + fmt.Fprintln(ctx.stdout, "") + } + fmt.Fprintln(ctx.stdout, "") + + return nil +} diff --git a/tools/compliance/cmd/htmlnotice_test.go b/tools/compliance/cmd/htmlnotice_test.go new file mode 100644 index 0000000000..8d3ea02703 --- /dev/null +++ b/tools/compliance/cmd/htmlnotice_test.go @@ -0,0 +1,812 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "html" + "os" + "regexp" + "strings" + "testing" +) + +var ( + horizontalRule = regexp.MustCompile(`^\s*
(.*)$`) + titleTag = regexp.MustCompile(`^\s*(.*) \s*$`) + h1Tag = regexp.MustCompile(`^\s*(.*)
\s*$`) + usedByTarget = regexp.MustCompile(`^\s*
` + html.EscapeString(text) +} + +type firstParty struct{} + +func (m firstParty) isMatch(line string) bool { + return matchesText(line, "&&&First Party License&&&") +} + +func (m firstParty) String() string { + return expectedText("&&&First Party License&&&") +} + +type notice struct{} + +func (m notice) isMatch(line string) bool { + return matchesText(line, "%%%Notice License%%%") +} + +func (m notice) String() string { + return expectedText("%%%Notice License%%%") +} + +type reciprocal struct{} + +func (m reciprocal) isMatch(line string) bool { + return matchesText(line, "$$$Reciprocal License$$$") +} + +func (m reciprocal) String() string { + return expectedText("$$$Reciprocal License$$$") +} + +type restricted struct{} + +func (m restricted) isMatch(line string) bool { + return matchesText(line, "###Restricted License###") +} + +func (m restricted) String() string { + return expectedText("###Restricted License###") +} + +type proprietary struct{} + +func (m proprietary) isMatch(line string) bool { + return matchesText(line, "@@@Proprietary License@@@") +} + +func (m proprietary) String() string { + return expectedText("@@@Proprietary License@@@") +} + +type matcherList []matcher + +func (l matcherList) String() string { + var sb strings.Builder + for _, m := range l { + s := m.String() + if s[:3] == s[len(s)-3:] { + fmt.Fprintln(&sb) + } + fmt.Fprintf(&sb, "%s\n", s) + if s[:3] == s[len(s)-3:] { + fmt.Fprintln(&sb) + } + } + return sb.String() +} diff --git a/tools/compliance/noticeindex.go b/tools/compliance/noticeindex.go index 06c2627d15..5b7f37655f 100644 --- a/tools/compliance/noticeindex.go +++ b/tools/compliance/noticeindex.go @@ -54,8 +54,8 @@ type NoticeIndex struct { text map[hash][]byte // hashLibInstall maps hashes to libraries to install paths. hashLibInstall map[hash]map[string]map[string]struct{} - // installLibHash maps install paths to libraries to hashes. - installLibHash map[string]map[string]map[hash]struct{} + // installHashLib maps install paths to libraries to hashes. + installHashLib map[string]map[hash]map[string]struct{} // libHash maps libraries to hashes. libHash map[string]map[hash]struct{} // targetHash maps target nodes to hashes. @@ -75,7 +75,7 @@ func IndexLicenseTexts(rootFS fs.FS, lg *LicenseGraph, rs ResolutionSet) (*Notic make(map[string]hash), make(map[hash][]byte), make(map[hash]map[string]map[string]struct{}), - make(map[string]map[string]map[hash]struct{}), + make(map[string]map[hash]map[string]struct{}), make(map[string]map[hash]struct{}), make(map[*TargetNode]map[hash]struct{}), make(map[string]string), @@ -115,15 +115,15 @@ func IndexLicenseTexts(rootFS fs.FS, lg *LicenseGraph, rs ResolutionSet) (*Notic ni.libHash[libName][h] = struct{}{} } for _, installPath := range installPaths { - if _, ok := ni.installLibHash[installPath]; !ok { - ni.installLibHash[installPath] = make(map[string]map[hash]struct{}) - ni.installLibHash[installPath][libName] = make(map[hash]struct{}) - ni.installLibHash[installPath][libName][h] = struct{}{} - } else if _, ok = ni.installLibHash[installPath][libName]; !ok { - ni.installLibHash[installPath][libName] = make(map[hash]struct{}) - ni.installLibHash[installPath][libName][h] = struct{}{} - } else if _, ok = ni.installLibHash[installPath][libName][h]; !ok { - ni.installLibHash[installPath][libName][h] = struct{}{} + if _, ok := ni.installHashLib[installPath]; !ok { + ni.installHashLib[installPath] = make(map[hash]map[string]struct{}) + ni.installHashLib[installPath][h] = make(map[string]struct{}) + ni.installHashLib[installPath][h][libName] = struct{}{} + } else if _, ok = ni.installHashLib[installPath][h]; !ok { + ni.installHashLib[installPath][h] = make(map[string]struct{}) + ni.installHashLib[installPath][h][libName] = struct{}{} + } else if _, ok = ni.installHashLib[installPath][h][libName]; !ok { + ni.installHashLib[installPath][h][libName] = struct{}{} } if _, ok := ni.hashLibInstall[h]; !ok { ni.hashLibInstall[h] = make(map[string]map[string]struct{}) @@ -197,7 +197,7 @@ func (ni *NoticeIndex) Hashes() chan hash { hl = append(hl, h) } if len(hl) > 0 { - sort.Sort(hashList{ni, libName, &hl}) + sort.Sort(hashList{ni, libName, "", &hl}) for _, h := range hl { c <- h } @@ -230,6 +230,46 @@ func (ni *NoticeIndex) HashLibInstalls(h hash, libName string) []string { return installs } +// InstallPaths returns the ordered channel of indexed install paths. +func (ni *NoticeIndex) InstallPaths() chan string { + c := make(chan string) + go func() { + paths := make([]string, 0, len(ni.installHashLib)) + for path := range ni.installHashLib { + paths = append(paths, path) + } + sort.Strings(paths) + for _, installPath := range paths { + c <- installPath + } + close(c) + }() + return c +} + +// InstallHashes returns the ordered array of hashes attached to `installPath`. +func (ni *NoticeIndex) InstallHashes(installPath string) []hash { + result := make([]hash, 0, len(ni.installHashLib[installPath])) + for h := range ni.installHashLib[installPath] { + result = append(result, h) + } + if len(result) > 0 { + sort.Sort(hashList{ni, "", installPath, &result}) + } + return result +} + +// InstallHashLibs returns the ordered array of library names attached to +// `installPath` as hash `h`. +func (ni *NoticeIndex) InstallHashLibs(installPath string, h hash) []string { + result := make([]string, 0, len(ni.installHashLib[installPath][h])) + for libName := range ni.installHashLib[installPath][h] { + result = append(result, libName) + } + sort.Strings(result) + return result +} + // HashText returns the file content of the license text hashed as `h`. func (ni *NoticeIndex) HashText(h hash) []byte { return ni.text[h] @@ -492,9 +532,10 @@ func (h hash) String() string { // hashList orders an array of hashes type hashList struct { - ni *NoticeIndex - libName string - hashes *[]hash + ni *NoticeIndex + libName string + installPath string + hashes *[]hash } // Len returns the count of elements in the slice. @@ -511,6 +552,14 @@ func (l hashList) Less(i, j int) bool { if 0 < len(l.libName) { insti = len(l.ni.hashLibInstall[(*l.hashes)[i]][l.libName]) instj = len(l.ni.hashLibInstall[(*l.hashes)[j]][l.libName]) + } else { + libsi := l.ni.InstallHashLibs(l.installPath, (*l.hashes)[i]) + libsj := l.ni.InstallHashLibs(l.installPath, (*l.hashes)[j]) + libsis := strings.Join(libsi, " ") + libsjs := strings.Join(libsj, " ") + if libsis != libsjs { + return libsis < libsjs + } } if insti == instj { leni := len(l.ni.text[(*l.hashes)[i]])