build/tools/compliance/cmd/sbom/sbom.go

// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"android/soong/response"
	"android/soong/tools/compliance"
	"android/soong/tools/compliance/projectmetadata"

	"github.com/google/blueprint/deptools"

	"github.com/spdx/tools-golang/builder/builder2v2"
	"github.com/spdx/tools-golang/json"
	"github.com/spdx/tools-golang/spdx/common"
	spdx "github.com/spdx/tools-golang/spdx/v2_2"
	"github.com/spdx/tools-golang/spdxlib"
)

var (
	failNoneRequested = fmt.Errorf("\nNo license metadata files requested")
	failNoLicenses    = fmt.Errorf("No licenses found")
)

const NOASSERTION = "NOASSERTION"

type context struct {
	stdout       io.Writer
	stderr       io.Writer
	rootFS       fs.FS
	product      string
	stripPrefix  []string
	creationTime creationTimeGetter
}

func (ctx context) strip(installPath string) string {
	for _, prefix := range ctx.stripPrefix {
		if strings.HasPrefix(installPath, prefix) {
			p := strings.TrimPrefix(installPath, prefix)
			if 0 == len(p) {
				p = ctx.product
			}
			if 0 == len(p) {
				continue
			}
			return p
		}
	}
	return installPath
}

// newMultiString creates a flag that allows multiple values in an array.
func newMultiString(flags *flag.FlagSet, name, usage string) *multiString {
	var f multiString
	flags.Var(&f, name, usage)
	return &f
}

// multiString implements the flag `Value` interface for multiple strings.
type multiString []string

func (ms *multiString) String() string     { return strings.Join(*ms, ", ") }
func (ms *multiString) Set(s string) error { *ms = append(*ms, s); return nil }

func main() {
	var expandedArgs []string
	for _, arg := range os.Args[1:] {
		if strings.HasPrefix(arg, "@") {
			f, err := os.Open(strings.TrimPrefix(arg, "@"))
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}

			respArgs, err := response.ReadRspFile(f)
			f.Close()
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
			expandedArgs = append(expandedArgs, respArgs...)
		} else {
			expandedArgs = append(expandedArgs, arg)
		}
	}

	flags := flag.NewFlagSet("flags", flag.ExitOnError)

	flags.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...}

Outputs an SBOM.spdx.

Options:
`, filepath.Base(os.Args[0]))
		flags.PrintDefaults()
	}

	outputFile := flags.String("o", "-", "Where to write the SBOM spdx file. (default stdout)")
	depsFile := flags.String("d", "", "Where to write the deps file")
	product := flags.String("product", "", "The name of the product for which the notice is generated.")
	stripPrefix := newMultiString(flags, "strip_prefix", "Prefix to remove from paths. i.e. path to root (multiple allowed)")

	flags.Parse(expandedArgs)

	// Must specify at least one root target.
	if flags.NArg() == 0 {
		flags.Usage()
		os.Exit(2)
	}

	if len(*outputFile) == 0 {
		flags.Usage()
		fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n")
		os.Exit(2)
	} else {
		dir, err := filepath.Abs(filepath.Dir(*outputFile))
		if err != nil {
			fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err)
			os.Exit(1)
		}
		fi, err := os.Stat(dir)
		if err != nil {
			fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err)
			os.Exit(1)
		}
		if !fi.IsDir() {
			fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile)
			os.Exit(1)
		}
	}

	var ofile io.Writer
	ofile = os.Stdout
	var obuf *bytes.Buffer
	if *outputFile != "-" {
		obuf = &bytes.Buffer{}
		ofile = obuf
	}

	ctx := &context{ofile, os.Stderr, compliance.FS, *product, *stripPrefix, actualTime}

	spdxDoc, deps, err := sbomGenerator(ctx, flags.Args()...)

	if err != nil {
		if err == failNoneRequested {
			flags.Usage()
		}
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		os.Exit(1)
	}

	// writing the spdx Doc created
	if err := spdx_json.Save2_2(spdxDoc, ofile); err != nil {
		fmt.Fprintf(os.Stderr, "failed to write document to %v: %v", *outputFile, err)
		os.Exit(1)
	}

	if *outputFile != "-" {
		err := os.WriteFile(*outputFile, obuf.Bytes(), 0666)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err)
			os.Exit(1)
		}
	}

	if *depsFile != "" {
		err := deptools.WriteDepFile(*depsFile, *outputFile, deps)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err)
			os.Exit(1)
		}
	}
	os.Exit(0)
}

type creationTimeGetter func() string

// actualTime returns current time in UTC
func actualTime() string {
	t := time.Now().UTC()
	return t.UTC().Format("2006-01-02T15:04:05Z")
}

// replaceSlashes replaces "/" by "-" for the library path to be used for packages & files SPDXID
func replaceSlashes(x string) string {
	return strings.ReplaceAll(x, "/", "-")
}

// stripDocName removes the outdir prefix and meta_lic suffix from a target Name
func stripDocName(name string) string {
	// remove outdir prefix
	if strings.HasPrefix(name, "out/") {
		name = name[4:]
	}

	// remove suffix
	if strings.HasSuffix(name, ".meta_lic") {
		name = name[:len(name)-9]
	} else if strings.HasSuffix(name, "/meta_lic") {
		name = name[:len(name)-9] + "/"
	}

	return name
}

// getPackageName returns a package name of a target Node
func getPackageName(_ *context, tn *compliance.TargetNode) string {
	return replaceSlashes(tn.Name())
}

// getDocumentName returns a package name of a target Node
func getDocumentName(ctx *context, tn *compliance.TargetNode, pm *projectmetadata.ProjectMetadata) string {
	if len(ctx.product) > 0 {
		return replaceSlashes(ctx.product)
	}
	if len(tn.ModuleName()) > 0 {
		if pm != nil {
			return replaceSlashes(pm.Name() + ":" + tn.ModuleName())
		}
		return replaceSlashes(tn.ModuleName())
	}

	return stripDocName(replaceSlashes(tn.Name()))
}

// getDownloadUrl returns the download URL if available (GIT, SVN, etc..),
// or NOASSERTION if not available, none determined or ambiguous
func getDownloadUrl(_ *context, pm *projectmetadata.ProjectMetadata) string {
	if pm == nil {
		return NOASSERTION
	}

	urlsByTypeName := pm.UrlsByTypeName()
	if urlsByTypeName == nil {
		return NOASSERTION
	}

	url := urlsByTypeName.DownloadUrl()
	if url == "" {
		return NOASSERTION
	}
	return url
}

// getProjectMetadata returns the optimal project metadata for the target node
func getProjectMetadata(_ *context, pmix *projectmetadata.Index,
	tn *compliance.TargetNode) (*projectmetadata.ProjectMetadata, error) {
	pms, err := pmix.MetadataForProjects(tn.Projects()...)
	if err != nil {
		return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn, err)
	}
	if len(pms) == 0 {
		return nil, nil
	}

	// Getting the project metadata that contains most of the info needed for sbomGenerator
	score := -1
	index := -1
	for i := 0; i < len(pms); i++ {
		tempScore := 0
		if pms[i].Name() != "" {
			tempScore += 1
		}
		if pms[i].Version() != "" {
			tempScore += 1
		}
		if pms[i].UrlsByTypeName().DownloadUrl() != "" {
			tempScore += 1
		}

		if tempScore == score {
			if pms[i].Project() < pms[index].Project() {
				index = i
			}
		} else if tempScore > score {
			score = tempScore
			index = i
		}
	}
	return pms[index], nil
}

// inputFiles returns the complete list of files read
func inputFiles(lg *compliance.LicenseGraph, pmix *projectmetadata.Index, licenseTexts []string) []string {
	projectMeta := pmix.AllMetadataFiles()
	targets := lg.TargetNames()
	files := make([]string, 0, len(licenseTexts)+len(targets)+len(projectMeta))
	files = append(files, licenseTexts...)
	files = append(files, targets...)
	files = append(files, projectMeta...)
	return files
}

// generateSPDXNamespace generates a unique SPDX Document Namespace using a SHA1 checksum
// and the CreationInfo.Created field as the date.
func generateSPDXNamespace(created string) string {
	// Compute a SHA1 checksum of the CreationInfo.Created field.
	hash := sha1.Sum([]byte(created))
	checksum := hex.EncodeToString(hash[:])

	// Combine the checksum and timestamp to generate the SPDX Namespace.
	namespace := fmt.Sprintf("SPDXRef-DOCUMENT-%s-%s", created, checksum)

	return namespace
}

// sbomGenerator implements the spdx bom utility

// SBOM is part of the new government regulation issued to improve national cyber security
// and enhance software supply chain and transparency, see https://www.cisa.gov/sbom

// sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/)
// sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide)
func sbomGenerator(ctx *context, files ...string) (*spdx.Document, []string, error) {
	// Must be at least one root file.
	if len(files) < 1 {
		return nil, nil, failNoneRequested
	}

	pmix := projectmetadata.NewIndex(ctx.rootFS)

	lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files)

	if err != nil {
		return nil, nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err)
	}

	// creating the packages section
	pkgs := []*spdx.Package{}

	// creating the relationship section
	relationships := []*spdx.Relationship{}

	// creating the license section
	otherLicenses := []*spdx.OtherLicense{}

	// spdx document name
	var docName string

	// main package name
	var mainPkgName string

	// implementing the licenses references for the packages
	licenses := make(map[string]string)
	concludedLicenses := func(licenseTexts []string) string {
		licenseRefs := make([]string, 0, len(licenseTexts))
		for _, licenseText := range licenseTexts {
			license := strings.SplitN(licenseText, ":", 2)[0]
			if _, ok := licenses[license]; !ok {
				licenseRef := "LicenseRef-" + replaceSlashes(license)
				licenses[license] = licenseRef
			}

			licenseRefs = append(licenseRefs, licenses[license])
		}
		if len(licenseRefs) > 1 {
			return "(" + strings.Join(licenseRefs, " AND ") + ")"
		} else if len(licenseRefs) == 1 {
			return licenseRefs[0]
		}
		return "NONE"
	}

	isMainPackage := true
	visitedNodes := make(map[*compliance.TargetNode]struct{})

	// performing a Breadth-first top down walk of licensegraph and building package information
	compliance.WalkTopDownBreadthFirst(nil, lg,
		func(lg *compliance.LicenseGraph, tn *compliance.TargetNode, path compliance.TargetEdgePath) bool {
			if err != nil {
				return false
			}
			var pm *projectmetadata.ProjectMetadata
			pm, err = getProjectMetadata(ctx, pmix, tn)
			if err != nil {
				return false
			}

			if isMainPackage {
				docName = getDocumentName(ctx, tn, pm)
				mainPkgName = replaceSlashes(getPackageName(ctx, tn))
				isMainPackage = false
			}

			if len(path) == 0 {
				// Add the describe relationship for the main package
				rln := &spdx.Relationship{
					RefA:         common.MakeDocElementID("" /* this document */, "DOCUMENT"),
					RefB:         common.MakeDocElementID("", mainPkgName),
					Relationship: "DESCRIBES",
				}
				relationships = append(relationships, rln)

			} else {
				// Check parent and identify annotation
				parent := path[len(path)-1]
				targetEdge := parent.Edge()
				if targetEdge.IsRuntimeDependency() {
					// Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship
					rln := &spdx.Relationship{
						RefA:         common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
						RefB:         common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
						Relationship: "RUNTIME_DEPENDENCY_OF",
					}
					relationships = append(relationships, rln)

				} else if targetEdge.IsDerivation() {
					// Adding the  derivation annotation as a CONTAINS relationship
					rln := &spdx.Relationship{
						RefA:         common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
						RefB:         common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
						Relationship: "CONTAINS",
					}
					relationships = append(relationships, rln)

				} else if targetEdge.IsBuildTool() {
					// Adding the toolchain annotation as a BUILD_TOOL_OF relationship
					rln := &spdx.Relationship{
						RefA:         common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
						RefB:         common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
						Relationship: "BUILD_TOOL_OF",
					}
					relationships = append(relationships, rln)

				} else {
					panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations()))
				}
			}

			if _, alreadyVisited := visitedNodes[tn]; alreadyVisited {
				return false
			}
			visitedNodes[tn] = struct{}{}
			pkgName := getPackageName(ctx, tn)

			// Making an spdx package and adding it to pkgs
			pkg := &spdx.Package{
				PackageName:             replaceSlashes(pkgName),
				PackageDownloadLocation: getDownloadUrl(ctx, pm),
				PackageSPDXIdentifier:   common.ElementID(replaceSlashes(pkgName)),
				PackageLicenseConcluded: concludedLicenses(tn.LicenseTexts()),
			}

			if pm != nil && pm.Version() != "" {
				pkg.PackageVersion = pm.Version()
			} else {
				pkg.PackageVersion = NOASSERTION
			}

			pkgs = append(pkgs, pkg)

			return true
		})

	// Adding Non-standard licenses

	licenseTexts := make([]string, 0, len(licenses))

	for licenseText := range licenses {
		licenseTexts = append(licenseTexts, licenseText)
	}

	sort.Strings(licenseTexts)

	for _, licenseText := range licenseTexts {
		// open the file
		f, err := ctx.rootFS.Open(filepath.Clean(licenseText))
		if err != nil {
			return nil, nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err)
		}

		// read the file
		text, err := io.ReadAll(f)
		if err != nil {
			return nil, nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err)
		}
		// Making an spdx License and adding it to otherLicenses
		otherLicenses = append(otherLicenses, &spdx.OtherLicense{
			LicenseName:       strings.Replace(licenses[licenseText], "LicenseRef-", "", -1),
			LicenseIdentifier: string(licenses[licenseText]),
			ExtractedText:     string(text),
		})
	}

	deps := inputFiles(lg, pmix, licenseTexts)
	sort.Strings(deps)

	// Making the SPDX doc
	ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil)
	if err != nil {
		return nil, nil, fmt.Errorf("Unable to build creation info section for SPDX doc: %v\n", err)
	}

	ci.Created = ctx.creationTime()

	doc := &spdx.Document{
		SPDXVersion:       "SPDX-2.2",
		DataLicense:       "CC0-1.0",
		SPDXIdentifier:    "DOCUMENT",
		DocumentName:      docName,
		DocumentNamespace: generateSPDXNamespace(ci.Created),
		CreationInfo:      ci,
		Packages:          pkgs,
		Relationships:     relationships,
		OtherLicenses:     otherLicenses,
	}

	if err := spdxlib.ValidateDocument2_2(doc); err != nil {
		return nil, nil, fmt.Errorf("Unable to validate the SPDX doc: %v\n", err)
	}

	return doc, deps, nil
}