Merge "Generate SBOM of products in Soong." into main
This commit is contained in:
Wei Li
Gerrit Code Review
@@ -93,6 +93,7 @@ bootstrap_go_package {
|
||||
"register.go",
|
||||
"rule_builder.go",
|
||||
"sandbox.go",
|
||||
"sbom.go",
|
||||
"sdk.go",
|
||||
"sdk_version.go",
|
||||
"shared_properties.go",
|
||||
|
||||
100
android/sbom.go
Normal file
100
android/sbom.go
Normal file
@@ -0,0 +1,100 @@
|
||||
// Copyright 2024 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package android
|
||||
|
||||
import (
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
)
|
||||
|
||||
var (
|
||||
// Command line tool to generate SBOM in Soong
|
||||
genSbom = pctx.HostBinToolVariable("genSbom", "gen_sbom")
|
||||
|
||||
// Command to generate SBOM in Soong.
|
||||
genSbomRule = pctx.AndroidStaticRule("genSbomRule", blueprint.RuleParams{
|
||||
Command: "rm -rf $out && ${genSbom} --output_file ${out} --metadata ${in} --product_out ${productOut} --soong_out ${soongOut} --build_version \"$$(cat ${buildFingerprintFile})\" --product_mfr \"${productManufacturer}\" --json",
|
||||
CommandDeps: []string{"${genSbom}"},
|
||||
}, "productOut", "soongOut", "buildFingerprintFile", "productManufacturer")
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterSbomSingleton(InitRegistrationContext)
|
||||
}
|
||||
|
||||
func RegisterSbomSingleton(ctx RegistrationContext) {
|
||||
ctx.RegisterParallelSingletonType("sbom_singleton", sbomSingletonFactory)
|
||||
}
|
||||
|
||||
// sbomSingleton is used to generate build actions of generating SBOM of products.
|
||||
type sbomSingleton struct{}
|
||||
|
||||
func sbomSingletonFactory() Singleton {
|
||||
return &sbomSingleton{}
|
||||
}
|
||||
|
||||
// Generates SBOM of products
|
||||
func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) {
|
||||
if !ctx.Config().HasDeviceProduct() {
|
||||
return
|
||||
}
|
||||
// Get all METADATA files and add them as implicit input
|
||||
metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list")
|
||||
f, err := ctx.Config().fs.Open(metadataFileListFile.String())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
allMetadataFiles := strings.Split(string(b), "\n")
|
||||
implicits := []Path{metadataFileListFile}
|
||||
for _, path := range allMetadataFiles {
|
||||
implicits = append(implicits, PathForSource(ctx, path))
|
||||
}
|
||||
prodVars := ctx.Config().productVariables
|
||||
buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt")
|
||||
implicits = append(implicits, buildFingerprintFile)
|
||||
|
||||
// Add installed_files.stamp as implicit input, which depends on all installed files of the product.
|
||||
installedFilesStamp := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "installed_files.stamp")
|
||||
implicits = append(implicits, installedFilesStamp)
|
||||
|
||||
metadataDb := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "compliance-metadata.db")
|
||||
sbomFile := PathForOutput(ctx, "sbom", ctx.Config().DeviceProduct(), "sbom.spdx.json")
|
||||
ctx.Build(pctx, BuildParams{
|
||||
Rule: genSbomRule,
|
||||
Input: metadataDb,
|
||||
Implicits: implicits,
|
||||
Output: sbomFile,
|
||||
Args: map[string]string{
|
||||
"productOut": filepath.Join(ctx.Config().OutDir(), "target", "product", String(prodVars.DeviceName)),
|
||||
"soongOut": ctx.Config().soongOutDir,
|
||||
"buildFingerprintFile": buildFingerprintFile.String(),
|
||||
"productManufacturer": ctx.Config().ProductVariables().ProductManufacturer,
|
||||
},
|
||||
})
|
||||
|
||||
// Phony rule "soong-sbom". "m soong-sbom" to generate product SBOM in Soong.
|
||||
ctx.Build(pctx, BuildParams{
|
||||
Rule: blueprint.Phony,
|
||||
Inputs: []Path{sbomFile},
|
||||
Output: PathForPhony(ctx, "soong-sbom"),
|
||||
})
|
||||
}
|
||||
@@ -70,13 +70,14 @@ function test_sbom_aosp_cf_x86_64_phone {
|
||||
# m droid, build sbom later in case additional dependencies might be built and included in partition images.
|
||||
run_soong "${out_dir}" "droid dump.erofs lz4"
|
||||
|
||||
soong_sbom_out=$out_dir/soong/sbom/$target_product
|
||||
product_out=$out_dir/target/product/vsoc_x86_64
|
||||
sbom_test=$product_out/sbom_test
|
||||
mkdir -p $sbom_test
|
||||
cp $product_out/*.img $sbom_test
|
||||
|
||||
# m sbom
|
||||
run_soong "${out_dir}" sbom
|
||||
# m sbom soong-sbom
|
||||
run_soong "${out_dir}" "sbom soong-sbom"
|
||||
|
||||
# Generate installed file list from .img files in PRODUCT_OUT
|
||||
dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
|
||||
@@ -118,6 +119,7 @@ function test_sbom_aosp_cf_x86_64_phone {
|
||||
partition_name=$(basename $f | cut -d. -f1)
|
||||
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
|
||||
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
|
||||
files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt"
|
||||
rm "$file_list_file" > /dev/null 2>&1 || true
|
||||
all_dirs="/"
|
||||
while [ ! -z "$all_dirs" ]; do
|
||||
@@ -145,6 +147,7 @@ function test_sbom_aosp_cf_x86_64_phone {
|
||||
done
|
||||
sort -n -o "$file_list_file" "$file_list_file"
|
||||
|
||||
# Diff the file list from image and file list in SBOM created by Make
|
||||
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"
|
||||
if [ "$partition_name" = "system" ]; then
|
||||
# system partition is mounted to /, so include FileName starts with /root/ too.
|
||||
@@ -154,6 +157,17 @@ function test_sbom_aosp_cf_x86_64_phone {
|
||||
|
||||
echo ============ Diffing files in $f and SBOM
|
||||
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
|
||||
|
||||
# Diff the file list from image and file list in SBOM created by Soong
|
||||
grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file"
|
||||
if [ "$partition_name" = "system" ]; then
|
||||
# system partition is mounted to /, so include FileName starts with /root/ too.
|
||||
grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file"
|
||||
fi
|
||||
sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file"
|
||||
|
||||
echo ============ Diffing files in $f and SBOM created by Soong
|
||||
diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
|
||||
done
|
||||
|
||||
RAMDISK_IMAGES="$product_out/ramdisk.img"
|
||||
@@ -161,6 +175,7 @@ function test_sbom_aosp_cf_x86_64_phone {
|
||||
partition_name=$(basename $f | cut -d. -f1)
|
||||
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
|
||||
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
|
||||
files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt"
|
||||
# lz4 decompress $f to stdout
|
||||
# cpio list all entries like ls -l
|
||||
# grep filter normal files and symlinks
|
||||
@@ -170,11 +185,19 @@ function test_sbom_aosp_cf_x86_64_phone {
|
||||
|
||||
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"
|
||||
|
||||
grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file"
|
||||
|
||||
echo ============ Diffing files in $f and SBOM
|
||||
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
|
||||
|
||||
echo ============ Diffing files in $f and SBOM created by Soong
|
||||
diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
|
||||
done
|
||||
|
||||
verify_package_verification_code "$product_out/sbom.spdx"
|
||||
verify_package_verification_code "$soong_sbom_out/sbom.spdx"
|
||||
|
||||
verify_packages_licenses "$soong_sbom_out/sbom.spdx"
|
||||
|
||||
# Teardown
|
||||
cleanup "${out_dir}"
|
||||
@@ -213,6 +236,41 @@ function verify_package_verification_code {
|
||||
fi
|
||||
}
|
||||
|
||||
function verify_packages_licenses {
|
||||
local sbom_file="$1"; shift
|
||||
|
||||
num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l)
|
||||
num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l)
|
||||
if [ "$num_of_packages" = "$num_of_declared_licenses" ]
|
||||
then
|
||||
echo "Number of packages with declared license is correct."
|
||||
else
|
||||
echo "Number of packages with declared license is WRONG."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# PRODUCT and 7 prebuilt packages have "PackageLicenseDeclared: NOASSERTION"
|
||||
# All other packages have declared licenses
|
||||
num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l)
|
||||
if [ $num_of_packages_with_noassertion_license = 15 ]
|
||||
then
|
||||
echo "Number of packages with NOASSERTION license is correct."
|
||||
else
|
||||
echo "Number of packages with NOASSERTION license is WRONG."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
num_of_files=$(grep 'FileName:' $sbom_file | wc -l)
|
||||
num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l)
|
||||
if [ "$num_of_files" = "$num_of_concluded_licenses" ]
|
||||
then
|
||||
echo "Number of files with concluded license is correct."
|
||||
else
|
||||
echo "Number of files with concluded license is WRONG."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function test_sbom_unbundled_apex {
|
||||
# Setup
|
||||
out_dir="$(setup)"
|
||||
@@ -274,7 +332,7 @@ function test_sbom_unbundled_apk {
|
||||
|
||||
target_product=aosp_cf_x86_64_phone
|
||||
target_release=trunk_staging
|
||||
target_build_variant=userdebug
|
||||
target_build_variant=eng
|
||||
for i in "$@"; do
|
||||
case $i in
|
||||
TARGET_PRODUCT=*)
|
||||
|
||||
@@ -15,14 +15,16 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"android/soong/ui/metrics"
|
||||
"android/soong/ui/status"
|
||||
"bufio"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"android/soong/ui/metrics"
|
||||
"android/soong/ui/status"
|
||||
)
|
||||
|
||||
// Checks for files in the out directory that have a rule that depends on them but no rule to
|
||||
@@ -84,6 +86,10 @@ func testForDanglingRules(ctx Context, config Config) {
|
||||
// before running soong and ninja.
|
||||
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
|
||||
|
||||
// out/target/product/<xxxxx>/build_fingerprint.txt is a source file created in sysprop.mk
|
||||
// ^out/target/product/[^/]+/build_fingerprint.txt$
|
||||
buildFingerPrintFilePattern := regexp.MustCompile("^" + filepath.Join(outDir, "target", "product") + "/[^/]+/build_fingerprint.txt$")
|
||||
|
||||
danglingRules := make(map[string]bool)
|
||||
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
@@ -100,7 +106,8 @@ func testForDanglingRules(ctx Context, config Config) {
|
||||
line == dexpreoptConfigFilePath ||
|
||||
line == buildDatetimeFilePath ||
|
||||
line == bpglob ||
|
||||
strings.HasPrefix(line, releaseConfigDir) {
|
||||
strings.HasPrefix(line, releaseConfigDir) ||
|
||||
buildFingerPrintFilePattern.MatchString(line) {
|
||||
// Leaf node is in one of Soong's bootstrap directories, which do not have
|
||||
// full build rules in the primary build.ninja file.
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user