StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generate SBOM of products in Soong.

Browse Source 
Bug: 324465531
Test: CIs
Test: m soong-sbom
Change-Id: If76776851d49282829a79bfb1c33f05b8f57de31
This commit is contained in:
Wei Li
2024-02-05 14:50:54 -08:00
parent c25dea9d73
commit b85a178b07
4 changed files with 172 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
android/Android.bp
View File

@@ -93,6 +93,7 @@ bootstrap_go_package {
"register.go",
"rule_builder.go",
"sandbox.go",
"sbom.go",
"sdk.go",
"sdk_version.go",
"shared_properties.go",

100
android/sbom.go Normal file
View File

@@ -0,0 +1,100 @@
// Copyright 2024 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"io"
"path/filepath"
"strings"
"github.com/google/blueprint"
)
var (
// Command line tool to generate SBOM in Soong
genSbom = pctx.HostBinToolVariable("genSbom", "gen_sbom")
// Command to generate SBOM in Soong.
genSbomRule = pctx.AndroidStaticRule("genSbomRule", blueprint.RuleParams{
Command: "rm -rf $out && ${genSbom} --output_file ${out} --metadata ${in} --product_out ${productOut} --soong_out ${soongOut} --build_version \"$$(cat ${buildFingerprintFile})\" --product_mfr \"${productManufacturer}\" --json",
CommandDeps: []string{"${genSbom}"},
}, "productOut", "soongOut", "buildFingerprintFile", "productManufacturer")
)
func init() {
RegisterSbomSingleton(InitRegistrationContext)
}
func RegisterSbomSingleton(ctx RegistrationContext) {
ctx.RegisterParallelSingletonType("sbom_singleton", sbomSingletonFactory)
}
// sbomSingleton is used to generate build actions of generating SBOM of products.
type sbomSingleton struct{}
func sbomSingletonFactory() Singleton {
return &sbomSingleton{}
}
// Generates SBOM of products
func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) {
if !ctx.Config().HasDeviceProduct() {
return
}
// Get all METADATA files and add them as implicit input
metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list")
f, err := ctx.Config().fs.Open(metadataFileListFile.String())
if err != nil {
panic(err)
}
b, err := io.ReadAll(f)
if err != nil {
panic(err)
}
allMetadataFiles := strings.Split(string(b), "\n")
implicits := []Path{metadataFileListFile}
for _, path := range allMetadataFiles {
implicits = append(implicits, PathForSource(ctx, path))
}
prodVars := ctx.Config().productVariables
buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt")
implicits = append(implicits, buildFingerprintFile)
// Add installed_files.stamp as implicit input, which depends on all installed files of the product.
installedFilesStamp := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "installed_files.stamp")
implicits = append(implicits, installedFilesStamp)
metadataDb := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "compliance-metadata.db")
sbomFile := PathForOutput(ctx, "sbom", ctx.Config().DeviceProduct(), "sbom.spdx.json")
ctx.Build(pctx, BuildParams{
Rule: genSbomRule,
Input: metadataDb,
Implicits: implicits,
Output: sbomFile,
Args: map[string]string{
"productOut": filepath.Join(ctx.Config().OutDir(), "target", "product", String(prodVars.DeviceName)),
"soongOut": ctx.Config().soongOutDir,
"buildFingerprintFile": buildFingerprintFile.String(),
"productManufacturer": ctx.Config().ProductVariables().ProductManufacturer,
},
})
// Phony rule "soong-sbom". "m soong-sbom" to generate product SBOM in Soong.
ctx.Build(pctx, BuildParams{
Rule: blueprint.Phony,
Inputs: []Path{sbomFile},
Output: PathForPhony(ctx, "soong-sbom"),
})
}

64
tests/sbom_test.sh
View File

@@ -70,13 +70,14 @@ function test_sbom_aosp_cf_x86_64_phone {
# m droid, build sbom later in case additional dependencies might be built and included in partition images.
run_soong "${out_dir}" "droid dump.erofs lz4"
soong_sbom_out=$out_dir/soong/sbom/$target_product
product_out=$out_dir/target/product/vsoc_x86_64
sbom_test=$product_out/sbom_test
mkdir -p $sbom_test
cp $product_out/*.img $sbom_test
# m sbom
run_soong "${out_dir}" sbom
# m sbom soong-sbom
run_soong "${out_dir}" "sbom soong-sbom"
# Generate installed file list from .img files in PRODUCT_OUT
dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
@@ -118,6 +119,7 @@ function test_sbom_aosp_cf_x86_64_phone {
partition_name=$(basename $f | cut -d. -f1)
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt"
rm "$file_list_file" > /dev/null 2>&1 || true
all_dirs="/"
while [ ! -z "$all_dirs" ]; do
@@ -145,6 +147,7 @@ function test_sbom_aosp_cf_x86_64_phone {
done
sort -n -o "$file_list_file" "$file_list_file"
# Diff the file list from image and file list in SBOM created by Make
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"
if [ "$partition_name" = "system" ]; then
# system partition is mounted to /, so include FileName starts with /root/ too.
@@ -154,6 +157,17 @@ function test_sbom_aosp_cf_x86_64_phone {
echo ============ Diffing files in $f and SBOM
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
# Diff the file list from image and file list in SBOM created by Soong
grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file"
if [ "$partition_name" = "system" ]; then
# system partition is mounted to /, so include FileName starts with /root/ too.
grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file"
fi
sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file"
echo ============ Diffing files in $f and SBOM created by Soong
diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
done
RAMDISK_IMAGES="$product_out/ramdisk.img"
@@ -161,6 +175,7 @@ function test_sbom_aosp_cf_x86_64_phone {
partition_name=$(basename $f | cut -d. -f1)
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt"
# lz4 decompress $f to stdout
# cpio list all entries like ls -l
# grep filter normal files and symlinks
@@ -170,11 +185,19 @@ function test_sbom_aosp_cf_x86_64_phone {
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"
grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file"
echo ============ Diffing files in $f and SBOM
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
echo ============ Diffing files in $f and SBOM created by Soong
diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
done
verify_package_verification_code "$product_out/sbom.spdx"
verify_package_verification_code "$soong_sbom_out/sbom.spdx"
verify_packages_licenses "$soong_sbom_out/sbom.spdx"
# Teardown
cleanup "${out_dir}"
@@ -213,6 +236,41 @@ function verify_package_verification_code {
fi
}
function verify_packages_licenses {
local sbom_file="$1"; shift
num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l)
num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l)
if [ "$num_of_packages" = "$num_of_declared_licenses" ]
then
echo "Number of packages with declared license is correct."
else
echo "Number of packages with declared license is WRONG."
exit 1
fi
# PRODUCT and 7 prebuilt packages have "PackageLicenseDeclared: NOASSERTION"
# All other packages have declared licenses
num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l)
if [ $num_of_packages_with_noassertion_license = 15 ]
then
echo "Number of packages with NOASSERTION license is correct."
else
echo "Number of packages with NOASSERTION license is WRONG."
exit 1
fi
num_of_files=$(grep 'FileName:' $sbom_file | wc -l)
num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l)
if [ "$num_of_files" = "$num_of_concluded_licenses" ]
then
echo "Number of files with concluded license is correct."
else
echo "Number of files with concluded license is WRONG."
exit 1
fi
}
function test_sbom_unbundled_apex {
# Setup
out_dir="$(setup)"
@@ -274,7 +332,7 @@ function test_sbom_unbundled_apk {
target_product=aosp_cf_x86_64_phone
target_release=trunk_staging
target_build_variant=userdebug
target_build_variant=eng
for i in "$@"; do
case $i in
TARGET_PRODUCT=*)

13
ui/build/test_build.go
View File

@@ -15,14 +15,16 @@
package build
import (
"android/soong/ui/metrics"
"android/soong/ui/status"
"bufio"
"fmt"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
"android/soong/ui/metrics"
"android/soong/ui/status"
)
// Checks for files in the out directory that have a rule that depends on them but no rule to
@@ -84,6 +86,10 @@ func testForDanglingRules(ctx Context, config Config) {
// before running soong and ninja.
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
// out/target/product/<xxxxx>/build_fingerprint.txt is a source file created in sysprop.mk
// ^out/target/product/[^/]+/build_fingerprint.txt$
buildFingerPrintFilePattern := regexp.MustCompile("^" + filepath.Join(outDir, "target", "product") + "/[^/]+/build_fingerprint.txt$")
danglingRules := make(map[string]bool)
scanner := bufio.NewScanner(stdout)
@@ -100,7 +106,8 @@ func testForDanglingRules(ctx Context, config Config) {
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||
line == bpglob ||
strings.HasPrefix(line, releaseConfigDir) {
strings.HasPrefix(line, releaseConfigDir) ||
buildFingerPrintFilePattern.MatchString(line) {
// Leaf node is in one of Soong's bootstrap directories, which do not have
// full build rules in the primary build.ninja file.
continue