StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Include static libraries information in Android SBOM." am: c20175155a

Browse Source 
Original change: https://android-review.googlesource.com/c/platform/build/+/2609475

Change-Id: I342757931802a04dd09d0f50e3eea2219fdd5948
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Wei Li
2023-06-19 22:51:32 +00:00
committed by Automerger Merge Worker
parent 9d9e540cc5 c20175155a
commit 7f298d8a1a
9 changed files with 216 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tools/sbom/generate-sbom.py
View File

@@ -332,14 +332,6 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path):
return external_doc_ref, packages, relationships
def generate_package_verification_code(files):
checksums = [file.checksum for file in files]
checksums.sort()
h = hashlib.sha1()
h.update(''.join(checksums).encode(encoding='utf-8'))
return h.hexdigest()
def save_report(report_file_path, report):
with open(report_file_path, 'w', encoding='utf-8') as report_file:
for type, issues in report.items():
@@ -487,20 +479,32 @@ def main():
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
build_output_path = installed_file_metadata['build_output_path']
is_static_lib = installed_file_metadata['is_static_lib']
if not installed_file_has_metadata(installed_file_metadata, report):
continue
if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
# Ignore non-existing static library files for now since they are not shipped on devices.
report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
continue
file_id = new_file_id(installed_file)
doc.files.append(
sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))
if not args.unbundled_apex:
product_package.file_ids.append(file_id)
elif len(doc.files) > 1:
doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
# TODO(b/285453664): Soong should report the information of statically linked libraries to Make.
# This happens when a different sanitized version of static libraries is used in linking.
# As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be
# located correctly because Soong doesn't report the information to Make.
sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string
if os.path.islink(build_output_path) or os.path.isfile(build_output_path):
sha1 = checksum(build_output_path)
doc.files.append(sbom_data.File(id=file_id,
name=installed_file,
checksum=sha1))
if not is_static_lib:
if not args.unbundled_apex:
product_package.file_ids.append(file_id)
elif len(doc.files) > 1:
doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
metadata_file_path = get_metadata_file_path(installed_file_metadata)
@@ -544,13 +548,21 @@ def main():
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
if not args.unbundled_apex:
product_package.verification_code = generate_package_verification_code(doc.files)
# Process static libraries and whole static libraries the installed file links to
static_libs = installed_file_metadata['static_libraries']
whole_static_libs = installed_file_metadata['whole_static_libraries']
all_static_libs = (static_libs + ' ' + whole_static_libs).strip()
if all_static_libs:
for lib in all_static_libs.split(' '):
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.STATIC_LINK,
id2=new_file_id(lib + '.a')))
if args.unbundled_apex:
doc.describes = doc.files[0].id
# Save SBOM records to output file
doc.generate_packages_verification_code()
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
prefix = args.output_file
if prefix.endswith('.spdx'):

16
tools/sbom/sbom_data.py
View File

@@ -25,6 +25,7 @@ fields in each data class.
from dataclasses import dataclass, field
from typing import List
import hashlib
SPDXID_DOC = 'SPDXRef-DOCUMENT'
SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
@@ -81,6 +82,7 @@ class RelationshipType:
VARIANT_OF = 'VARIANT_OF'
GENERATED_FROM = 'GENERATED_FROM'
CONTAINS = 'CONTAINS'
STATIC_LINK = 'STATIC_LINK'
@dataclass
@@ -122,3 +124,17 @@ class Document:
if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
for r in self.relationships):
self.relationships.append(rel)
def generate_packages_verification_code(self):
for package in self.packages:
if not package.file_ids:
continue
checksums = []
for file in self.files:
if file.id in package.file_ids:
checksums.append(file.checksum)
checksums.sort()
h = hashlib.sha1()
h.update(''.join(checksums).encode(encoding='utf-8'))
package.verification_code = h.hexdigest()

77
tools/sbom/sbom_writers.py
View File

@@ -85,7 +85,7 @@ class TagValueWriter:
return headers
@staticmethod
def marshal_package(package):
def marshal_package(sbom_doc, package, fragment):
download_location = sbom_data.VALUE_NOASSERTION
if package.download_location:
download_location = package.download_location
@@ -107,50 +107,32 @@ class TagValueWriter:
f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')
tagvalues.append('')
if package.id == sbom_doc.describes and not fragment:
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
for file in sbom_doc.files:
if file.id in package.file_ids:
tagvalues += TagValueWriter.marshal_file(file)
return tagvalues
@staticmethod
def marshal_described_element(sbom_doc, fragment):
if not sbom_doc.describes:
return None
product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
if product_package:
tagvalues = TagValueWriter.marshal_package(product_package[0])
if not fragment:
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
return tagvalues
file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
if file:
tagvalues = TagValueWriter.marshal_file(file[0])
if not fragment:
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
return tagvalues
return None
@staticmethod
def marshal_packages(sbom_doc):
def marshal_packages(sbom_doc, fragment):
tagvalues = []
marshaled_relationships = []
i = 0
packages = sbom_doc.packages
while i < len(packages):
if packages[i].id == sbom_doc.describes:
i += 1
continue
if i + 1 < len(packages) \
and packages[i].id.startswith('SPDXRef-SOURCE-') \
and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
tagvalues += TagValueWriter.marshal_package(packages[i])
tagvalues += TagValueWriter.marshal_package(packages[i + 1])
if (i + 1 < len(packages)
and packages[i].id.startswith('SPDXRef-SOURCE-')
and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')):
# Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other
# in SBOMs in tagvalue format.
tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment)
rel = next((r for r in sbom_doc.relationships if
r.id1 == packages[i].id and
r.id2 == packages[i + 1].id and
@@ -162,7 +144,7 @@ class TagValueWriter:
i += 2
else:
tagvalues += TagValueWriter.marshal_package(packages[i])
tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
i += 1
return tagvalues, marshaled_relationships
@@ -179,12 +161,20 @@ class TagValueWriter:
return tagvalues
@staticmethod
def marshal_files(sbom_doc):
def marshal_files(sbom_doc, fragment):
tagvalues = []
files_in_packages = []
for package in sbom_doc.packages:
files_in_packages += package.file_ids
for file in sbom_doc.files:
if file.id == sbom_doc.describes:
if file.id in files_in_packages:
continue
tagvalues += TagValueWriter.marshal_file(file)
if file.id == sbom_doc.describes and not fragment:
# Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable.
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
return tagvalues
@staticmethod
@@ -208,11 +198,8 @@ class TagValueWriter:
content = []
if not fragment:
content += TagValueWriter.marshal_doc_headers(sbom_doc)
described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)
if described_element:
content += described_element
content += TagValueWriter.marshal_files(sbom_doc)
tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
content += TagValueWriter.marshal_files(sbom_doc, fragment)
tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)
content += tagvalues
content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
file.write('\n'.join(content))

15
tools/sbom/sbom_writers_test.py
View File

@@ -31,6 +31,7 @@ SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1'
SPDXID_FILE1 = 'SPDXRef-file1'
SPDXID_FILE2 = 'SPDXRef-file2'
SPDXID_FILE3 = 'SPDXRef-file3'
SPDXID_FILE4 = 'SPDXRef-file4'
class SBOMWritersTest(unittest.TestCase):
@@ -101,6 +102,8 @@ class SBOMWritersTest(unittest.TestCase):
sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444'))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
@@ -112,6 +115,10 @@ class SBOMWritersTest(unittest.TestCase):
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_SOURCE_PACKAGE1
))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.STATIC_LINK,
id2=SPDXID_FILE4
))
# SBOM fragment of a APK
self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
@@ -139,6 +146,14 @@ class SBOMWritersTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
def test_tagvalue_writer_doc_describes_file(self):
with io.StringIO() as output:
self.sbom_doc.describes = SPDXID_FILE4
sbom_writers.TagValueWriter.write(self.sbom_doc, output)
expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_doc_describes_file.spdx').read_text()
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
def test_tagvalue_writer_unbundled(self):
with io.StringIO() as output:
sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True)

15
tools/sbom/testdata/expected_json_sbom.spdx.json vendored
View File

@@ -110,6 +110,16 @@
"checksumValue": "33333"
}
]
},
{
"fileName": "file4.a",
"SPDXID": "SPDXRef-file4",
"checksums": [
{
"algorithm": "SHA1",
"checksumValue": "44444"
}
]
}
],
"relationships": [
@@ -128,6 +138,11 @@
"relatedSpdxElement": "SPDXRef-SOURCE-package1",
"relationshipType": "GENERATED_FROM"
},
{
"spdxElementId": "SPDXRef-file1",
"relatedSpdxElement": "SPDXRef-file4",
"relationshipType": "STATIC_LINK"
},
{
"spdxElementId": "SPDXRef-SOURCE-package1",
"relatedSpdxElement": "SPDXRef-UPSTREAM-package1",

5
tools/sbom/testdata/expected_tagvalue_sbom.spdx vendored
View File

@@ -7,6 +7,10 @@ Creator: Organization: Google
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
FileName: file4.a
SPDXID: SPDXRef-file4
FileChecksum: SHA1: 44444
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
@@ -63,3 +67,4 @@ Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4

70
tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx vendored Normal file
View File

@@ -0,0 +1,70 @@
SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: test doc
DocumentNamespace: http://www.google.com/sbom/spdx/android
Creator: Organization: Google
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
FileName: file4.a
SPDXID: SPDXRef-file4
FileChecksum: SHA1: 44444
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-file4
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
FilesAnalyzed: true
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageVerificationCode: 123456
FileName: /bin/file1
SPDXID: SPDXRef-file1
FileChecksum: SHA1: 11111
FileName: /bin/file2
SPDXID: SPDXRef-file2
FileChecksum: SHA1: 22222
FileName: /bin/file3
SPDXID: SPDXRef-file3
FileChecksum: SHA1: 33333
PackageName: PLATFORM
SPDXID: SPDXRef-PLATFORM
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageName: Prebuilt package1
SPDXID: SPDXRef-PREBUILT-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageName: Source package1
SPDXID: SPDXRef-SOURCE-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4
PackageName: Upstream package1
SPDXID: SPDXRef-UPSTREAM-package1
PackageDownloadLocation: NOASSERTION
FilesAnalyzed: false
PackageVersion: 1.1
PackageSupplier: Organization: upstream
Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4