StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Revert "Generate SBOM of the target product in file sbom.spdx in...""

Browse Source
This commit is contained in:
Kevin Han
2023-03-13 21:03:22 +00:00
committed by Gerrit Code Review
parent 909a5b40a6 161d711d4d
commit afb2495f6f
15 changed files with 2 additions and 1171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tools/Android.bp
View File

@@ -69,19 +69,3 @@ python_binary_host {
name: "generate_gts_shared_report",
srcs: ["generate_gts_shared_report.py"],
}
python_binary_host {
name: "generate-sbom",
srcs: [
"generate-sbom.py",
],
version: {
py3: {
embedded_launcher: true,
},
},
libs: [
"metadata_file_proto_py",
"libprotobuf-python",
],
}

684
tools/generate-sbom.py
View File

@@ -1,684 +0,0 @@
#!/usr/bin/env python3
#
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Generate the SBOM of the current target product in SPDX format.
Usage example:
generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \
--metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \
--product_out_dir=out/target/product/vsoc_x86_64 \
--build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
--product_mfr=Google
"""
import argparse
import csv
import datetime
import google.protobuf.text_format as text_format
import hashlib
import json
import os
import metadata_file_pb2
# Common
SPDXID = 'SPDXID'
SPDX_VERSION = 'SPDXVersion'
DATA_LICENSE = 'DataLicense'
DOCUMENT_NAME = 'DocumentName'
DOCUMENT_NAMESPACE = 'DocumentNamespace'
CREATED = 'Created'
CREATOR = 'Creator'
EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef'
# Package
PACKAGE_NAME = 'PackageName'
PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation'
PACKAGE_VERSION = 'PackageVersion'
PACKAGE_SUPPLIER = 'PackageSupplier'
FILES_ANALYZED = 'FilesAnalyzed'
PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode'
PACKAGE_EXTERNAL_REF = 'ExternalRef'
# Package license
PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded'
PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles'
PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared'
PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments'
# File
FILE_NAME = 'FileName'
FILE_CHECKSUM = 'FileChecksum'
# File license
FILE_LICENSE_CONCLUDED = 'LicenseConcluded'
FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile'
FILE_LICENSE_COMMENTS = 'LicenseComments'
FILE_COPYRIGHT_TEXT = 'FileCopyrightText'
FILE_NOTICE = 'FileNotice'
FILE_ATTRIBUTION_TEXT = 'FileAttributionText'
# Relationship
RELATIONSHIP = 'Relationship'
REL_DESCRIBES = 'DESCRIBES'
REL_VARIANT_OF = 'VARIANT_OF'
REL_GENERATED_FROM = 'GENERATED_FROM'
# Package type
PKG_SOURCE = 'SOURCE'
PKG_UPSTREAM = 'UPSTREAM'
PKG_PREBUILT = 'PREBUILT'
# Security tag
NVD_CPE23 = 'NVD-CPE2.3:'
# Report
ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:'
ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:'
ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:'
ISSUE_UNKNOWN_SECURITY_TAG_TYPE = "Unknown security tag type:"
INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:'
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.')
parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.')
parser.add_argument('--build_version', required=True, help='The build version.')
parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module')
return parser.parse_args()
def log(*info):
if args.verbose:
for i in info:
print(i)
def new_doc_header(doc_id):
return {
SPDX_VERSION: 'SPDX-2.3',
DATA_LICENSE: 'CC0-1.0',
SPDXID: doc_id,
DOCUMENT_NAME: args.build_version,
DOCUMENT_NAMESPACE: 'https://www.google.com/sbom/spdx/android/' + args.build_version,
CREATOR: 'Organization: Google, LLC',
CREATED: '<timestamp>',
EXTERNAL_DOCUMENT_REF: [],
}
def new_package_record(id, name, version, supplier, download_location=None, files_analyzed='false', external_refs=[]):
package = {
PACKAGE_NAME: name,
SPDXID: id,
PACKAGE_DOWNLOAD_LOCATION: download_location if download_location else 'NONE',
FILES_ANALYZED: files_analyzed,
}
if version:
package[PACKAGE_VERSION] = version
if supplier:
package[PACKAGE_SUPPLIER] = 'Organization: ' + supplier
if external_refs:
package[PACKAGE_EXTERNAL_REF] = external_refs
return package
def new_file_record(id, name, checksum):
return {
FILE_NAME: name,
SPDXID: id,
FILE_CHECKSUM: checksum
}
def encode_for_spdxid(s):
"""Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-"""
result = ''
for c in s:
if c.isalnum() or c in '.-':
result += c
elif c in '_@/':
result += '-'
else:
result += '0x' + c.encode('utf-8').hex()
return result.lstrip('-')
def new_package_id(package_name, type):
return 'SPDXRef-{}-{}'.format(type, encode_for_spdxid(package_name))
def new_external_doc_ref(package_name, sbom_url, sbom_checksum):
doc_ref_id = 'DocumentRef-{}-{}'.format(PKG_UPSTREAM, encode_for_spdxid(package_name))
return '{}: {} {} {}'.format(EXTERNAL_DOCUMENT_REF, doc_ref_id, sbom_url, sbom_checksum), doc_ref_id
def new_file_id(file_path):
return 'SPDXRef-' + encode_for_spdxid(file_path)
def new_relationship_record(id1, relationship, id2):
return '{}: {} {} {}'.format(RELATIONSHIP, id1, relationship, id2)
def checksum(file_path):
file_path = args.product_out_dir + '/' + file_path
h = hashlib.sha1()
if os.path.islink(file_path):
h.update(os.readlink(file_path).encode('utf-8'))
else:
with open(file_path, "rb") as f:
h.update(f.read())
return "SHA1: " + h.hexdigest()
def is_soong_prebuilt_module(file_metadata):
return file_metadata['soong_module_type'] and file_metadata['soong_module_type'] in [
'android_app_import', 'android_library_import', 'cc_prebuilt_binary', 'cc_prebuilt_library',
'cc_prebuilt_library_headers', 'cc_prebuilt_library_shared', 'cc_prebuilt_library_static', 'cc_prebuilt_object',
'dex_import', 'java_import', 'java_sdk_library_import', 'java_system_modules_import',
'libclang_rt_prebuilt_library_static', 'libclang_rt_prebuilt_library_shared', 'llvm_prebuilt_library_static',
'ndk_prebuilt_object', 'ndk_prebuilt_shared_stl', 'nkd_prebuilt_static_stl', 'prebuilt_apex',
'prebuilt_bootclasspath_fragment', 'prebuilt_dsp', 'prebuilt_firmware', 'prebuilt_kernel_modules',
'prebuilt_rfsa', 'prebuilt_root', 'rust_prebuilt_dylib', 'rust_prebuilt_library', 'rust_prebuilt_rlib',
'vndk_prebuilt_shared',
# 'android_test_import',
# 'cc_prebuilt_test_library_shared',
# 'java_import_host',
# 'java_test_import',
# 'llvm_host_prebuilt_library_shared',
# 'prebuilt_apis',
# 'prebuilt_build_tool',
# 'prebuilt_defaults',
# 'prebuilt_etc',
# 'prebuilt_etc_host',
# 'prebuilt_etc_xml',
# 'prebuilt_font',
# 'prebuilt_hidl_interfaces',
# 'prebuilt_platform_compat_config',
# 'prebuilt_stubs_sources',
# 'prebuilt_usr_share',
# 'prebuilt_usr_share_host',
# 'soong_config_module_type_import',
]
def is_source_package(file_metadata):
module_path = file_metadata['module_path']
return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)
def is_prebuilt_package(file_metadata):
module_path = file_metadata['module_path']
if module_path:
return (module_path.startswith('prebuilts/') or
is_soong_prebuilt_module(file_metadata) or
file_metadata['is_prebuilt_make_module'])
kernel_module_copy_files = file_metadata['kernel_module_copy_files']
if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
return True
return False
def get_source_package_info(file_metadata, metadata_file_path):
if not metadata_file_path:
return file_metadata['module_path'], []
metadata_proto = metadata_file_protos[metadata_file_path]
external_refs = []
for tag in metadata_proto.third_party.security.tag:
if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
external_refs.append("{}: SECURITY cpe23Type {}".format(PACKAGE_EXTERNAL_REF, tag.removeprefix(NVD_CPE23)))
elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
external_refs.append("{}: SECURITY cpe22Type {}".format(PACKAGE_EXTERNAL_REF, tag.removeprefix(NVD_CPE23)))
if metadata_proto.name:
return metadata_proto.name, external_refs
else:
return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name
def get_prebuilt_package_name(file_metadata, metadata_file_path):
name = None
if metadata_file_path:
metadata_proto = metadata_file_protos[metadata_file_path]
if metadata_proto.name:
name = metadata_proto.name
else:
name = metadata_file_path
elif file_metadata['module_path']:
name = file_metadata['module_path']
elif file_metadata['kernel_module_copy_files']:
src_path = file_metadata['kernel_module_copy_files'].split(':')[0]
name = os.path.dirname(src_path)
return name.removeprefix('prebuilts/').replace('/', '-')
def get_metadata_file_path(file_metadata):
metadata_path = ''
if file_metadata['module_path']:
metadata_path = file_metadata['module_path']
elif file_metadata['kernel_module_copy_files']:
metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0])
while metadata_path and not os.path.exists(metadata_path + '/METADATA'):
metadata_path = os.path.dirname(metadata_path)
return metadata_path
def get_package_version(metadata_file_path):
if not metadata_file_path:
return None
metadata_proto = metadata_file_protos[metadata_file_path]
return metadata_proto.third_party.version
def get_package_homepage(metadata_file_path):
if not metadata_file_path:
return None
metadata_proto = metadata_file_protos[metadata_file_path]
if metadata_proto.third_party.homepage:
return metadata_proto.third_party.homepage
for url in metadata_proto.third_party.url:
if url.type == metadata_file_pb2.URL.Type.HOMEPAGE:
return url.value
return None
def get_package_download_location(metadata_file_path):
if not metadata_file_path:
return None
metadata_proto = metadata_file_protos[metadata_file_path]
if metadata_proto.third_party.url:
urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type)
if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE:
return urls[0].value
elif len(urls) > 1:
return urls[1].value
return None
def get_sbom_fragments(installed_file_metadata, metadata_file_path):
external_doc_ref = None
packages = []
relationships = []
# Info from METADATA file
homepage = get_package_homepage(metadata_file_path)
version = get_package_version(metadata_file_path)
download_location = get_package_download_location(metadata_file_path)
if is_source_package(installed_file_metadata):
# Source fork packages
name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
source_package_id = new_package_id(name, PKG_SOURCE)
source_package = new_package_record(source_package_id, name, args.build_version, args.product_mfr,
external_refs=external_refs)
upstream_package_id = new_package_id(name, PKG_UPSTREAM)
upstream_package = new_package_record(upstream_package_id, name, version, homepage, download_location)
packages += [source_package, upstream_package]
relationships.append(new_relationship_record(source_package_id, REL_VARIANT_OF, upstream_package_id))
elif is_prebuilt_package(installed_file_metadata):
# Prebuilt fork packages
name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
prebuilt_package = new_package_record(prebuilt_package_id, name, args.build_version, args.product_mfr)
packages.append(prebuilt_package)
if metadata_file_path:
metadata_proto = metadata_file_protos[metadata_file_path]
if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
sbom_url = metadata_proto.third_party.sbom_ref.url
sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
if sbom_url and sbom_checksum and upstream_element_id:
external_doc_ref, doc_ref_id = new_external_doc_ref(name, sbom_url, sbom_checksum)
relationships.append(
new_relationship_record(prebuilt_package_id, REL_VARIANT_OF, doc_ref_id + ':' + upstream_element_id))
return external_doc_ref, packages, relationships
def generate_package_verification_code(files):
checksums = [file[FILE_CHECKSUM] for file in files]
checksums.sort()
h = hashlib.sha1()
h.update(''.join(checksums).encode(encoding='utf-8'))
return h.hexdigest()
def write_record(f, record):
if record.__class__.__name__ == 'dict':
for k, v in record.items():
if k == EXTERNAL_DOCUMENT_REF or k == PACKAGE_EXTERNAL_REF:
for ref in v:
f.write(ref + '\n')
else:
f.write('{}: {}\n'.format(k, v))
elif record.__class__.__name__ == 'str':
f.write(record + '\n')
f.write('\n')
def write_tagvalue_sbom(all_records):
with open(args.output_file, 'w', encoding="utf-8") as output_file:
for rec in all_records:
write_record(output_file, rec)
def write_json_sbom(all_records, product_package_id):
doc = {}
product_package = None
for r in all_records:
if r.__class__.__name__ == 'dict':
if DOCUMENT_NAME in r: # Doc header
doc['spdxVersion'] = r[SPDX_VERSION]
doc['dataLicense'] = r[DATA_LICENSE]
doc[SPDXID] = r[SPDXID]
doc['name'] = r[DOCUMENT_NAME]
doc['documentNamespace'] = r[DOCUMENT_NAMESPACE]
doc['creationInfo'] = {
'creators': [r[CREATOR]],
'created': r[CREATED],
}
doc['externalDocumentRefs'] = []
for ref in r[EXTERNAL_DOCUMENT_REF]:
# ref is 'ExternalDocumentRef: <doc id> <doc url> SHA1: xxxxx'
fields = ref.split(' ')
doc_ref = {
'externalDocumentId': fields[1],
'spdxDocument': fields[2],
'checksum': {
'algorithm': fields[3][:-1],
'checksumValue': fields[4]
}
}
doc['externalDocumentRefs'].append(doc_ref)
doc['documentDescribes'] = []
doc['packages'] = []
doc['files'] = []
doc['relationships'] = []
elif PACKAGE_NAME in r: # packages
package = {
'name': r[PACKAGE_NAME],
SPDXID: r[SPDXID],
'downloadLocation': r[PACKAGE_DOWNLOAD_LOCATION],
'filesAnalyzed': r[FILES_ANALYZED] == "true"
}
if PACKAGE_VERSION in r:
package['versionInfo'] = r[PACKAGE_VERSION]
if PACKAGE_SUPPLIER in r:
package['supplier'] = r[PACKAGE_SUPPLIER]
if PACKAGE_VERIFICATION_CODE in r:
package['packageVerificationCode'] = {
'packageVerificationCodeValue': r[PACKAGE_VERIFICATION_CODE]
}
if PACKAGE_EXTERNAL_REF in r:
package['externalRefs'] = []
for ref in r[PACKAGE_EXTERNAL_REF]:
# ref is 'ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4'
fields = ref.split(' ')
ext_ref = {
'referenceCategory': fields[1],
'referenceType': fields[2],
'referenceLocator': fields[3],
}
package['externalRefs'].append(ext_ref)
doc['packages'].append(package)
if r[SPDXID] == product_package_id:
product_package = package
product_package['hasFiles'] = []
elif FILE_NAME in r: # files
file = {
'fileName': r[FILE_NAME],
SPDXID: r[SPDXID]
}
checksum = r[FILE_CHECKSUM].split(': ')
file['checksums'] = [{
'algorithm': checksum[0],
'checksumValue': checksum[1],
}]
doc['files'].append(file)
product_package['hasFiles'].append(r[SPDXID])
elif r.__class__.__name__ == 'str':
if r.startswith(RELATIONSHIP):
# r is 'Relationship: <spdxid> <relationship> <spdxid>'
fields = r.split(' ')
rel = {
'spdxElementId': fields[1],
'relatedSpdxElement': fields[3],
'relationshipType': fields[2],
}
if fields[2] == REL_DESCRIBES:
doc['documentDescribes'].append(fields[3])
else:
doc['relationships'].append(rel)
with open(args.output_file + '.json', 'w', encoding="utf-8") as output_file:
output_file.write(json.dumps(doc, indent=4))
def save_report(report):
prefix, _ = os.path.splitext(args.output_file)
with open(prefix + '-gen-report.txt', 'w', encoding="utf-8") as report_file:
for type, issues in report.items():
report_file.write(type + '\n')
for issue in issues:
report_file.write('\t' + issue + '\n')
report_file.write('\n')
def sort_rels(rel):
# rel = 'Relationship file_id GENERATED_FROM package_id'
fields = rel.split(' ')
return fields[3] + fields[1]
# Validate the metadata generated by Make for installed files and report if there is no metadata.
def installed_file_has_metadata(installed_file_metadata, report):
installed_file = installed_file_metadata['installed_file']
module_path = installed_file_metadata['module_path']
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
is_platform_generated = installed_file_metadata['is_platform_generated']
if (not module_path and
not product_copy_files and
not kernel_module_copy_files and
not is_platform_generated and
not installed_file.endswith('.fsv_meta')):
report[ISSUE_NO_METADATA].append(installed_file)
return False
return True
def report_metadata_file(metadata_file_path, installed_file_metadata, report):
if metadata_file_path:
report[INFO_METADATA_FOUND_FOR_PACKAGE].append(
"installed_file: {}, module_path: {}, METADATA file: {}".format(
installed_file_metadata['installed_file'],
installed_file_metadata['module_path'],
metadata_file_path + '/METADATA'))
package_metadata = metadata_file_pb2.Metadata()
with open(metadata_file_path + '/METADATA', "rt") as f:
text_format.Parse(f.read(), package_metadata)
if not metadata_file_path in metadata_file_protos:
metadata_file_protos[metadata_file_path] = package_metadata
if not package_metadata.name:
report[ISSUE_METADATA_FILE_INCOMPLETE].append('{} does not has "name"'.format(metadata_file_path + '/METADATA'))
if not package_metadata.third_party.version:
report[ISSUE_METADATA_FILE_INCOMPLETE].append(
'{} does not has "third_party.version"'.format(metadata_file_path + '/METADATA'))
for tag in package_metadata.third_party.security.tag:
if not tag.startswith(NVD_CPE23):
report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append(
"Unknown security tag type: {} in {}".format(tag, metadata_file_path + '/METADATA'))
else:
report[ISSUE_NO_METADATA_FILE].append(
"installed_file: {}, module_path: {}".format(
installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
def generate_fragment():
with open(args.metadata, newline='') as sbom_metadata_file:
reader = csv.DictReader(sbom_metadata_file)
for installed_file_metadata in reader:
installed_file = installed_file_metadata['installed_file']
if args.output_file != args.product_out_dir + installed_file + ".spdx":
continue
module_path = installed_file_metadata['module_path']
package_id = new_package_id(encode_for_spdxid(module_path), PKG_PREBUILT)
package = new_package_record(package_id, module_path, args.build_version, args.product_mfr)
file_id = new_file_id(installed_file)
file = new_file_record(file_id, installed_file, checksum(installed_file))
relationship = new_relationship_record(file_id, REL_GENERATED_FROM, package_id)
records = [package, file, relationship]
write_tagvalue_sbom(records)
break
def main():
global args
args = get_args()
log("Args:", vars(args))
if args.unbundled:
generate_fragment()
return
global metadata_file_protos
metadata_file_protos = {}
doc_id = 'SPDXRef-DOCUMENT'
doc_header = new_doc_header(doc_id)
product_package_id = 'SPDXRef-PRODUCT'
product_package = new_package_record(product_package_id, 'PRODUCT', args.build_version, args.product_mfr,
files_analyzed='true')
platform_package_id = 'SPDXRef-PLATFORM'
platform_package = new_package_record(platform_package_id, 'PLATFORM', args.build_version, args.product_mfr)
# Report on some issues and information
report = {
ISSUE_NO_METADATA: [],
ISSUE_NO_METADATA_FILE: [],
ISSUE_METADATA_FILE_INCOMPLETE: [],
ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [],
INFO_METADATA_FOUND_FOR_PACKAGE: []
}
# Scan the metadata in CSV file and create the corresponding package and file records in SPDX
product_files = []
package_ids = []
package_records = []
rels_file_gen_from = []
with open(args.metadata, newline='') as sbom_metadata_file:
reader = csv.DictReader(sbom_metadata_file)
for installed_file_metadata in reader:
installed_file = installed_file_metadata['installed_file']
module_path = installed_file_metadata['module_path']
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
if not installed_file_has_metadata(installed_file_metadata, report):
continue
file_id = new_file_id(installed_file)
product_files.append(new_file_record(file_id, installed_file, checksum(installed_file)))
if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
metadata_file_path = get_metadata_file_path(installed_file_metadata)
report_metadata_file(metadata_file_path, installed_file_metadata, report)
# File from source fork packages or prebuilt fork packages
external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path)
if len(pkgs) > 0:
if external_doc_ref and external_doc_ref not in doc_header[EXTERNAL_DOCUMENT_REF]:
doc_header[EXTERNAL_DOCUMENT_REF].append(external_doc_ref)
for p in pkgs:
if not p[SPDXID] in package_ids:
package_ids.append(p[SPDXID])
package_records.append(p)
for rel in rels:
if not rel in package_records:
package_records.append(rel)
fork_package_id = pkgs[0][SPDXID] # The first package should be the source/prebuilt fork package
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, fork_package_id))
elif module_path or installed_file_metadata['is_platform_generated']:
# File from PLATFORM package
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
elif product_copy_files:
# Format of product_copy_files: <source path>:<dest path>
src_path = product_copy_files.split(':')[0]
# So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
# so process them as files from PLATFORM package
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
elif installed_file.endswith('.fsv_meta'):
# See build/make/core/Makefile:2988
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
elif kernel_module_copy_files.startswith('ANDROID-GEN'):
# For the four files generated for _dlkm, _ramdisk partitions
# See build/make/core/Makefile:323
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
product_package[PACKAGE_VERIFICATION_CODE] = generate_package_verification_code(product_files)
all_records = [
doc_header,
product_package,
new_relationship_record(doc_id, REL_DESCRIBES, product_package_id),
]
all_records += product_files
all_records.append(platform_package)
all_records += package_records
rels_file_gen_from.sort(key=sort_rels)
all_records += rels_file_gen_from
# Save SBOM records to output file
doc_header[CREATED] = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
write_tagvalue_sbom(all_records)
if args.json:
write_json_sbom(all_records, product_package_id)
save_report(report)
if __name__ == '__main__':
main()

32
tools/protos/Android.bp
View File

@@ -1,32 +0,0 @@
// Copyright 2023 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
python_library_host {
name: "metadata_file_proto_py",
version: {
py3: {
enabled: true,
},
},
srcs: [
"metadata_file.proto",
],
proto: {
canonical_path_from_root: false,
},
}

281
tools/protos/metadata_file.proto
View File

@@ -1,281 +0,0 @@
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package metadata_file;
// Proto definition of METADATA files of packages in AOSP codebase.
message Metadata {
// Name of the package.
optional string name = 1;
// A short description (a few lines) of the package.
// Example: "Handles location lookups, throttling, batching, etc."
optional string description = 2;
// Specifies additional data about third-party packages.
optional ThirdParty third_party = 3;
}
message ThirdParty {
// URL(s) associated with the package.
//
// At a minimum, all packages must specify a URL which identifies where it
// came from, containing a type of: ARCHIVE, GIT or OTHER. Typically,
// a package should contain only a single URL from these types. Occasionally,
// a package may be broken across multiple archive files for whatever reason,
// in which case having multiple ARCHIVE URLs is okay. However, this should
// not be used to combine different logical packages that are versioned and
// possibly licensed differently.
repeated URL url = 1;
// The package version. In order of preference, this should contain:
// - If the package comes from Git or another source control system,
// a specific tag or revision in source control, such as "r123" or
// "58e27d2". This MUST NOT be a mutable ref such as a branch name.
// - a released package version such as "1.0", "2.3-beta", etc.
// - the date the package was retrieved, formatted as "As of YYYY-MM-DD".
optional string version = 2;
// The date of the change in which the package was last upgraded from
// upstream.
// This should only identify package upgrades from upstream, not local
// modifications. This may identify the date of either the original or
// merged change.
//
// Note: this is NOT the date that this version of the package was released
// externally.
optional Date last_upgrade_date = 3;
// License type that identifies how the package may be used.
optional LicenseType license_type = 4;
// An additional note explaining the licensing of this package. This is most
// commonly used with commercial license.
optional string license_note = 5;
// Description of local changes that have been made to the package. This does
// not need to (and in most cases should not) attempt to include an exhaustive
// list of all changes, but may instead direct readers to review the local
// commit history, a collection of patch files, a separate README.md (or
// similar) document, etc.
// Note: Use of this field to store IDs of advisories fixed with a backported
// patch is deprecated, use "security.mitigated_security_patch" instead.
optional string local_modifications = 6;
// Security related metadata including risk category and any special
// instructions for using the package, as determined by an ISE-TPS review.
optional Security security = 7;
// The type of directory this metadata represents.
optional DirectoryType type = 8 [default = PACKAGE];
// The homepage for the package. This will eventually replace
// `url { type: HOMEPAGE }`
optional string homepage = 9;
// SBOM information of the package. It is mandatory for prebuilt packages.
oneof sbom {
// Reference to external SBOM document provided as URL.
SBOMRef sbom_ref = 10;
}
}
// URL associated with a third-party package.
message URL {
enum Type {
// The homepage for the package. For example, "https://bazel.io/". This URL
// is optional, but encouraged to help disambiguate similarly named packages
// or to get more information about the package. This is especially helpful
// when no other URLs provide human readable resources (such as git:// or
// sso:// URLs).
HOMEPAGE = 1;
// The URL of the archive containing the source code for the package, for
// example a zip or tgz file.
ARCHIVE = 2;
// The URL of the upstream git repository this package is retrieved from.
// For example:
// - https://github.com/git/git.git
// - git://git.kernel.org/pub/scm/git/git.git
//
// Use of a git URL requires that the package "version" value must specify a
// specific git tag or revision.
GIT = 3;
// The URL of the upstream SVN repository this package is retrieved from.
// For example:
// - http://llvm.org/svn/llvm-project/llvm/
//
// Use of an SVN URL requires that the package "version" value must specify
// a specific SVN tag or revision.
SVN = 4;
// The URL of the upstream mercurial repository this package is retrieved
// from. For example:
// - https://mercurial-scm.org/repo/evolve
//
// Use of a mercurial URL requires that the package "version" value must
// specify a specific tag or revision.
HG = 5;
// The URL of the upstream darcs repository this package is retrieved
// from. For example:
// - https://hub.darcs.net/hu.dwim/hu.dwim.util
//
// Use of a DARCS URL requires that the package "version" value must
// specify a specific tag or revision.
DARCS = 6;
PIPER = 7;
// A URL that does not fit any other type. This may also indicate that the
// source code was received via email or some other out-of-band way. This is
// most commonly used with commercial software received directly from the
// vendor. In the case of email, the URL value can be used to provide
// additional information about how it was received.
OTHER = 8;
// The URL identifying where the local copy of the package source code can
// be found.
//
// Typically, the metadata files describing a package reside in the same
// directory as the source code for the package. In a few rare cases where
// they are separate, the LOCAL_SOURCE URL identifies where to find the
// source code. This only describes where to find the local copy of the
// source; there should always be an additional URL describing where the
// package was retrieved from.
//
// Examples:
// - https://android.googlesource.com/platform/external/apache-http/
LOCAL_SOURCE = 9;
}
// The type of resource this URL identifies.
optional Type type = 1;
// The actual URL value. URLs should be absolute and start with 'http://' or
// 'https://' (or occasionally 'git://' or 'ftp://' where appropriate).
optional string value = 2;
}
// License type that identifies how the packages may be used.
enum LicenseType {
BY_EXCEPTION_ONLY = 1;
NOTICE = 2;
PERMISSIVE = 3;
RECIPROCAL = 4;
RESTRICTED_IF_STATICALLY_LINKED = 5;
RESTRICTED = 6;
UNENCUMBERED = 7;
}
// Identifies security related metadata including risk category and any special
// instructions for using the package.
message Security {
// Security risk category for a package, as determined by an ISE-TPS review.
enum Category {
CATEGORY_UNSPECIFIED = 0;
// Package should only be used in a sandboxed environment.
// Package should have restricted visibility.
SANDBOXED_ONLY = 1;
// Package should not be used to process user content. It is considered
// safe to use to process trusted data only. Package should have restricted
// visibility.
TRUSTED_DATA_ONLY = 2;
// Package is considered safe to use.
REVIEWED_AND_SECURE = 3;
}
// Identifies the security risk category for the package. This will be
// provided by the ISE-TPS team as the result of a security review of the
// package.
optional Category category = 1;
// An additional security note for the package.
optional string note = 2;
// Text tag to categorize the package. It's currently used by security to:
// - to disable OSV (https://osv.dev)
// support via the `OSV:disable` tag
// - to attach CPE to their corresponding packages, for vulnerability
// monitoring:
//
// Please do document your usecase here should you want to add one.
repeated string tag = 3;
// ID of advisories fixed with a mitigated patch, for example CVE-2018-1111.
repeated string mitigated_security_patch = 4;
}
enum DirectoryType {
UNDEFINED = 0;
// This directory represents a package.
PACKAGE = 1;
// This directory is designed to organize multiple third-party PACKAGE
// directories.
GROUP = 2;
// This directory contains several PACKAGE directories representing
// different versions of the same third-party project.
VERSIONS = 3;
}
// Represents a whole or partial calendar date, such as a birthday. The time of
// day and time zone are either specified elsewhere or are insignificant. The
// date is relative to the Gregorian Calendar. This can represent one of the
// following:
//
// * A full date, with non-zero year, month, and day values.
// * A month and day, with a zero year (for example, an anniversary).
// * A year on its own, with a zero month and a zero day.
// * A year and month, with a zero day (for example, a credit card expiration
// date).
message Date {
// Year of the date. Must be from 1 to 9999, or 0 to specify a date without
// a year.
optional int32 year = 1;
// Month of a year. Must be from 1 to 12, or 0 to specify a year without a
// month and day.
optional int32 month = 2;
// Day of a month. Must be from 1 to 31 and valid for the year and month, or 0
// to specify a year by itself or a year and month where the day isn't
// significant.
optional int32 day = 3;
}
// Reference to external SBOM document and element corresponding to the package.
// See https://spdx.github.io/spdx-spec/v2.3/document-creation-information/#66-external-document-references-field
message SBOMRef {
// The URL that points to the SBOM document of the upstream package of this
// third_party package.
optional string url = 1;
// Checksum of the SBOM document the url field points to.
// Format: e.g. SHA1:<checksum>, or any algorithm defined in
// https://spdx.github.io/spdx-spec/v2.3/file-information/#8.4
optional string checksum = 2;
// SPDXID of the upstream package/file defined in the SBOM document the url field points to.
// Format: SPDXRef-[a-zA-Z0-9.-]+, see
// https://spdx.github.io/spdx-spec/v2.3/package-information/#72-package-spdx-identifier-field or
// https://spdx.github.io/spdx-spec/v2.3/file-information/#82-file-spdx-identifier-field
optional string element_id = 3;
}