Merge "Create separate python libraries for the following logic and refactor SBOM generation script accordingly." am: ebf41e9a91 am: 28fc5a97a7 am: 074bfc1264

Original change: https://android-review.googlesource.com/c/platform/build/+/2525827

Change-Id: I0391804c0924a64ae58b71e99815cecd6c078309
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Treehugger Robot 2023-04-14 21:56:20 +00:00 committed by Automerger Merge Worker
commit 55decf2e29
9 changed files with 1020 additions and 285 deletions

View file

@ -70,22 +70,6 @@ python_binary_host {
srcs: ["generate_gts_shared_report.py"],
}
python_binary_host {
name: "generate-sbom",
srcs: [
"generate-sbom.py",
],
version: {
py3: {
embedded_launcher: true,
},
},
libs: [
"metadata_file_proto_py",
"libprotobuf-python",
],
}
python_binary_host {
name: "list_files",
main: "list_files.py",

53
tools/sbom/Android.bp Normal file
View file

@ -0,0 +1,53 @@
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
python_binary_host {
name: "generate-sbom",
srcs: [
"generate-sbom.py",
],
version: {
py3: {
embedded_launcher: true,
},
},
libs: [
"metadata_file_proto_py",
"libprotobuf-python",
"sbom_lib",
],
}
python_library_host {
name: "sbom_lib",
srcs: [
"sbom_data.py",
"sbom_writers.py",
],
}
python_test_host {
name: "sbom_writers_test",
main: "sbom_writers_test.py",
srcs: [
"sbom_writers_test.py",
],
data: [
"testdata/*",
],
libs: [
"sbom_lib",
],
test_suites: ["general-tests"],
}

View file

@ -29,50 +29,11 @@ import csv
import datetime
import google.protobuf.text_format as text_format
import hashlib
import json
import os
import metadata_file_pb2
import sbom_data
import sbom_writers
# Common
SPDXID = 'SPDXID'
SPDX_VERSION = 'SPDXVersion'
DATA_LICENSE = 'DataLicense'
DOCUMENT_NAME = 'DocumentName'
DOCUMENT_NAMESPACE = 'DocumentNamespace'
CREATED = 'Created'
CREATOR = 'Creator'
EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef'
# Package
PACKAGE_NAME = 'PackageName'
PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation'
PACKAGE_VERSION = 'PackageVersion'
PACKAGE_SUPPLIER = 'PackageSupplier'
FILES_ANALYZED = 'FilesAnalyzed'
PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode'
PACKAGE_EXTERNAL_REF = 'ExternalRef'
# Package license
PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded'
PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles'
PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared'
PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments'
# File
FILE_NAME = 'FileName'
FILE_CHECKSUM = 'FileChecksum'
# File license
FILE_LICENSE_CONCLUDED = 'LicenseConcluded'
FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile'
FILE_LICENSE_COMMENTS = 'LicenseComments'
FILE_COPYRIGHT_TEXT = 'FileCopyrightText'
FILE_NOTICE = 'FileNotice'
FILE_ATTRIBUTION_TEXT = 'FileAttributionText'
# Relationship
RELATIONSHIP = 'Relationship'
REL_DESCRIBES = 'DESCRIBES'
REL_VARIANT_OF = 'VARIANT_OF'
REL_GENERATED_FROM = 'GENERATED_FROM'
# Package type
PKG_SOURCE = 'SOURCE'
@ -111,44 +72,6 @@ def log(*info):
print(i)
def new_doc_header(doc_id):
return {
SPDX_VERSION: 'SPDX-2.3',
DATA_LICENSE: 'CC0-1.0',
SPDXID: doc_id,
DOCUMENT_NAME: args.build_version,
DOCUMENT_NAMESPACE: f'https://www.google.com/sbom/spdx/android/{args.build_version}',
CREATOR: 'Organization: Google, LLC',
CREATED: '<timestamp>',
EXTERNAL_DOCUMENT_REF: [],
}
def new_package_record(id, name, version, supplier, download_location=None, files_analyzed='false', external_refs=[]):
package = {
PACKAGE_NAME: name,
SPDXID: id,
PACKAGE_DOWNLOAD_LOCATION: download_location if download_location else 'NONE',
FILES_ANALYZED: files_analyzed,
}
if version:
package[PACKAGE_VERSION] = version
if supplier:
package[PACKAGE_SUPPLIER] = f'Organization: {supplier}'
if external_refs:
package[PACKAGE_EXTERNAL_REF] = external_refs
return package
def new_file_record(id, name, checksum):
return {
FILE_NAME: name,
SPDXID: id,
FILE_CHECKSUM: checksum
}
def encode_for_spdxid(s):
"""Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-"""
result = ''
@ -167,19 +90,10 @@ def new_package_id(package_name, type):
return f'SPDXRef-{type}-{encode_for_spdxid(package_name)}'
def new_external_doc_ref(package_name, sbom_url, sbom_checksum):
doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(package_name)}'
return f'{EXTERNAL_DOCUMENT_REF}: {doc_ref_id} {sbom_url} {sbom_checksum}', doc_ref_id
def new_file_id(file_path):
return f'SPDXRef-{encode_for_spdxid(file_path)}'
def new_relationship_record(id1, relationship, id2):
return f'{RELATIONSHIP}: {id1} {relationship} {id2}'
def checksum(file_path):
file_path = args.product_out_dir + '/' + file_path
h = hashlib.sha1()
@ -243,6 +157,11 @@ def is_prebuilt_package(file_metadata):
def get_source_package_info(file_metadata, metadata_file_path):
"""Return source package info exists in its METADATA file, currently including name, security tag
and external SBOM reference.
See go/android-spdx and go/android-sbom-gen for more details.
"""
if not metadata_file_path:
return file_metadata['module_path'], []
@ -250,9 +169,15 @@ def get_source_package_info(file_metadata, metadata_file_path):
external_refs = []
for tag in metadata_proto.third_party.security.tag:
if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
external_refs.append(f'{PACKAGE_EXTERNAL_REF}: SECURITY cpe23Type {tag.removeprefix(NVD_CPE23)}')
external_refs.append(
sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
type=sbom_data.PackageExternalRefType.cpe23Type,
locator=tag.removeprefix(NVD_CPE23)))
elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
external_refs.append(f'{PACKAGE_EXTERNAL_REF}: SECURITY cpe22Type {tag.removeprefix(NVD_CPE23)}')
external_refs.append(
sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
type=sbom_data.PackageExternalRefType.cpe22Type,
locator=tag.removeprefix(NVD_CPE23)))
if metadata_proto.name:
return metadata_proto.name, external_refs
@ -261,6 +186,11 @@ def get_source_package_info(file_metadata, metadata_file_path):
def get_prebuilt_package_name(file_metadata, metadata_file_path):
"""Return name of a prebuilt package, which can be from the METADATA file, metadata file path,
module path or kernel module's source path if the installed file is a kernel module.
See go/android-spdx and go/android-sbom-gen for more details.
"""
name = None
if metadata_file_path:
metadata_proto = metadata_file_protos[metadata_file_path]
@ -278,6 +208,7 @@ def get_prebuilt_package_name(file_metadata, metadata_file_path):
def get_metadata_file_path(file_metadata):
"""Search for METADATA file of a package and return its path."""
metadata_path = ''
if file_metadata['module_path']:
metadata_path = file_metadata['module_path']
@ -291,6 +222,7 @@ def get_metadata_file_path(file_metadata):
def get_package_version(metadata_file_path):
"""Return a package's version in its METADATA file."""
if not metadata_file_path:
return None
metadata_proto = metadata_file_protos[metadata_file_path]
@ -298,6 +230,7 @@ def get_package_version(metadata_file_path):
def get_package_homepage(metadata_file_path):
"""Return a package's homepage URL in its METADATA file."""
if not metadata_file_path:
return None
metadata_proto = metadata_file_protos[metadata_file_path]
@ -311,6 +244,7 @@ def get_package_homepage(metadata_file_path):
def get_package_download_location(metadata_file_path):
"""Return a package's code repository URL in its METADATA file."""
if not metadata_file_path:
return None
metadata_proto = metadata_file_protos[metadata_file_path]
@ -325,6 +259,12 @@ def get_package_download_location(metadata_file_path):
def get_sbom_fragments(installed_file_metadata, metadata_file_path):
"""Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT
package, a UPSTREAM package if it's a source package and a external SBOM document reference if
it's a prebuilt package with sbom_ref defined in its METADATA file.
See go/android-spdx and go/android-sbom-gen for more details.
"""
external_doc_ref = None
packages = []
relationships = []
@ -338,18 +278,26 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path):
# Source fork packages
name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
source_package_id = new_package_id(name, PKG_SOURCE)
source_package = new_package_record(source_package_id, name, args.build_version, args.product_mfr,
external_refs=external_refs)
source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version,
supplier='Organization: ' + args.product_mfr,
external_refs=external_refs)
upstream_package_id = new_package_id(name, PKG_UPSTREAM)
upstream_package = new_package_record(upstream_package_id, name, version, homepage, download_location)
upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
supplier='Organization: ' + homepage if homepage else None,
download_location=download_location)
packages += [source_package, upstream_package]
relationships.append(new_relationship_record(source_package_id, REL_VARIANT_OF, upstream_package_id))
relationships.append(sbom_data.Relationship(id1=source_package_id,
relationship=sbom_data.RelationshipType.VARIANT_OF,
id2=upstream_package_id))
elif is_prebuilt_package(installed_file_metadata):
# Prebuilt fork packages
name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
prebuilt_package = new_package_record(prebuilt_package_id, name, args.build_version, args.product_mfr)
prebuilt_package = sbom_data.Package(id=prebuilt_package_id,
name=name,
version=args.build_version,
supplier='Organization: ' + args.product_mfr)
packages.append(prebuilt_package)
if metadata_file_path:
@ -359,136 +307,26 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path):
sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
if sbom_url and sbom_checksum and upstream_element_id:
external_doc_ref, doc_ref_id = new_external_doc_ref(name, sbom_url, sbom_checksum)
doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}'
external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
uri=sbom_url,
checksum=sbom_checksum)
relationships.append(
new_relationship_record(prebuilt_package_id, REL_VARIANT_OF, doc_ref_id + ':' + upstream_element_id))
sbom_data.Relationship(id1=prebuilt_package_id,
relationship=sbom_data.RelationshipType.VARIANT_OF,
id2=doc_ref_id + ':' + upstream_element_id))
return external_doc_ref, packages, relationships
def generate_package_verification_code(files):
checksums = [file[FILE_CHECKSUM] for file in files]
checksums = [file.checksum for file in files]
checksums.sort()
h = hashlib.sha1()
h.update(''.join(checksums).encode(encoding='utf-8'))
return h.hexdigest()
def write_record(f, record):
if record.__class__.__name__ == 'dict':
for k, v in record.items():
if k == EXTERNAL_DOCUMENT_REF or k == PACKAGE_EXTERNAL_REF:
for ref in v:
f.write(ref + '\n')
else:
f.write('{}: {}\n'.format(k, v))
elif record.__class__.__name__ == 'str':
f.write(record + '\n')
f.write('\n')
def write_tagvalue_sbom(all_records):
with open(args.output_file, 'w', encoding="utf-8") as output_file:
for rec in all_records:
write_record(output_file, rec)
def write_json_sbom(all_records, product_package_id):
doc = {}
product_package = None
for r in all_records:
if r.__class__.__name__ == 'dict':
if DOCUMENT_NAME in r: # Doc header
doc['spdxVersion'] = r[SPDX_VERSION]
doc['dataLicense'] = r[DATA_LICENSE]
doc[SPDXID] = r[SPDXID]
doc['name'] = r[DOCUMENT_NAME]
doc['documentNamespace'] = r[DOCUMENT_NAMESPACE]
doc['creationInfo'] = {
'creators': [r[CREATOR]],
'created': r[CREATED],
}
doc['externalDocumentRefs'] = []
for ref in r[EXTERNAL_DOCUMENT_REF]:
# ref is 'ExternalDocumentRef: <doc id> <doc url> SHA1: xxxxx'
fields = ref.split(' ')
doc_ref = {
'externalDocumentId': fields[1],
'spdxDocument': fields[2],
'checksum': {
'algorithm': fields[3][:-1],
'checksumValue': fields[4]
}
}
doc['externalDocumentRefs'].append(doc_ref)
doc['documentDescribes'] = []
doc['packages'] = []
doc['files'] = []
doc['relationships'] = []
elif PACKAGE_NAME in r: # packages
package = {
'name': r[PACKAGE_NAME],
SPDXID: r[SPDXID],
'downloadLocation': r[PACKAGE_DOWNLOAD_LOCATION],
'filesAnalyzed': r[FILES_ANALYZED] == "true"
}
if PACKAGE_VERSION in r:
package['versionInfo'] = r[PACKAGE_VERSION]
if PACKAGE_SUPPLIER in r:
package['supplier'] = r[PACKAGE_SUPPLIER]
if PACKAGE_VERIFICATION_CODE in r:
package['packageVerificationCode'] = {
'packageVerificationCodeValue': r[PACKAGE_VERIFICATION_CODE]
}
if PACKAGE_EXTERNAL_REF in r:
package['externalRefs'] = []
for ref in r[PACKAGE_EXTERNAL_REF]:
# ref is 'ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4'
fields = ref.split(' ')
ext_ref = {
'referenceCategory': fields[1],
'referenceType': fields[2],
'referenceLocator': fields[3],
}
package['externalRefs'].append(ext_ref)
doc['packages'].append(package)
if r[SPDXID] == product_package_id:
product_package = package
product_package['hasFiles'] = []
elif FILE_NAME in r: # files
file = {
'fileName': r[FILE_NAME],
SPDXID: r[SPDXID]
}
checksum = r[FILE_CHECKSUM].split(': ')
file['checksums'] = [{
'algorithm': checksum[0],
'checksumValue': checksum[1],
}]
doc['files'].append(file)
product_package['hasFiles'].append(r[SPDXID])
elif r.__class__.__name__ == 'str':
if r.startswith(RELATIONSHIP):
# r is 'Relationship: <spdxid> <relationship> <spdxid>'
fields = r.split(' ')
rel = {
'spdxElementId': fields[1],
'relatedSpdxElement': fields[3],
'relationshipType': fields[2],
}
if fields[2] == REL_DESCRIBES:
doc['documentDescribes'].append(fields[3])
else:
doc['relationships'].append(rel)
with open(args.output_file + '.json', 'w', encoding="utf-8") as output_file:
output_file.write(json.dumps(doc, indent=4))
def save_report(report):
prefix, _ = os.path.splitext(args.output_file)
with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file:
@ -499,12 +337,6 @@ def save_report(report):
report_file.write('\n')
def sort_rels(rel):
# rel = 'Relationship file_id GENERATED_FROM package_id'
fields = rel.split(' ')
return fields[3] + fields[1]
# Validate the metadata generated by Make for installed files and report if there is no metadata.
def installed_file_has_metadata(installed_file_metadata, report):
installed_file = installed_file_metadata['installed_file']
@ -555,24 +387,38 @@ def report_metadata_file(metadata_file_path, installed_file_metadata, report):
installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
def generate_fragment():
def generate_sbom_for_unbundled():
with open(args.metadata, newline='') as sbom_metadata_file:
reader = csv.DictReader(sbom_metadata_file)
doc = sbom_data.Document(name=args.build_version,
namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
creators=['Organization: ' + args.product_mfr])
for installed_file_metadata in reader:
installed_file = installed_file_metadata['installed_file']
if args.output_file != args.product_out_dir + installed_file + ".spdx":
continue
module_path = installed_file_metadata['module_path']
package_id = new_package_id(encode_for_spdxid(module_path), PKG_PREBUILT)
package = new_package_record(package_id, module_path, args.build_version, args.product_mfr)
package_id = new_package_id(module_path, PKG_PREBUILT)
package = sbom_data.Package(id=package_id,
name=module_path,
version=args.build_version,
supplier='Organization: ' + args.product_mfr)
file_id = new_file_id(installed_file)
file = new_file_record(file_id, installed_file, checksum(installed_file))
relationship = new_relationship_record(file_id, REL_GENERATED_FROM, package_id)
records = [package, file, relationship]
write_tagvalue_sbom(records)
file = sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file))
relationship = sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=package_id)
doc.add_package(package)
doc.files.append(file)
doc.describes = file_id
doc.add_relationship(relationship)
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
break
with open(args.output_file, 'w', encoding="utf-8") as file:
sbom_writers.TagValueWriter.write(doc, file, fragment=True)
def main():
global args
@ -580,21 +426,27 @@ def main():
log('Args:', vars(args))
if args.unbundled:
generate_fragment()
generate_sbom_for_unbundled()
return
global metadata_file_protos
metadata_file_protos = {}
doc_id = 'SPDXRef-DOCUMENT'
doc_header = new_doc_header(doc_id)
doc = sbom_data.Document(name=args.build_version,
namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
creators=['Organization: ' + args.product_mfr])
product_package_id = 'SPDXRef-PRODUCT'
product_package = new_package_record(product_package_id, 'PRODUCT', args.build_version, args.product_mfr,
files_analyzed='true')
product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,
name=sbom_data.PACKAGE_NAME_PRODUCT,
version=args.build_version,
supplier='Organization: ' + args.product_mfr,
files_analyzed=True)
doc.packages.append(product_package)
platform_package_id = 'SPDXRef-PLATFORM'
platform_package = new_package_record(platform_package_id, 'PLATFORM', args.build_version, args.product_mfr)
doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
name=sbom_data.PACKAGE_NAME_PLATFORM,
version=args.build_version,
supplier='Organization: ' + args.product_mfr))
# Report on some issues and information
report = {
@ -607,10 +459,6 @@ def main():
}
# Scan the metadata in CSV file and create the corresponding package and file records in SPDX
product_files = []
package_ids = []
package_records = []
rels_file_gen_from = []
with open(args.metadata, newline='') as sbom_metadata_file:
reader = csv.DictReader(sbom_metadata_file)
for installed_file_metadata in reader:
@ -627,7 +475,9 @@ def main():
continue
file_id = new_file_id(installed_file)
product_files.append(new_file_record(file_id, installed_file, checksum(installed_file)))
doc.files.append(
sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file)))
product_package.file_ids.append(file_id)
if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
metadata_file_path = get_metadata_file_path(installed_file_metadata)
@ -636,54 +486,50 @@ def main():
# File from source fork packages or prebuilt fork packages
external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path)
if len(pkgs) > 0:
if external_doc_ref and external_doc_ref not in doc_header[EXTERNAL_DOCUMENT_REF]:
doc_header[EXTERNAL_DOCUMENT_REF].append(external_doc_ref)
if external_doc_ref:
doc.add_external_ref(external_doc_ref)
for p in pkgs:
if not p[SPDXID] in package_ids:
package_ids.append(p[SPDXID])
package_records.append(p)
doc.add_package(p)
for rel in rels:
if not rel in package_records:
package_records.append(rel)
fork_package_id = pkgs[0][SPDXID] # The first package should be the source/prebuilt fork package
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, fork_package_id))
doc.add_relationship(rel)
fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=fork_package_id))
elif module_path or installed_file_metadata['is_platform_generated']:
# File from PLATFORM package
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
elif product_copy_files:
# Format of product_copy_files: <source path>:<dest path>
src_path = product_copy_files.split(':')[0]
# So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
# so process them as files from PLATFORM package
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
elif installed_file.endswith('.fsv_meta'):
# See build/make/core/Makefile:2988
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
elif kernel_module_copy_files.startswith('ANDROID-GEN'):
# For the four files generated for _dlkm, _ramdisk partitions
# See build/make/core/Makefile:323
rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id))
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
product_package[PACKAGE_VERIFICATION_CODE] = generate_package_verification_code(product_files)
all_records = [
doc_header,
product_package,
new_relationship_record(doc_id, REL_DESCRIBES, product_package_id),
]
all_records += product_files
all_records.append(platform_package)
all_records += package_records
rels_file_gen_from.sort(key=sort_rels)
all_records += rels_file_gen_from
product_package.verification_code = generate_package_verification_code(doc.files)
# Save SBOM records to output file
doc_header[CREATED] = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
write_tagvalue_sbom(all_records)
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
with open(args.output_file, 'w', encoding="utf-8") as file:
sbom_writers.TagValueWriter.write(doc, file)
if args.json:
write_json_sbom(all_records, product_package_id)
save_report(report)
with open(args.output_file+'.json', 'w', encoding="utf-8") as file:
sbom_writers.JSONWriter.write(doc, file)
if __name__ == '__main__':

120
tools/sbom/sbom_data.py Normal file
View file

@ -0,0 +1,120 @@
#!/usr/bin/env python3
#
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Define data classes that model SBOMs defined by SPDX. The data classes could be
written out to different formats (tagvalue, JSON, etc) of SPDX with corresponding
writer utilities.
Rrefer to SPDX 2.3 spec: https://spdx.github.io/spdx-spec/v2.3/ and go/android-spdx for details of
fields in each data class.
"""
from dataclasses import dataclass, field
from typing import List
SPDXID_DOC = 'SPDXRef-DOCUMENT'
SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
SPDXID_PLATFORM = 'SPDXRef-PLATFORM'
PACKAGE_NAME_PRODUCT = 'PRODUCT'
PACKAGE_NAME_PLATFORM = 'PLATFORM'
class PackageExternalRefCategory:
SECURITY = 'SECURITY'
PACKAGE_MANAGER = 'PACKAGE-MANAGER'
PERSISTENT_ID = 'PERSISTENT-ID'
OTHER = 'OTHER'
class PackageExternalRefType:
cpe22Type = 'cpe22Type'
cpe23Type = 'cpe23Type'
@dataclass
class PackageExternalRef:
category: PackageExternalRefCategory
type: PackageExternalRefType
locator: str
@dataclass
class Package:
name: str
id: str
version: str = None
supplier: str = None
download_location: str = None
files_analyzed: bool = False
verification_code: str = None
file_ids: List[str] = field(default_factory=list)
external_refs: List[PackageExternalRef] = field(default_factory=list)
@dataclass
class File:
id: str
name: str
checksum: str
class RelationshipType:
DESCRIBES = 'DESCRIBES'
VARIANT_OF = 'VARIANT_OF'
GENERATED_FROM = 'GENERATED_FROM'
@dataclass
class Relationship:
id1: str
relationship: RelationshipType
id2: str
@dataclass
class DocumentExternalReference:
id: str
uri: str
checksum: str
@dataclass
class Document:
name: str
namespace: str
id: str = SPDXID_DOC
describes: str = SPDXID_PRODUCT
creators: List[str] = field(default_factory=list)
created: str = None
external_refs: List[DocumentExternalReference] = field(default_factory=list)
packages: List[Package] = field(default_factory=list)
files: List[File] = field(default_factory=list)
relationships: List[Relationship] = field(default_factory=list)
def add_external_ref(self, external_ref):
if not any(external_ref.uri == ref.uri for ref in self.external_refs):
self.external_refs.append(external_ref)
def add_package(self, package):
if not any(package.id == p.id for p in self.packages):
self.packages.append(package)
def add_relationship(self, rel):
if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
for r in self.relationships):
self.relationships.append(rel)

365
tools/sbom/sbom_writers.py Normal file
View file

@ -0,0 +1,365 @@
#!/usr/bin/env python3
#
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Serialize objects defined in package sbom_data to SPDX format: tagvalue, JSON.
"""
import json
import sbom_data
SPDX_VER = 'SPDX-2.3'
DATA_LIC = 'CC0-1.0'
class Tags:
# Common
SPDXID = 'SPDXID'
SPDX_VERSION = 'SPDXVersion'
DATA_LICENSE = 'DataLicense'
DOCUMENT_NAME = 'DocumentName'
DOCUMENT_NAMESPACE = 'DocumentNamespace'
CREATED = 'Created'
CREATOR = 'Creator'
EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef'
# Package
PACKAGE_NAME = 'PackageName'
PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation'
PACKAGE_VERSION = 'PackageVersion'
PACKAGE_SUPPLIER = 'PackageSupplier'
FILES_ANALYZED = 'FilesAnalyzed'
PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode'
PACKAGE_EXTERNAL_REF = 'ExternalRef'
# Package license
PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded'
PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles'
PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared'
PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments'
# File
FILE_NAME = 'FileName'
FILE_CHECKSUM = 'FileChecksum'
# File license
FILE_LICENSE_CONCLUDED = 'LicenseConcluded'
FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile'
FILE_LICENSE_COMMENTS = 'LicenseComments'
FILE_COPYRIGHT_TEXT = 'FileCopyrightText'
FILE_NOTICE = 'FileNotice'
FILE_ATTRIBUTION_TEXT = 'FileAttributionText'
# Relationship
RELATIONSHIP = 'Relationship'
class TagValueWriter:
@staticmethod
def marshal_doc_headers(sbom_doc):
headers = [
f'{Tags.SPDX_VERSION}: {SPDX_VER}',
f'{Tags.DATA_LICENSE}: {DATA_LIC}',
f'{Tags.SPDXID}: {sbom_doc.id}',
f'{Tags.DOCUMENT_NAME}: {sbom_doc.name}',
f'{Tags.DOCUMENT_NAMESPACE}: {sbom_doc.namespace}',
]
for creator in sbom_doc.creators:
headers.append(f'{Tags.CREATOR}: {creator}')
headers.append(f'{Tags.CREATED}: {sbom_doc.created}')
for doc_ref in sbom_doc.external_refs:
headers.append(
f'{Tags.EXTERNAL_DOCUMENT_REF}: {doc_ref.id} {doc_ref.uri} {doc_ref.checksum}')
headers.append('')
return headers
@staticmethod
def marshal_package(package):
download_location = 'NONE'
if package.download_location:
download_location = package.download_location
tagvalues = [
f'{Tags.PACKAGE_NAME}: {package.name}',
f'{Tags.SPDXID}: {package.id}',
f'{Tags.PACKAGE_DOWNLOAD_LOCATION}: {download_location}',
f'{Tags.FILES_ANALYZED}: {str(package.files_analyzed).lower()}',
]
if package.version:
tagvalues.append(f'{Tags.PACKAGE_VERSION}: {package.version}')
if package.supplier:
tagvalues.append(f'{Tags.PACKAGE_SUPPLIER}: {package.supplier}')
if package.verification_code:
tagvalues.append(f'{Tags.PACKAGE_VERIFICATION_CODE}: {package.verification_code}')
if package.external_refs:
for external_ref in package.external_refs:
tagvalues.append(
f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')
tagvalues.append('')
return tagvalues
@staticmethod
def marshal_described_element(sbom_doc):
if not sbom_doc.describes:
return None
product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
if product_package:
tagvalues = TagValueWriter.marshal_package(product_package[0])
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
return tagvalues
file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
if file:
tagvalues = [
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}'
]
return tagvalues
return None
@staticmethod
def marshal_packages(sbom_doc):
tagvalues = []
marshaled_relationships = []
i = 0
packages = sbom_doc.packages
while i < len(packages):
if packages[i].id == sbom_doc.describes:
i += 1
continue
if i + 1 < len(packages) \
and packages[i].id.startswith('SPDXRef-SOURCE-') \
and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
tagvalues += TagValueWriter.marshal_package(packages[i])
tagvalues += TagValueWriter.marshal_package(packages[i + 1])
rel = next((r for r in sbom_doc.relationships if
r.id1 == packages[i].id and
r.id2 == packages[i + 1].id and
r.relationship == sbom_data.RelationshipType.VARIANT_OF), None)
if rel:
marshaled_relationships.append(rel)
tagvalues.append(TagValueWriter.marshal_relationship(rel))
tagvalues.append('')
i += 2
else:
tagvalues += TagValueWriter.marshal_package(packages[i])
i += 1
return tagvalues, marshaled_relationships
@staticmethod
def marshal_file(file):
tagvalues = [
f'{Tags.FILE_NAME}: {file.name}',
f'{Tags.SPDXID}: {file.id}',
f'{Tags.FILE_CHECKSUM}: {file.checksum}',
'',
]
return tagvalues
@staticmethod
def marshal_files(sbom_doc):
tagvalues = []
for file in sbom_doc.files:
tagvalues += TagValueWriter.marshal_file(file)
return tagvalues
@staticmethod
def marshal_relationship(rel):
return f'{Tags.RELATIONSHIP}: {rel.id1} {rel.relationship} {rel.id2}'
@staticmethod
def marshal_relationships(sbom_doc, marshaled_rels):
tagvalues = []
sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.id2 + r.id1)
for rel in sorted_rels:
if any(r.id1 == rel.id1 and r.id2 == rel.id2 and r.relationship == rel.relationship
for r in marshaled_rels):
continue
tagvalues.append(TagValueWriter.marshal_relationship(rel))
tagvalues.append('')
return tagvalues
@staticmethod
def write(sbom_doc, file, fragment=False):
content = []
if not fragment:
content += TagValueWriter.marshal_doc_headers(sbom_doc)
described_element = TagValueWriter.marshal_described_element(sbom_doc)
if described_element:
content += described_element
content += TagValueWriter.marshal_files(sbom_doc)
tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
content += tagvalues
content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
file.write('\n'.join(content))
class PropNames:
# Common
SPDXID = 'SPDXID'
SPDX_VERSION = 'spdxVersion'
DATA_LICENSE = 'dataLicense'
NAME = 'name'
DOCUMENT_NAMESPACE = 'documentNamespace'
CREATION_INFO = 'creationInfo'
CREATORS = 'creators'
CREATED = 'created'
EXTERNAL_DOCUMENT_REF = 'externalDocumentRefs'
DOCUMENT_DESCRIBES = 'documentDescribes'
EXTERNAL_DOCUMENT_ID = 'externalDocumentId'
EXTERNAL_DOCUMENT_URI = 'spdxDocument'
EXTERNAL_DOCUMENT_CHECKSUM = 'checksum'
ALGORITHM = 'algorithm'
CHECKSUM_VALUE = 'checksumValue'
# Package
PACKAGES = 'packages'
PACKAGE_DOWNLOAD_LOCATION = 'downloadLocation'
PACKAGE_VERSION = 'versionInfo'
PACKAGE_SUPPLIER = 'supplier'
FILES_ANALYZED = 'filesAnalyzed'
PACKAGE_VERIFICATION_CODE = 'packageVerificationCode'
PACKAGE_VERIFICATION_CODE_VALUE = 'packageVerificationCodeValue'
PACKAGE_EXTERNAL_REFS = 'externalRefs'
PACKAGE_EXTERNAL_REF_CATEGORY = 'referenceCategory'
PACKAGE_EXTERNAL_REF_TYPE = 'referenceType'
PACKAGE_EXTERNAL_REF_LOCATOR = 'referenceLocator'
PACKAGE_HAS_FILES = 'hasFiles'
# File
FILES = 'files'
FILE_NAME = 'fileName'
FILE_CHECKSUMS = 'checksums'
# Relationship
RELATIONSHIPS = 'relationships'
REL_ELEMENT_ID = 'spdxElementId'
REL_RELATED_ELEMENT_ID = 'relatedSpdxElement'
REL_TYPE = 'relationshipType'
class JSONWriter:
@staticmethod
def marshal_doc_headers(sbom_doc):
headers = {
PropNames.SPDX_VERSION: SPDX_VER,
PropNames.DATA_LICENSE: DATA_LIC,
PropNames.SPDXID: sbom_doc.id,
PropNames.NAME: sbom_doc.name,
PropNames.DOCUMENT_NAMESPACE: sbom_doc.namespace,
PropNames.CREATION_INFO: {}
}
creators = [creator for creator in sbom_doc.creators]
headers[PropNames.CREATION_INFO][PropNames.CREATORS] = creators
headers[PropNames.CREATION_INFO][PropNames.CREATED] = sbom_doc.created
external_refs = []
for doc_ref in sbom_doc.external_refs:
checksum = doc_ref.checksum.split(': ')
external_refs.append({
PropNames.EXTERNAL_DOCUMENT_ID: f'{doc_ref.id}',
PropNames.EXTERNAL_DOCUMENT_URI: doc_ref.uri,
PropNames.EXTERNAL_DOCUMENT_CHECKSUM: {
PropNames.ALGORITHM: checksum[0],
PropNames.CHECKSUM_VALUE: checksum[1]
}
})
if external_refs:
headers[PropNames.EXTERNAL_DOCUMENT_REF] = external_refs
headers[PropNames.DOCUMENT_DESCRIBES] = [sbom_doc.describes]
return headers
@staticmethod
def marshal_packages(sbom_doc):
packages = []
for p in sbom_doc.packages:
package = {
PropNames.NAME: p.name,
PropNames.SPDXID: p.id,
PropNames.PACKAGE_DOWNLOAD_LOCATION: p.download_location if p.download_location else 'NONE',
PropNames.FILES_ANALYZED: p.files_analyzed
}
if p.version:
package[PropNames.PACKAGE_VERSION] = p.version
if p.supplier:
package[PropNames.PACKAGE_SUPPLIER] = p.supplier
if p.verification_code:
package[PropNames.PACKAGE_VERIFICATION_CODE] = {
PropNames.PACKAGE_VERIFICATION_CODE_VALUE: p.verification_code
}
if p.external_refs:
package[PropNames.PACKAGE_EXTERNAL_REFS] = []
for ref in p.external_refs:
ext_ref = {
PropNames.PACKAGE_EXTERNAL_REF_CATEGORY: ref.category,
PropNames.PACKAGE_EXTERNAL_REF_TYPE: ref.type,
PropNames.PACKAGE_EXTERNAL_REF_LOCATOR: ref.locator,
}
package[PropNames.PACKAGE_EXTERNAL_REFS].append(ext_ref)
if p.file_ids:
package[PropNames.PACKAGE_HAS_FILES] = []
for file_id in p.file_ids:
package[PropNames.PACKAGE_HAS_FILES].append(file_id)
packages.append(package)
return {PropNames.PACKAGES: packages}
@staticmethod
def marshal_files(sbom_doc):
files = []
for f in sbom_doc.files:
file = {
PropNames.FILE_NAME: f.name,
PropNames.SPDXID: f.id
}
checksum = f.checksum.split(': ')
file[PropNames.FILE_CHECKSUMS] = [{
PropNames.ALGORITHM: checksum[0],
PropNames.CHECKSUM_VALUE: checksum[1],
}]
files.append(file)
return {PropNames.FILES: files}
@staticmethod
def marshal_relationships(sbom_doc):
relationships = []
sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.relationship + r.id2 + r.id1)
for r in sorted_rels:
rel = {
PropNames.REL_ELEMENT_ID: r.id1,
PropNames.REL_RELATED_ELEMENT_ID: r.id2,
PropNames.REL_TYPE: r.relationship,
}
relationships.append(rel)
return {PropNames.RELATIONSHIPS: relationships}
@staticmethod
def write(sbom_doc, file):
doc = {}
doc.update(JSONWriter.marshal_doc_headers(sbom_doc))
doc.update(JSONWriter.marshal_packages(sbom_doc))
doc.update(JSONWriter.marshal_files(sbom_doc))
doc.update(JSONWriter.marshal_relationships(sbom_doc))
file.write(json.dumps(doc, indent=4))

View file

@ -0,0 +1,153 @@
#!/usr/bin/env python3
#
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import pathlib
import unittest
import sbom_data
import sbom_writers
BUILD_FINGER_PRINT = 'build_finger_print'
SUPPLIER_GOOGLE = 'Organization: Google'
SUPPLIER_UPSTREAM = 'Organization: upstream'
SPDXID_PREBUILT_PACKAGE1 = 'SPDXRef-PREBUILT-package1'
SPDXID_SOURCE_PACKAGE1 = 'SPDXRef-SOURCE-package1'
SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1'
SPDXID_FILE1 = 'SPDXRef-file1'
SPDXID_FILE2 = 'SPDXRef-file2'
SPDXID_FILE3 = 'SPDXRef-file3'
class SBOMWritersTest(unittest.TestCase):
def setUp(self):
# SBOM of a product
self.sbom_doc = sbom_data.Document(name='test doc',
namespace='http://www.google.com/sbom/spdx/android',
creators=[SUPPLIER_GOOGLE],
created='2023-03-31T22:17:58Z',
describes=sbom_data.SPDXID_PRODUCT)
self.sbom_doc.add_external_ref(
sbom_data.DocumentExternalReference(id='DocumentRef-external_doc_ref',
uri='external_doc_uri',
checksum='SHA1: 1234567890'))
self.sbom_doc.add_package(
sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,
name=sbom_data.PACKAGE_NAME_PRODUCT,
supplier=SUPPLIER_GOOGLE,
version=BUILD_FINGER_PRINT,
files_analyzed=True,
verification_code='123456',
file_ids=[SPDXID_FILE1, SPDXID_FILE2, SPDXID_FILE3]))
self.sbom_doc.add_package(
sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
name=sbom_data.PACKAGE_NAME_PLATFORM,
supplier=SUPPLIER_GOOGLE,
version=BUILD_FINGER_PRINT,
))
self.sbom_doc.add_package(
sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE1,
name='Prebuilt package1',
supplier=SUPPLIER_GOOGLE,
version=BUILD_FINGER_PRINT,
))
self.sbom_doc.add_package(
sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1,
name='Source package1',
supplier=SUPPLIER_GOOGLE,
version=BUILD_FINGER_PRINT,
external_refs=[sbom_data.PackageExternalRef(
category=sbom_data.PackageExternalRefCategory.SECURITY,
type=sbom_data.PackageExternalRefType.cpe22Type,
locator='cpe:/a:jsoncpp_project:jsoncpp:1.9.4')]
))
self.sbom_doc.add_package(
sbom_data.Package(id=SPDXID_UPSTREAM_PACKAGE1,
name='Upstream package1',
supplier=SUPPLIER_UPSTREAM,
version='1.1',
))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_SOURCE_PACKAGE1,
relationship=sbom_data.RelationshipType.VARIANT_OF,
id2=SPDXID_UPSTREAM_PACKAGE1))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', checksum='SHA1: 11111'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE2,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_PREBUILT_PACKAGE1))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE3,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_SOURCE_PACKAGE1
))
# SBOM fragment of a APK
self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
namespace='http://www.google.com/sbom/spdx/android',
creators=[SUPPLIER_GOOGLE],
created='2023-03-31T22:17:58Z',
describes=SPDXID_FILE1)
self.unbundled_sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE1, name='/bin/file1.apk', checksum='SHA1: 11111'))
self.unbundled_sbom_doc.add_package(
sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1,
name='Unbundled apk package',
supplier=SUPPLIER_GOOGLE,
version=BUILD_FINGER_PRINT))
self.unbundled_sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_SOURCE_PACKAGE1))
def test_tagvalue_writer(self):
with io.StringIO() as output:
sbom_writers.TagValueWriter.write(self.sbom_doc, output)
expected_output = pathlib.Path('testdata/expected_tagvalue_sbom.spdx').read_text()
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
def test_tagvalue_writer_unbundled(self):
with io.StringIO() as output:
sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True)
expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_unbundled.spdx').read_text()
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
def test_json_writer(self):
with io.StringIO() as output:
sbom_writers.JSONWriter.write(self.sbom_doc, output)
expected_output = pathlib.Path('testdata/expected_json_sbom.spdx.json').read_text()
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
if __name__ == '__main__':
unittest.main(verbosity=2)

View file

@ -0,0 +1,137 @@
{
"spdxVersion": "SPDX-2.3",
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": "test doc",
"documentNamespace": "http://www.google.com/sbom/spdx/android",
"creationInfo": {
"creators": [
"Organization: Google"
],
"created": "2023-03-31T22:17:58Z"
},
"externalDocumentRefs": [
{
"externalDocumentId": "DocumentRef-external_doc_ref",
"spdxDocument": "external_doc_uri",
"checksum": {
"algorithm": "SHA1",
"checksumValue": "1234567890"
}
}
],
"documentDescribes": [
"SPDXRef-PRODUCT"
],
"packages": [
{
"name": "PRODUCT",
"SPDXID": "SPDXRef-PRODUCT",
"downloadLocation": "NONE",
"filesAnalyzed": true,
"versionInfo": "build_finger_print",
"supplier": "Organization: Google",
"packageVerificationCode": {
"packageVerificationCodeValue": "123456"
},
"hasFiles": [
"SPDXRef-file1",
"SPDXRef-file2",
"SPDXRef-file3"
]
},
{
"name": "PLATFORM",
"SPDXID": "SPDXRef-PLATFORM",
"downloadLocation": "NONE",
"filesAnalyzed": false,
"versionInfo": "build_finger_print",
"supplier": "Organization: Google"
},
{
"name": "Prebuilt package1",
"SPDXID": "SPDXRef-PREBUILT-package1",
"downloadLocation": "NONE",
"filesAnalyzed": false,
"versionInfo": "build_finger_print",
"supplier": "Organization: Google"
},
{
"name": "Source package1",
"SPDXID": "SPDXRef-SOURCE-package1",
"downloadLocation": "NONE",
"filesAnalyzed": false,
"versionInfo": "build_finger_print",
"supplier": "Organization: Google",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceType": "cpe22Type",
"referenceLocator": "cpe:/a:jsoncpp_project:jsoncpp:1.9.4"
}
]
},
{
"name": "Upstream package1",
"SPDXID": "SPDXRef-UPSTREAM-package1",
"downloadLocation": "NONE",
"filesAnalyzed": false,
"versionInfo": "1.1",
"supplier": "Organization: upstream"
}
],
"files": [
{
"fileName": "/bin/file1",
"SPDXID": "SPDXRef-file1",
"checksums": [
{
"algorithm": "SHA1",
"checksumValue": "11111"
}
]
},
{
"fileName": "/bin/file2",
"SPDXID": "SPDXRef-file2",
"checksums": [
{
"algorithm": "SHA1",
"checksumValue": "22222"
}
]
},
{
"fileName": "/bin/file3",
"SPDXID": "SPDXRef-file3",
"checksums": [
{
"algorithm": "SHA1",
"checksumValue": "33333"
}
]
}
],
"relationships": [
{
"spdxElementId": "SPDXRef-file1",
"relatedSpdxElement": "SPDXRef-PLATFORM",
"relationshipType": "GENERATED_FROM"
},
{
"spdxElementId": "SPDXRef-file2",
"relatedSpdxElement": "SPDXRef-PREBUILT-package1",
"relationshipType": "GENERATED_FROM"
},
{
"spdxElementId": "SPDXRef-file3",
"relatedSpdxElement": "SPDXRef-SOURCE-package1",
"relationshipType": "GENERATED_FROM"
},
{
"spdxElementId": "SPDXRef-SOURCE-package1",
"relatedSpdxElement": "SPDXRef-UPSTREAM-package1",
"relationshipType": "VARIANT_OF"
}
]
}

View file

@ -0,0 +1,65 @@
SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: test doc
DocumentNamespace: http://www.google.com/sbom/spdx/android
Creator: Organization: Google
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
FilesAnalyzed: true
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageVerificationCode: 123456
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-PRODUCT
FileName: /bin/file1
SPDXID: SPDXRef-file1
FileChecksum: SHA1: 11111
FileName: /bin/file2
SPDXID: SPDXRef-file2
FileChecksum: SHA1: 22222
FileName: /bin/file3
SPDXID: SPDXRef-file3
FileChecksum: SHA1: 33333
PackageName: PLATFORM
SPDXID: SPDXRef-PLATFORM
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageName: Prebuilt package1
SPDXID: SPDXRef-PREBUILT-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageName: Source package1
SPDXID: SPDXRef-SOURCE-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4
PackageName: Upstream package1
SPDXID: SPDXRef-UPSTREAM-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: 1.1
PackageSupplier: Organization: upstream
Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1

View file

@ -0,0 +1,12 @@
FileName: /bin/file1.apk
SPDXID: SPDXRef-file1
FileChecksum: SHA1: 11111
PackageName: Unbundled apk package
SPDXID: SPDXRef-SOURCE-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-SOURCE-package1