Include static libraries information in Android SBOM. am: ccf5023a1e

Original change: https://googleplex-android-review.googlesource.com/c/platform/build/+/24029569

Change-Id: I25b0c963047450dd61b22457fc8739b54fe96211
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Wei Li 2023-07-13 00:35:51 +00:00 committed by Automerger Merge Worker
commit 7e3e842f07
9 changed files with 216 additions and 65 deletions

View file

@ -2157,13 +2157,19 @@ endif # TARGET_BUILD_APPS
# is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile.
# is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile.
# is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile.
# build_output_path: the path of the built file, used to calculate checksum
# static_libraries/whole_static_libraries: list of module name of the static libraries the file links against, e.g. libclang_rt.builtins or libclang_rt.builtins_32
# Info of all static libraries of all installed files are collected in variable _all_static_libs that is used to list all the static library files in sbom-metadata.csv.
# See the second foreach loop in the rule of sbom-metadata.csv for the detailed info of static libraries collected in _all_static_libs.
# is_static_lib: whether the file is a static library
# (TODO: b/272358583 find another way of always rebuilding this target)
# Remove the sbom-metadata.csv whenever makefile is evaluated
$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1)
$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)
rm -f $@
@echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@
echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $@
$(eval _all_static_libs :=)
$(foreach f,$(installed_files),\
$(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \
$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \
@ -2185,11 +2191,25 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)
$(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \
$(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \
$(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \
@echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \
$(eval _static_libs := $(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES)) \
$(eval _whole_static_libs := $(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES)) \
$(foreach l,$(_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \
$(foreach l,$(_whole_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \
echo /$(_path_on_device),$(_module_path),$(_soong_module_type),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_build_output_path),$(_static_libs),$(_whole_static_libs), >> $@; \
$(if $(_post_installed_dexpreopt_zip), \
for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \
for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i$(comma)$(_static_libs)$(comma)$(_whole_static_libs)$(comma) >> $@ ; done ; \
) \
)
$(foreach l,$(sort $(_all_static_libs)), \
$(eval _lib_stem := $(call word-colon,1,$l)) \
$(eval _module_path := $(call word-colon,2,$l)) \
$(eval _soong_module_type := $(call word-colon,3,$l)) \
$(eval _built_file := $(call word-colon,4,$l)) \
$(eval _static_libs := $(ALL_STATIC_LIBRARIES.$l.STATIC_LIBRARIES)) \
$(eval _whole_static_libs := $(ALL_STATIC_LIBRARIES.$l.WHOLE_STATIC_LIBRARIES)) \
$(eval _is_static_lib := Y) \
echo $(_lib_stem).a,$(_module_path),$(_soong_module_type),,,,,$(_built_file),$(_static_libs),$(_whole_static_libs),$(_is_static_lib) >> $@; \
)
.PHONY: sbom
ifeq ($(TARGET_BUILD_APPS),)

View file

@ -3,9 +3,20 @@
# unless a .mk file changes its installed file after including base_rules.mk.
ifdef my_register_name
# ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle libclang_rt.builtins, for primary arch
# ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle_32 libclang_rt.builtins_32, for 2nd arch.
ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED)))
$(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\
$(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\
$(eval ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\
$(eval ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\
)
endif
ifeq (STATIC_LIBRARIES,$(LOCAL_MODULE_CLASS))
ALL_STATIC_LIBRARIES.$(my_register_name).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))
ALL_STATIC_LIBRARIES.$(my_register_name).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))
ifdef LOCAL_SOONG_MODULE_TYPE
ALL_STATIC_LIBRARIES.$(my_register_name).BUILT_FILE := $(LOCAL_PREBUILT_MODULE_FILE)
endif
endif
endif

View file

@ -332,14 +332,6 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path):
return external_doc_ref, packages, relationships
def generate_package_verification_code(files):
checksums = [file.checksum for file in files]
checksums.sort()
h = hashlib.sha1()
h.update(''.join(checksums).encode(encoding='utf-8'))
return h.hexdigest()
def save_report(report_file_path, report):
with open(report_file_path, 'w', encoding='utf-8') as report_file:
for type, issues in report.items():
@ -487,16 +479,28 @@ def main():
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
build_output_path = installed_file_metadata['build_output_path']
is_static_lib = installed_file_metadata['is_static_lib']
if not installed_file_has_metadata(installed_file_metadata, report):
continue
if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
# Ignore non-existing static library files for now since they are not shipped on devices.
report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
continue
file_id = new_file_id(installed_file)
doc.files.append(
sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))
# TODO(b/285453664): Soong should report the information of statically linked libraries to Make.
# This happens when a different sanitized version of static libraries is used in linking.
# As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be
# located correctly because Soong doesn't report the information to Make.
sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string
if os.path.islink(build_output_path) or os.path.isfile(build_output_path):
sha1 = checksum(build_output_path)
doc.files.append(sbom_data.File(id=file_id,
name=installed_file,
checksum=sha1))
if not is_static_lib:
if not args.unbundled_apex:
product_package.file_ids.append(file_id)
elif len(doc.files) > 1:
@ -544,13 +548,21 @@ def main():
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
if not args.unbundled_apex:
product_package.verification_code = generate_package_verification_code(doc.files)
# Process static libraries and whole static libraries the installed file links to
static_libs = installed_file_metadata['static_libraries']
whole_static_libs = installed_file_metadata['whole_static_libraries']
all_static_libs = (static_libs + ' ' + whole_static_libs).strip()
if all_static_libs:
for lib in all_static_libs.split(' '):
doc.add_relationship(sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.STATIC_LINK,
id2=new_file_id(lib + '.a')))
if args.unbundled_apex:
doc.describes = doc.files[0].id
# Save SBOM records to output file
doc.generate_packages_verification_code()
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
prefix = args.output_file
if prefix.endswith('.spdx'):

View file

@ -25,6 +25,7 @@ fields in each data class.
from dataclasses import dataclass, field
from typing import List
import hashlib
SPDXID_DOC = 'SPDXRef-DOCUMENT'
SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
@ -81,6 +82,7 @@ class RelationshipType:
VARIANT_OF = 'VARIANT_OF'
GENERATED_FROM = 'GENERATED_FROM'
CONTAINS = 'CONTAINS'
STATIC_LINK = 'STATIC_LINK'
@dataclass
@ -122,3 +124,17 @@ class Document:
if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
for r in self.relationships):
self.relationships.append(rel)
def generate_packages_verification_code(self):
for package in self.packages:
if not package.file_ids:
continue
checksums = []
for file in self.files:
if file.id in package.file_ids:
checksums.append(file.checksum)
checksums.sort()
h = hashlib.sha1()
h.update(''.join(checksums).encode(encoding='utf-8'))
package.verification_code = h.hexdigest()

View file

@ -85,7 +85,7 @@ class TagValueWriter:
return headers
@staticmethod
def marshal_package(package):
def marshal_package(sbom_doc, package, fragment):
download_location = sbom_data.VALUE_NOASSERTION
if package.download_location:
download_location = package.download_location
@ -107,50 +107,32 @@ class TagValueWriter:
f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')
tagvalues.append('')
return tagvalues
@staticmethod
def marshal_described_element(sbom_doc, fragment):
if not sbom_doc.describes:
return None
product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
if product_package:
tagvalues = TagValueWriter.marshal_package(product_package[0])
if not fragment:
if package.id == sbom_doc.describes and not fragment:
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
return tagvalues
file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
if file:
tagvalues = TagValueWriter.marshal_file(file[0])
if not fragment:
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
for file in sbom_doc.files:
if file.id in package.file_ids:
tagvalues += TagValueWriter.marshal_file(file)
return tagvalues
return None
@staticmethod
def marshal_packages(sbom_doc):
def marshal_packages(sbom_doc, fragment):
tagvalues = []
marshaled_relationships = []
i = 0
packages = sbom_doc.packages
while i < len(packages):
if packages[i].id == sbom_doc.describes:
i += 1
continue
if i + 1 < len(packages) \
and packages[i].id.startswith('SPDXRef-SOURCE-') \
and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
tagvalues += TagValueWriter.marshal_package(packages[i])
tagvalues += TagValueWriter.marshal_package(packages[i + 1])
if (i + 1 < len(packages)
and packages[i].id.startswith('SPDXRef-SOURCE-')
and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')):
# Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other
# in SBOMs in tagvalue format.
tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment)
rel = next((r for r in sbom_doc.relationships if
r.id1 == packages[i].id and
r.id2 == packages[i + 1].id and
@ -162,7 +144,7 @@ class TagValueWriter:
i += 2
else:
tagvalues += TagValueWriter.marshal_package(packages[i])
tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
i += 1
return tagvalues, marshaled_relationships
@ -179,12 +161,20 @@ class TagValueWriter:
return tagvalues
@staticmethod
def marshal_files(sbom_doc):
def marshal_files(sbom_doc, fragment):
tagvalues = []
files_in_packages = []
for package in sbom_doc.packages:
files_in_packages += package.file_ids
for file in sbom_doc.files:
if file.id == sbom_doc.describes:
if file.id in files_in_packages:
continue
tagvalues += TagValueWriter.marshal_file(file)
if file.id == sbom_doc.describes and not fragment:
# Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable.
tagvalues.append(
f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
return tagvalues
@staticmethod
@ -208,11 +198,8 @@ class TagValueWriter:
content = []
if not fragment:
content += TagValueWriter.marshal_doc_headers(sbom_doc)
described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)
if described_element:
content += described_element
content += TagValueWriter.marshal_files(sbom_doc)
tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
content += TagValueWriter.marshal_files(sbom_doc, fragment)
tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)
content += tagvalues
content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
file.write('\n'.join(content))

View file

@ -31,6 +31,7 @@ SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1'
SPDXID_FILE1 = 'SPDXRef-file1'
SPDXID_FILE2 = 'SPDXRef-file2'
SPDXID_FILE3 = 'SPDXRef-file3'
SPDXID_FILE4 = 'SPDXRef-file4'
class SBOMWritersTest(unittest.TestCase):
@ -101,6 +102,8 @@ class SBOMWritersTest(unittest.TestCase):
sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444'))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
@ -112,6 +115,10 @@ class SBOMWritersTest(unittest.TestCase):
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_SOURCE_PACKAGE1
))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.STATIC_LINK,
id2=SPDXID_FILE4
))
# SBOM fragment of a APK
self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
@ -139,6 +146,14 @@ class SBOMWritersTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
def test_tagvalue_writer_doc_describes_file(self):
with io.StringIO() as output:
self.sbom_doc.describes = SPDXID_FILE4
sbom_writers.TagValueWriter.write(self.sbom_doc, output)
expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_doc_describes_file.spdx').read_text()
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
def test_tagvalue_writer_unbundled(self):
with io.StringIO() as output:
sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True)

View file

@ -110,6 +110,16 @@
"checksumValue": "33333"
}
]
},
{
"fileName": "file4.a",
"SPDXID": "SPDXRef-file4",
"checksums": [
{
"algorithm": "SHA1",
"checksumValue": "44444"
}
]
}
],
"relationships": [
@ -128,6 +138,11 @@
"relatedSpdxElement": "SPDXRef-SOURCE-package1",
"relationshipType": "GENERATED_FROM"
},
{
"spdxElementId": "SPDXRef-file1",
"relatedSpdxElement": "SPDXRef-file4",
"relationshipType": "STATIC_LINK"
},
{
"spdxElementId": "SPDXRef-SOURCE-package1",
"relatedSpdxElement": "SPDXRef-UPSTREAM-package1",

View file

@ -7,6 +7,10 @@ Creator: Organization: Google
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
FileName: file4.a
SPDXID: SPDXRef-file4
FileChecksum: SHA1: 44444
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
@ -63,3 +67,4 @@ Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4

View file

@ -0,0 +1,70 @@
SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: test doc
DocumentNamespace: http://www.google.com/sbom/spdx/android
Creator: Organization: Google
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
FileName: file4.a
SPDXID: SPDXRef-file4
FileChecksum: SHA1: 44444
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-file4
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
FilesAnalyzed: true
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageVerificationCode: 123456
FileName: /bin/file1
SPDXID: SPDXRef-file1
FileChecksum: SHA1: 11111
FileName: /bin/file2
SPDXID: SPDXRef-file2
FileChecksum: SHA1: 22222
FileName: /bin/file3
SPDXID: SPDXRef-file3
FileChecksum: SHA1: 33333
PackageName: PLATFORM
SPDXID: SPDXRef-PLATFORM
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageName: Prebuilt package1
SPDXID: SPDXRef-PREBUILT-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
PackageName: Source package1
SPDXID: SPDXRef-SOURCE-package1
PackageDownloadLocation: NONE
FilesAnalyzed: false
PackageVersion: build_finger_print
PackageSupplier: Organization: Google
ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4
PackageName: Upstream package1
SPDXID: SPDXRef-UPSTREAM-package1
PackageDownloadLocation: NOASSERTION
FilesAnalyzed: false
PackageVersion: 1.1
PackageSupplier: Organization: upstream
Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4