diff --git a/android/notices.go b/android/notices.go index 2a4c17cd8..194a734d3 100644 --- a/android/notices.go +++ b/android/notices.go @@ -15,9 +15,93 @@ package android import ( + "path/filepath" "strings" + + "github.com/google/blueprint" ) +func init() { + pctx.SourcePathVariable("merge_notices", "build/soong/scripts/mergenotice.py") + pctx.SourcePathVariable("generate_notice", "build/soong/scripts/generate-notice-files.py") + + pctx.HostBinToolVariable("minigzip", "minigzip") +} + +type NoticeOutputs struct { + Merged OptionalPath + TxtOutput OptionalPath + HtmlOutput OptionalPath + HtmlGzOutput OptionalPath +} + +var ( + mergeNoticesRule = pctx.AndroidStaticRule("mergeNoticesRule", blueprint.RuleParams{ + Command: `${merge_notices} --output $out $in`, + CommandDeps: []string{"${merge_notices}"}, + Description: "merge notice files into $out", + }) + + generateNoticeRule = pctx.AndroidStaticRule("generateNoticeRule", blueprint.RuleParams{ + Command: `rm -rf $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` + + `mkdir -p $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` + + `${generate_notice} --text-output $txtOut --html-output $htmlOut -t "$title" -s $inputDir && ` + + `${minigzip} -c $htmlOut > $out`, + CommandDeps: []string{"${generate_notice}", "${minigzip}"}, + Description: "produce notice file $out", + }, "txtOut", "htmlOut", "title", "inputDir") +) + +func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Path) { + ctx.Build(pctx, BuildParams{ + Rule: mergeNoticesRule, + Description: "merge notices", + Inputs: noticePaths, + Output: mergedNotice, + }) +} + +func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilename string, + noticePaths []Path) NoticeOutputs { + // Merge all NOTICE files into one. + // TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass. + // + // generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules + // about input NOTICE file paths. + // 1. Their relative paths to the src root become their NOTICE index titles. We want to use + // on-device paths as titles, and so output the merged NOTICE file the corresponding location. + // 2. They must end with .txt extension. Otherwise, they're ignored. + noticeRelPath := InstallPathToOnDevicePath(ctx, installPath.Join(ctx, installFilename+".txt")) + mergedNotice := PathForModuleOut(ctx, filepath.Join("NOTICE_FILES/src", noticeRelPath)) + MergeNotices(ctx, mergedNotice, noticePaths) + + // Transform the merged NOTICE file into a gzipped HTML file. + txtOuptut := PathForModuleOut(ctx, "NOTICE_txt", "NOTICE.txt") + htmlOutput := PathForModuleOut(ctx, "NOTICE_html", "NOTICE.html") + htmlGzOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") + title := "Notices for " + ctx.ModuleName() + ctx.Build(pctx, BuildParams{ + Rule: generateNoticeRule, + Description: "generate notice output", + Input: mergedNotice, + Output: htmlGzOutput, + ImplicitOutputs: WritablePaths{txtOuptut, htmlOutput}, + Args: map[string]string{ + "txtOut": txtOuptut.String(), + "htmlOut": htmlOutput.String(), + "title": title, + "inputDir": PathForModuleOut(ctx, "NOTICE_FILES/src").String(), + }, + }) + + return NoticeOutputs{ + Merged: OptionalPathForPath(mergedNotice), + TxtOutput: OptionalPathForPath(txtOuptut), + HtmlOutput: OptionalPathForPath(htmlOutput), + HtmlGzOutput: OptionalPathForPath(htmlGzOutput), + } +} + // BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's // generated license metadata file. func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { @@ -28,18 +112,5 @@ func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile Writ FlagWithOutput("-o ", outputFile). FlagWithDepFile("-d ", depsFile). Input(ctx.Module().base().licenseMetadataFile) - rule.Build("text_notice", "container notice file") -} - -// BuildNoticeHtmlOutputFromLicenseMetadata writes out a notice text file based on the module's -// generated license metadata file. -func BuildNoticeHtmlOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { - depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", ".")) - rule := NewRuleBuilder(pctx, ctx) - rule.Command(). - BuiltTool("htmlnotice"). - FlagWithOutput("-o ", outputFile). - FlagWithDepFile("-d ", depsFile). - Input(ctx.Module().base().licenseMetadataFile) - rule.Build("html_notice", "container notice file") + rule.Build("container_notice", "container notice file") } diff --git a/apex/androidmk.go b/apex/androidmk.go index e094a1276..059b4d76c 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -396,6 +396,10 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { } a.writeRequiredModules(w, moduleNames) + if a.mergedNotices.Merged.Valid() { + fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", a.mergedNotices.Merged.Path().String()) + } + fmt.Fprintln(w, "include $(BUILD_PREBUILT)") if apexType == imageApex { diff --git a/apex/apex.go b/apex/apex.go index 014918f69..5e8ccf5c0 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -414,8 +414,8 @@ type apexBundle struct { // Processed file_contexts files fileContexts android.WritablePath - // Path to notice file in html.gz format. - htmlGzNotice android.WritablePath + // Struct holding the merged notice file paths in different formats + mergedNotices android.NoticeOutputs // The built APEX file. This is the main product. // Could be .apex or .capex @@ -487,10 +487,11 @@ const ( // for each of the files in case when the APEX is flattened. type apexFile struct { // buildFile is put in the installDir inside the APEX. - builtFile android.Path - installDir string - customStem string - symlinks []string // additional symlinks + builtFile android.Path + noticeFiles android.Paths + installDir string + customStem string + symlinks []string // additional symlinks // Info for Android.mk Module name of `module` in AndroidMk. Note the generated AndroidMk // module for apexFile is named something like .[ 0 { + for _, path := range paths { + noticePathSet[path] = true + } + } + return true + }) + + // If the app has one, add it too. + if len(a.NoticeFiles()) > 0 { + for _, path := range a.NoticeFiles() { + noticePathSet[path] = true + } + } + + if len(noticePathSet) == 0 { + return + } + var noticePaths []android.Path + for path := range noticePathSet { + noticePaths = append(noticePaths, path) + } + sort.Slice(noticePaths, func(i, j int) bool { + return noticePaths[i].String() < noticePaths[j].String() + }) + + a.noticeOutputs = android.BuildNoticeOutput(ctx, a.installDir, a.installApkName+".apk", noticePaths) +} + // Reads and prepends a main cert from the default cert dir if it hasn't been set already, i.e. it // isn't a cert module reference. Also checks and enforces system cert restriction if applicable. func processMainCert(m android.ModuleBase, certPropValue string, certificates []Certificate, ctx android.ModuleContext) []Certificate { @@ -589,10 +639,9 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { } a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir) - noticeFile := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") - android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, noticeFile) + a.noticeBuildActions(ctx) if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { - a.aapt.noticeFile = android.OptionalPathForPath(noticeFile) + a.aapt.noticeFile = a.noticeOutputs.HtmlGzOutput } a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) diff --git a/java/app_test.go b/java/app_test.go index 48eeedeca..73cf09227 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -27,6 +27,7 @@ import ( "android/soong/android" "android/soong/cc" "android/soong/dexpreopt" + "android/soong/genrule" ) // testApp runs tests using the prepareForJavaTest @@ -2782,6 +2783,116 @@ func TestCodelessApp(t *testing.T) { } } +func TestEmbedNotice(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + cc.PrepareForTestWithCcDefaultModules, + genrule.PrepareForTestWithGenRuleBuildComponents, + android.MockFS{ + "APP_NOTICE": nil, + "GENRULE_NOTICE": nil, + "LIB_NOTICE": nil, + "TOOL_NOTICE": nil, + }.AddToFixture(), + ).RunTestWithBp(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + static_libs: ["javalib"], + jni_libs: ["libjni"], + notice: "APP_NOTICE", + embed_notices: true, + sdk_version: "current", + } + + // No embed_notice flag + android_app { + name: "bar", + srcs: ["a.java"], + jni_libs: ["libjni"], + notice: "APP_NOTICE", + sdk_version: "current", + } + + // No NOTICE files + android_app { + name: "baz", + srcs: ["a.java"], + embed_notices: true, + sdk_version: "current", + } + + cc_library { + name: "libjni", + system_shared_libs: [], + stl: "none", + notice: "LIB_NOTICE", + sdk_version: "current", + } + + java_library { + name: "javalib", + srcs: [ + ":gen", + ], + sdk_version: "current", + } + + genrule { + name: "gen", + tools: ["gentool"], + out: ["gen.java"], + notice: "GENRULE_NOTICE", + } + + java_binary_host { + name: "gentool", + srcs: ["b.java"], + notice: "TOOL_NOTICE", + } + `) + + // foo has NOTICE files to process, and embed_notices is true. + foo := result.ModuleForTests("foo", "android_common") + // verify merge notices rule. + mergeNotices := foo.Rule("mergeNoticesRule") + noticeInputs := mergeNotices.Inputs.Strings() + // TOOL_NOTICE should be excluded as it's a host module. + if len(mergeNotices.Inputs) != 3 { + t.Errorf("number of input notice files: expected = 3, actual = %q", noticeInputs) + } + if !inList("APP_NOTICE", noticeInputs) { + t.Errorf("APP_NOTICE is missing from notice files, %q", noticeInputs) + } + if !inList("LIB_NOTICE", noticeInputs) { + t.Errorf("LIB_NOTICE is missing from notice files, %q", noticeInputs) + } + if !inList("GENRULE_NOTICE", noticeInputs) { + t.Errorf("GENRULE_NOTICE is missing from notice files, %q", noticeInputs) + } + // aapt2 flags should include -A so that its contents are put in the APK's /assets. + res := foo.Output("package-res.apk") + aapt2Flags := res.Args["flags"] + e := "-A out/soong/.intermediates/foo/android_common/NOTICE" + android.AssertStringDoesContain(t, "expected.apkPath", aapt2Flags, e) + + // bar has NOTICE files to process, but embed_notices is not set. + bar := result.ModuleForTests("bar", "android_common") + res = bar.Output("package-res.apk") + aapt2Flags = res.Args["flags"] + e = "-A out/soong/.intermediates/bar/android_common/NOTICE" + android.AssertStringDoesNotContain(t, "bar shouldn't have the asset dir flag for NOTICE", aapt2Flags, e) + + // baz's embed_notice is true, but it doesn't have any NOTICE files. + baz := result.ModuleForTests("baz", "android_common") + res = baz.Output("package-res.apk") + aapt2Flags = res.Args["flags"] + e = "-A out/soong/.intermediates/baz/android_common/NOTICE" + if strings.Contains(aapt2Flags, e) { + t.Errorf("baz shouldn't have the asset dir flag for NOTICE: %q", e) + } +} + func TestUncompressDex(t *testing.T) { testCases := []struct { name string diff --git a/scripts/generate-notice-files.py b/scripts/generate-notice-files.py new file mode 100755 index 000000000..1b4acfaaf --- /dev/null +++ b/scripts/generate-notice-files.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2012 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: generate-notice-files --text-output [plain text output file] \ + --html-output [html output file] \ + --xml-output [xml output file] \ + -t [file title] -s [directory of notices] + +Generate the Android notice files, including both text and html files. + +-h to display this usage message and exit. +""" +from collections import defaultdict +import argparse +import hashlib +import itertools +import os +import os.path +import re +import struct +import sys + +MD5_BLOCKSIZE = 1024 * 1024 +HTML_ESCAPE_TABLE = { + b"&": b"&", + b'"': b""", + b"'": b"'", + b">": b">", + b"<": b"<", + } + +def md5sum(filename): + """Calculate an MD5 of the file given by FILENAME, + and return hex digest as a string. + Output should be compatible with md5sum command""" + + f = open(filename, "rb") + sum = hashlib.md5() + while 1: + block = f.read(MD5_BLOCKSIZE) + if not block: + break + sum.update(block) + f.close() + return sum.hexdigest() + + +def html_escape(text): + """Produce entities within text.""" + # Using for i in text doesn't work since i will be an int, not a byte. + # There are multiple ways to solve this, but the most performant way + # to iterate over a byte array is to use unpack. Using the + # for i in range(len(text)) and using that to get a byte using array + # slices is twice as slow as this method. + return b"".join(HTML_ESCAPE_TABLE.get(i,i) for i in struct.unpack(str(len(text)) + 'c', text)) + +HTML_OUTPUT_CSS=b""" + + +""" + +def combine_notice_files_html(file_hash, input_dir, output_filename): + """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + + # Set up a filename to row id table (anchors inside tables don't work in + # most browsers, but href's to table row ids do) + id_table = {} + id_count = 0 + for value in file_hash: + for filename in value: + id_table[filename] = id_count + id_count += 1 + + # Open the output file, and output the header pieces + output_file = open(output_filename, "wb") + + output_file.write(b"\n") + output_file.write(HTML_OUTPUT_CSS) + output_file.write(b'\n') + + # Output our table of contents + output_file.write(b'
\n') + output_file.write(b"
    \n") + + # Flatten the list of lists into a single list of filenames + sorted_filenames = sorted(itertools.chain.from_iterable(file_hash)) + + # Print out a nice table of contents + for filename in sorted_filenames: + stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) + output_file.write(('
  • %s
    • \n' % (id_table.get(filename), stripped_filename)).encode()) + + output_file.write(b"
\n") + output_file.write(b"
\n") + # Output the individual notice file lists + output_file.write(b'\n') + for value in file_hash: + output_file.write(('\n\n\n\n") + + # Finish off the file output + output_file.write(b"
\n' % id_table.get(value[0])).encode()) + output_file.write(b'
Notices for file(s):
\n') + output_file.write(b'
\n') + for filename in value: + output_file.write(("%s
\n" % (SRC_DIR_STRIP_RE.sub(r"\1", filename))).encode()) + output_file.write(b"
\n\n") + output_file.write(b'
\n')
+        with open(value[0], "rb") as notice_file:
+            output_file.write(html_escape(notice_file.read()))
+        output_file.write(b"\n
\n") + output_file.write(b"
\n") + output_file.write(b"\n") + output_file.close() + +def combine_notice_files_text(file_hash, input_dir, output_filename, file_title): + """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + output_file = open(output_filename, "wb") + output_file.write(file_title.encode()) + output_file.write(b"\n") + for value in file_hash: + output_file.write(b"============================================================\n") + output_file.write(b"Notices for file(s):\n") + for filename in value: + output_file.write(SRC_DIR_STRIP_RE.sub(r"\1", filename).encode()) + output_file.write(b"\n") + output_file.write(b"------------------------------------------------------------\n") + with open(value[0], "rb") as notice_file: + output_file.write(notice_file.read()) + output_file.write(b"\n") + output_file.close() + +def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename): + """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME.""" + + SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") + + # Set up a filename to row id table (anchors inside tables don't work in + # most browsers, but href's to table row ids do) + id_table = {} + for file_key, files in files_with_same_hash.items(): + for filename in files: + id_table[filename] = file_key + + # Open the output file, and output the header pieces + output_file = open(output_filename, "wb") + + output_file.write(b'\n') + output_file.write(b"\n") + + # Flatten the list of lists into a single list of filenames + sorted_filenames = sorted(list(id_table)) + + # Print out a nice table of contents + for filename in sorted_filenames: + stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) + output_file.write(('%s\n' % (id_table.get(filename), stripped_filename)).encode()) + output_file.write(b"\n\n") + + processed_file_keys = [] + # Output the individual notice file lists + for filename in sorted_filenames: + file_key = id_table.get(filename) + if file_key in processed_file_keys: + continue + processed_file_keys.append(file_key) + + output_file.write(('\n\n") + + # Finish off the file output + output_file.write(b"\n") + output_file.close() + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--text-output', required=True, + help='The text output file path.') + parser.add_argument( + '--html-output', + help='The html output file path.') + parser.add_argument( + '--xml-output', + help='The xml output file path.') + parser.add_argument( + '-t', '--title', required=True, + help='The file title.') + parser.add_argument( + '-s', '--source-dir', required=True, + help='The directory containing notices.') + parser.add_argument( + '-i', '--included-subdirs', action='append', + help='The sub directories which should be included.') + parser.add_argument( + '-e', '--excluded-subdirs', action='append', + help='The sub directories which should be excluded.') + return parser.parse_args() + +def main(argv): + args = get_args() + + txt_output_file = args.text_output + html_output_file = args.html_output + xml_output_file = args.xml_output + file_title = args.title + included_subdirs = [] + excluded_subdirs = [] + if args.included_subdirs is not None: + included_subdirs = args.included_subdirs + if args.excluded_subdirs is not None: + excluded_subdirs = args.excluded_subdirs + + # Find all the notice files and md5 them + input_dir = os.path.normpath(args.source_dir) + files_with_same_hash = defaultdict(list) + for root, dir, files in os.walk(input_dir): + for file in files: + matched = True + if len(included_subdirs) > 0: + matched = False + for subdir in included_subdirs: + if (root == (input_dir + '/' + subdir) or + root.startswith(input_dir + '/' + subdir + '/')): + matched = True + break + elif len(excluded_subdirs) > 0: + for subdir in excluded_subdirs: + if (root == (input_dir + '/' + subdir) or + root.startswith(input_dir + '/' + subdir + '/')): + matched = False + break + if matched and file.endswith(".txt"): + filename = os.path.join(root, file) + file_md5sum = md5sum(filename) + files_with_same_hash[file_md5sum].append(filename) + + filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(list(files_with_same_hash))] + + combine_notice_files_text(filesets, input_dir, txt_output_file, file_title) + + if html_output_file is not None: + combine_notice_files_html(filesets, input_dir, html_output_file) + + if xml_output_file is not None: + combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/mergenotice.py b/scripts/mergenotice.py new file mode 100755 index 000000000..fe990735b --- /dev/null +++ b/scripts/mergenotice.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Merges input notice files to the output file while ignoring duplicated files +This script shouldn't be confused with build/soong/scripts/generate-notice-files.py +which is responsible for creating the final notice file for all artifacts +installed. This script has rather limited scope; it is meant to create a merged +notice file for a set of modules that are packaged together, e.g. in an APEX. +The merged notice file does not reveal the individual files in the package. +""" + +import sys +import argparse + +def get_args(): + parser = argparse.ArgumentParser(description='Merge notice files.') + parser.add_argument('--output', help='output file path.') + parser.add_argument('inputs', metavar='INPUT', nargs='+', + help='input notice file') + return parser.parse_args() + +def main(argv): + args = get_args() + + processed = set() + with open(args.output, 'w+') as output: + for input in args.inputs: + with open(input, 'r') as f: + data = f.read().strip() + if data not in processed: + processed.add(data) + output.write('%s\n\n' % data) + +if __name__ == '__main__': + main(sys.argv)