Build notice files from license metadata. am: 43c2dcaef6 am: 6f7de5b1a9
Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2047886 Change-Id: I09993978f902d3e4b569cd676de0a56d5638ff2e Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Bob Badour
Automerger Merge Worker
commit
cbcf8d9b05
10 changed files with 28 additions and 638 deletions
|
|
@ -15,93 +15,9 @@
|
|||
package android
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
)
|
||||
|
||||
func init() {
|
||||
pctx.SourcePathVariable("merge_notices", "build/soong/scripts/mergenotice.py")
|
||||
pctx.SourcePathVariable("generate_notice", "build/soong/scripts/generate-notice-files.py")
|
||||
|
||||
pctx.HostBinToolVariable("minigzip", "minigzip")
|
||||
}
|
||||
|
||||
type NoticeOutputs struct {
|
||||
Merged OptionalPath
|
||||
TxtOutput OptionalPath
|
||||
HtmlOutput OptionalPath
|
||||
HtmlGzOutput OptionalPath
|
||||
}
|
||||
|
||||
var (
|
||||
mergeNoticesRule = pctx.AndroidStaticRule("mergeNoticesRule", blueprint.RuleParams{
|
||||
Command: `${merge_notices} --output $out $in`,
|
||||
CommandDeps: []string{"${merge_notices}"},
|
||||
Description: "merge notice files into $out",
|
||||
})
|
||||
|
||||
generateNoticeRule = pctx.AndroidStaticRule("generateNoticeRule", blueprint.RuleParams{
|
||||
Command: `rm -rf $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` +
|
||||
`mkdir -p $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` +
|
||||
`${generate_notice} --text-output $txtOut --html-output $htmlOut -t "$title" -s $inputDir && ` +
|
||||
`${minigzip} -c $htmlOut > $out`,
|
||||
CommandDeps: []string{"${generate_notice}", "${minigzip}"},
|
||||
Description: "produce notice file $out",
|
||||
}, "txtOut", "htmlOut", "title", "inputDir")
|
||||
)
|
||||
|
||||
func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Path) {
|
||||
ctx.Build(pctx, BuildParams{
|
||||
Rule: mergeNoticesRule,
|
||||
Description: "merge notices",
|
||||
Inputs: noticePaths,
|
||||
Output: mergedNotice,
|
||||
})
|
||||
}
|
||||
|
||||
func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilename string,
|
||||
noticePaths []Path) NoticeOutputs {
|
||||
// Merge all NOTICE files into one.
|
||||
// TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass.
|
||||
//
|
||||
// generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules
|
||||
// about input NOTICE file paths.
|
||||
// 1. Their relative paths to the src root become their NOTICE index titles. We want to use
|
||||
// on-device paths as titles, and so output the merged NOTICE file the corresponding location.
|
||||
// 2. They must end with .txt extension. Otherwise, they're ignored.
|
||||
noticeRelPath := InstallPathToOnDevicePath(ctx, installPath.Join(ctx, installFilename+".txt"))
|
||||
mergedNotice := PathForModuleOut(ctx, filepath.Join("NOTICE_FILES/src", noticeRelPath))
|
||||
MergeNotices(ctx, mergedNotice, noticePaths)
|
||||
|
||||
// Transform the merged NOTICE file into a gzipped HTML file.
|
||||
txtOuptut := PathForModuleOut(ctx, "NOTICE_txt", "NOTICE.txt")
|
||||
htmlOutput := PathForModuleOut(ctx, "NOTICE_html", "NOTICE.html")
|
||||
htmlGzOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz")
|
||||
title := "Notices for " + ctx.ModuleName()
|
||||
ctx.Build(pctx, BuildParams{
|
||||
Rule: generateNoticeRule,
|
||||
Description: "generate notice output",
|
||||
Input: mergedNotice,
|
||||
Output: htmlGzOutput,
|
||||
ImplicitOutputs: WritablePaths{txtOuptut, htmlOutput},
|
||||
Args: map[string]string{
|
||||
"txtOut": txtOuptut.String(),
|
||||
"htmlOut": htmlOutput.String(),
|
||||
"title": title,
|
||||
"inputDir": PathForModuleOut(ctx, "NOTICE_FILES/src").String(),
|
||||
},
|
||||
})
|
||||
|
||||
return NoticeOutputs{
|
||||
Merged: OptionalPathForPath(mergedNotice),
|
||||
TxtOutput: OptionalPathForPath(txtOuptut),
|
||||
HtmlOutput: OptionalPathForPath(htmlOutput),
|
||||
HtmlGzOutput: OptionalPathForPath(htmlGzOutput),
|
||||
}
|
||||
}
|
||||
|
||||
// BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's
|
||||
// generated license metadata file.
|
||||
func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) {
|
||||
|
|
@ -112,5 +28,18 @@ func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile Writ
|
|||
FlagWithOutput("-o ", outputFile).
|
||||
FlagWithDepFile("-d ", depsFile).
|
||||
Input(ctx.Module().base().licenseMetadataFile)
|
||||
rule.Build("container_notice", "container notice file")
|
||||
rule.Build("text_notice", "container notice file")
|
||||
}
|
||||
|
||||
// BuildNoticeHtmlOutputFromLicenseMetadata writes out a notice text file based on the module's
|
||||
// generated license metadata file.
|
||||
func BuildNoticeHtmlOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) {
|
||||
depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", "."))
|
||||
rule := NewRuleBuilder(pctx, ctx)
|
||||
rule.Command().
|
||||
BuiltTool("htmlnotice").
|
||||
FlagWithOutput("-o ", outputFile).
|
||||
FlagWithDepFile("-d ", depsFile).
|
||||
Input(ctx.Module().base().licenseMetadataFile)
|
||||
rule.Build("html_notice", "container notice file")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -396,10 +396,6 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData {
|
|||
}
|
||||
a.writeRequiredModules(w, moduleNames)
|
||||
|
||||
if a.mergedNotices.Merged.Valid() {
|
||||
fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", a.mergedNotices.Merged.Path().String())
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "include $(BUILD_PREBUILT)")
|
||||
|
||||
if apexType == imageApex {
|
||||
|
|
|
|||
|
|
@ -414,8 +414,8 @@ type apexBundle struct {
|
|||
// Processed file_contexts files
|
||||
fileContexts android.WritablePath
|
||||
|
||||
// Struct holding the merged notice file paths in different formats
|
||||
mergedNotices android.NoticeOutputs
|
||||
// Path to notice file in html.gz format.
|
||||
htmlGzNotice android.WritablePath
|
||||
|
||||
// The built APEX file. This is the main product.
|
||||
// Could be .apex or .capex
|
||||
|
|
@ -488,7 +488,6 @@ const (
|
|||
type apexFile struct {
|
||||
// buildFile is put in the installDir inside the APEX.
|
||||
builtFile android.Path
|
||||
noticeFiles android.Paths
|
||||
installDir string
|
||||
customStem string
|
||||
symlinks []string // additional symlinks
|
||||
|
|
@ -528,7 +527,6 @@ func newApexFile(ctx android.BaseModuleContext, builtFile android.Path, androidM
|
|||
module: module,
|
||||
}
|
||||
if module != nil {
|
||||
ret.noticeFiles = module.NoticeFiles()
|
||||
ret.moduleDir = ctx.OtherModuleDir(module)
|
||||
ret.requiredModuleNames = module.RequiredModuleNames()
|
||||
ret.targetRequiredModuleNames = module.TargetRequiredModuleNames()
|
||||
|
|
|
|||
|
|
@ -591,15 +591,6 @@ func TestBasicApex(t *testing.T) {
|
|||
t.Errorf("Could not find all expected symlinks! foo: %t, foo_link_64: %t. Command was %s", found_foo, found_foo_link_64, copyCmds)
|
||||
}
|
||||
|
||||
mergeNoticesRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("mergeNoticesRule")
|
||||
noticeInputs := mergeNoticesRule.Inputs.Strings()
|
||||
if len(noticeInputs) != 3 {
|
||||
t.Errorf("number of input notice files: expected = 3, actual = %q", len(noticeInputs))
|
||||
}
|
||||
ensureListContains(t, noticeInputs, "NOTICE")
|
||||
ensureListContains(t, noticeInputs, "custom_notice")
|
||||
ensureListContains(t, noticeInputs, "custom_notice_for_static_lib")
|
||||
|
||||
fullDepsInfo := strings.Split(ctx.ModuleForTests("myapex", "android_common_myapex_image").Output("depsinfo/fulllist.txt").Args["content"], "\\n")
|
||||
ensureListContains(t, fullDepsInfo, " myjar(minSdkVersion:(no version)) <- myapex")
|
||||
ensureListContains(t, fullDepsInfo, " mylib2(minSdkVersion:(no version)) <- mylib")
|
||||
|
|
|
|||
|
|
@ -305,32 +305,6 @@ func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Output
|
|||
return output.OutputPath
|
||||
}
|
||||
|
||||
// buildNoticeFiles creates a buile rule for aggregating notice files from the modules that
|
||||
// contributes to this APEX. The notice files are merged into a big notice file.
|
||||
func (a *apexBundle) buildNoticeFiles(ctx android.ModuleContext, apexFileName string) android.NoticeOutputs {
|
||||
var noticeFiles android.Paths
|
||||
|
||||
a.WalkPayloadDeps(ctx, func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool {
|
||||
if externalDep {
|
||||
// As soon as the dependency graph crosses the APEX boundary, don't go further.
|
||||
return false
|
||||
}
|
||||
noticeFiles = append(noticeFiles, to.NoticeFiles()...)
|
||||
return true
|
||||
})
|
||||
|
||||
// TODO(jiyong): why do we need this? WalkPayloadDeps should have already covered this.
|
||||
for _, fi := range a.filesInfo {
|
||||
noticeFiles = append(noticeFiles, fi.noticeFiles...)
|
||||
}
|
||||
|
||||
if len(noticeFiles) == 0 {
|
||||
return android.NoticeOutputs{}
|
||||
}
|
||||
|
||||
return android.BuildNoticeOutput(ctx, a.installDir, apexFileName, android.SortedUniquePaths(noticeFiles))
|
||||
}
|
||||
|
||||
// buildInstalledFilesFile creates a build rule for the installed-files.txt file where the list of
|
||||
// files included in this APEX is shown. The text file is dist'ed so that people can see what's
|
||||
// included in the APEX without actually downloading and extracting it.
|
||||
|
|
@ -642,12 +616,11 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
|
|||
optFlags = append(optFlags, "--logging_parent ", a.overridableProperties.Logging_parent)
|
||||
}
|
||||
|
||||
a.mergedNotices = a.buildNoticeFiles(ctx, a.Name()+suffix)
|
||||
if a.mergedNotices.HtmlGzOutput.Valid() {
|
||||
// If there's a NOTICE file, embed it as an asset file in the APEX.
|
||||
implicitInputs = append(implicitInputs, a.mergedNotices.HtmlGzOutput.Path())
|
||||
optFlags = append(optFlags, "--assets_dir "+filepath.Dir(a.mergedNotices.HtmlGzOutput.String()))
|
||||
}
|
||||
// Create a NOTICE file, and embed it as an asset file in the APEX.
|
||||
a.htmlGzNotice = android.PathForModuleOut(ctx, "NOTICE/NOTICES.html.gz")
|
||||
android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, a.htmlGzNotice)
|
||||
implicitInputs = append(implicitInputs, a.htmlGzNotice)
|
||||
optFlags = append(optFlags, "--assets_dir "+filepath.Dir(a.htmlGzNotice.String()))
|
||||
|
||||
if (moduleMinSdkVersion.GreaterThan(android.SdkVersion_Android10) && !a.shouldGenerateHashtree()) && !compressionEnabled {
|
||||
// Apexes which are supposed to be installed in builtin dirs(/system, etc)
|
||||
|
|
|
|||
|
|
@ -409,22 +409,6 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries {
|
|||
entries.SetOptionalPaths("LOCAL_SOONG_LINT_REPORTS", app.linter.reports)
|
||||
},
|
||||
},
|
||||
ExtraFooters: []android.AndroidMkExtraFootersFunc{
|
||||
func(w io.Writer, name, prefix, moduleDir string) {
|
||||
if app.noticeOutputs.Merged.Valid() {
|
||||
fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n",
|
||||
app.installApkName, app.noticeOutputs.Merged.String(), app.installApkName+"_NOTICE")
|
||||
}
|
||||
if app.noticeOutputs.TxtOutput.Valid() {
|
||||
fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n",
|
||||
app.installApkName, app.noticeOutputs.TxtOutput.String(), app.installApkName+"_NOTICE.txt")
|
||||
}
|
||||
if app.noticeOutputs.HtmlOutput.Valid() {
|
||||
fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n",
|
||||
app.installApkName, app.noticeOutputs.HtmlOutput.String(), app.installApkName+"_NOTICE.html")
|
||||
}
|
||||
},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
|
|
|
|||
55
java/app.go
55
java/app.go
|
|
@ -19,7 +19,6 @@ package java
|
|||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
|
|
@ -164,8 +163,6 @@ type AndroidApp struct {
|
|||
|
||||
additionalAaptFlags []string
|
||||
|
||||
noticeOutputs android.NoticeOutputs
|
||||
|
||||
overriddenManifestPackageName string
|
||||
|
||||
android.ApexBundleDepsInfo
|
||||
|
|
@ -523,53 +520,6 @@ func (a *AndroidApp) JNISymbolsInstalls(installPath string) android.RuleBuilderI
|
|||
return jniSymbols
|
||||
}
|
||||
|
||||
func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext) {
|
||||
// Collect NOTICE files from all dependencies.
|
||||
seenModules := make(map[android.Module]bool)
|
||||
noticePathSet := make(map[android.Path]bool)
|
||||
|
||||
ctx.WalkDeps(func(child android.Module, parent android.Module) bool {
|
||||
// Have we already seen this?
|
||||
if _, ok := seenModules[child]; ok {
|
||||
return false
|
||||
}
|
||||
seenModules[child] = true
|
||||
|
||||
// Skip host modules.
|
||||
if child.Target().Os.Class == android.Host {
|
||||
return false
|
||||
}
|
||||
|
||||
paths := child.(android.Module).NoticeFiles()
|
||||
if len(paths) > 0 {
|
||||
for _, path := range paths {
|
||||
noticePathSet[path] = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
// If the app has one, add it too.
|
||||
if len(a.NoticeFiles()) > 0 {
|
||||
for _, path := range a.NoticeFiles() {
|
||||
noticePathSet[path] = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(noticePathSet) == 0 {
|
||||
return
|
||||
}
|
||||
var noticePaths []android.Path
|
||||
for path := range noticePathSet {
|
||||
noticePaths = append(noticePaths, path)
|
||||
}
|
||||
sort.Slice(noticePaths, func(i, j int) bool {
|
||||
return noticePaths[i].String() < noticePaths[j].String()
|
||||
})
|
||||
|
||||
a.noticeOutputs = android.BuildNoticeOutput(ctx, a.installDir, a.installApkName+".apk", noticePaths)
|
||||
}
|
||||
|
||||
// Reads and prepends a main cert from the default cert dir if it hasn't been set already, i.e. it
|
||||
// isn't a cert module reference. Also checks and enforces system cert restriction if applicable.
|
||||
func processMainCert(m android.ModuleBase, certPropValue string, certificates []Certificate, ctx android.ModuleContext) []Certificate {
|
||||
|
|
@ -636,9 +586,10 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) {
|
|||
}
|
||||
a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir)
|
||||
|
||||
a.noticeBuildActions(ctx)
|
||||
noticeFile := android.PathForModuleOut(ctx, "NOTICES.html.gz")
|
||||
android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, noticeFile)
|
||||
if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") {
|
||||
a.aapt.noticeFile = a.noticeOutputs.HtmlGzOutput
|
||||
a.aapt.noticeFile = android.OptionalPathForPath(noticeFile)
|
||||
}
|
||||
|
||||
a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx)
|
||||
|
|
|
|||
111
java/app_test.go
111
java/app_test.go
|
|
@ -27,7 +27,6 @@ import (
|
|||
"android/soong/android"
|
||||
"android/soong/cc"
|
||||
"android/soong/dexpreopt"
|
||||
"android/soong/genrule"
|
||||
)
|
||||
|
||||
// testApp runs tests using the prepareForJavaTest
|
||||
|
|
@ -2722,116 +2721,6 @@ func TestCodelessApp(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestEmbedNotice(t *testing.T) {
|
||||
result := android.GroupFixturePreparers(
|
||||
PrepareForTestWithJavaDefaultModules,
|
||||
cc.PrepareForTestWithCcDefaultModules,
|
||||
genrule.PrepareForTestWithGenRuleBuildComponents,
|
||||
android.MockFS{
|
||||
"APP_NOTICE": nil,
|
||||
"GENRULE_NOTICE": nil,
|
||||
"LIB_NOTICE": nil,
|
||||
"TOOL_NOTICE": nil,
|
||||
}.AddToFixture(),
|
||||
).RunTestWithBp(t, `
|
||||
android_app {
|
||||
name: "foo",
|
||||
srcs: ["a.java"],
|
||||
static_libs: ["javalib"],
|
||||
jni_libs: ["libjni"],
|
||||
notice: "APP_NOTICE",
|
||||
embed_notices: true,
|
||||
sdk_version: "current",
|
||||
}
|
||||
|
||||
// No embed_notice flag
|
||||
android_app {
|
||||
name: "bar",
|
||||
srcs: ["a.java"],
|
||||
jni_libs: ["libjni"],
|
||||
notice: "APP_NOTICE",
|
||||
sdk_version: "current",
|
||||
}
|
||||
|
||||
// No NOTICE files
|
||||
android_app {
|
||||
name: "baz",
|
||||
srcs: ["a.java"],
|
||||
embed_notices: true,
|
||||
sdk_version: "current",
|
||||
}
|
||||
|
||||
cc_library {
|
||||
name: "libjni",
|
||||
system_shared_libs: [],
|
||||
stl: "none",
|
||||
notice: "LIB_NOTICE",
|
||||
sdk_version: "current",
|
||||
}
|
||||
|
||||
java_library {
|
||||
name: "javalib",
|
||||
srcs: [
|
||||
":gen",
|
||||
],
|
||||
sdk_version: "current",
|
||||
}
|
||||
|
||||
genrule {
|
||||
name: "gen",
|
||||
tools: ["gentool"],
|
||||
out: ["gen.java"],
|
||||
notice: "GENRULE_NOTICE",
|
||||
}
|
||||
|
||||
java_binary_host {
|
||||
name: "gentool",
|
||||
srcs: ["b.java"],
|
||||
notice: "TOOL_NOTICE",
|
||||
}
|
||||
`)
|
||||
|
||||
// foo has NOTICE files to process, and embed_notices is true.
|
||||
foo := result.ModuleForTests("foo", "android_common")
|
||||
// verify merge notices rule.
|
||||
mergeNotices := foo.Rule("mergeNoticesRule")
|
||||
noticeInputs := mergeNotices.Inputs.Strings()
|
||||
// TOOL_NOTICE should be excluded as it's a host module.
|
||||
if len(mergeNotices.Inputs) != 3 {
|
||||
t.Errorf("number of input notice files: expected = 3, actual = %q", noticeInputs)
|
||||
}
|
||||
if !inList("APP_NOTICE", noticeInputs) {
|
||||
t.Errorf("APP_NOTICE is missing from notice files, %q", noticeInputs)
|
||||
}
|
||||
if !inList("LIB_NOTICE", noticeInputs) {
|
||||
t.Errorf("LIB_NOTICE is missing from notice files, %q", noticeInputs)
|
||||
}
|
||||
if !inList("GENRULE_NOTICE", noticeInputs) {
|
||||
t.Errorf("GENRULE_NOTICE is missing from notice files, %q", noticeInputs)
|
||||
}
|
||||
// aapt2 flags should include -A <NOTICE dir> so that its contents are put in the APK's /assets.
|
||||
res := foo.Output("package-res.apk")
|
||||
aapt2Flags := res.Args["flags"]
|
||||
e := "-A out/soong/.intermediates/foo/android_common/NOTICE"
|
||||
android.AssertStringDoesContain(t, "expected.apkPath", aapt2Flags, e)
|
||||
|
||||
// bar has NOTICE files to process, but embed_notices is not set.
|
||||
bar := result.ModuleForTests("bar", "android_common")
|
||||
res = bar.Output("package-res.apk")
|
||||
aapt2Flags = res.Args["flags"]
|
||||
e = "-A out/soong/.intermediates/bar/android_common/NOTICE"
|
||||
android.AssertStringDoesNotContain(t, "bar shouldn't have the asset dir flag for NOTICE", aapt2Flags, e)
|
||||
|
||||
// baz's embed_notice is true, but it doesn't have any NOTICE files.
|
||||
baz := result.ModuleForTests("baz", "android_common")
|
||||
res = baz.Output("package-res.apk")
|
||||
aapt2Flags = res.Args["flags"]
|
||||
e = "-A out/soong/.intermediates/baz/android_common/NOTICE"
|
||||
if strings.Contains(aapt2Flags, e) {
|
||||
t.Errorf("baz shouldn't have the asset dir flag for NOTICE: %q", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUncompressDex(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
|
|
|||
|
|
@ -1,272 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (C) 2012 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Usage: generate-notice-files --text-output [plain text output file] \
|
||||
--html-output [html output file] \
|
||||
--xml-output [xml output file] \
|
||||
-t [file title] -s [directory of notices]
|
||||
|
||||
Generate the Android notice files, including both text and html files.
|
||||
|
||||
-h to display this usage message and exit.
|
||||
"""
|
||||
from collections import defaultdict
|
||||
import argparse
|
||||
import hashlib
|
||||
import itertools
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
|
||||
MD5_BLOCKSIZE = 1024 * 1024
|
||||
HTML_ESCAPE_TABLE = {
|
||||
b"&": b"&",
|
||||
b'"': b""",
|
||||
b"'": b"'",
|
||||
b">": b">",
|
||||
b"<": b"<",
|
||||
}
|
||||
|
||||
def md5sum(filename):
|
||||
"""Calculate an MD5 of the file given by FILENAME,
|
||||
and return hex digest as a string.
|
||||
Output should be compatible with md5sum command"""
|
||||
|
||||
f = open(filename, "rb")
|
||||
sum = hashlib.md5()
|
||||
while 1:
|
||||
block = f.read(MD5_BLOCKSIZE)
|
||||
if not block:
|
||||
break
|
||||
sum.update(block)
|
||||
f.close()
|
||||
return sum.hexdigest()
|
||||
|
||||
|
||||
def html_escape(text):
|
||||
"""Produce entities within text."""
|
||||
# Using for i in text doesn't work since i will be an int, not a byte.
|
||||
# There are multiple ways to solve this, but the most performant way
|
||||
# to iterate over a byte array is to use unpack. Using the
|
||||
# for i in range(len(text)) and using that to get a byte using array
|
||||
# slices is twice as slow as this method.
|
||||
return b"".join(HTML_ESCAPE_TABLE.get(i,i) for i in struct.unpack(str(len(text)) + 'c', text))
|
||||
|
||||
HTML_OUTPUT_CSS=b"""
|
||||
<style type="text/css">
|
||||
body { padding: 0; font-family: sans-serif; }
|
||||
.same-license { background-color: #eeeeee; border-top: 20px solid white; padding: 10px; }
|
||||
.label { font-weight: bold; }
|
||||
.file-list { margin-left: 1em; color: blue; }
|
||||
</style>
|
||||
|
||||
"""
|
||||
|
||||
def combine_notice_files_html(file_hash, input_dir, output_filename):
|
||||
"""Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME."""
|
||||
|
||||
SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
|
||||
|
||||
# Set up a filename to row id table (anchors inside tables don't work in
|
||||
# most browsers, but href's to table row ids do)
|
||||
id_table = {}
|
||||
id_count = 0
|
||||
for value in file_hash:
|
||||
for filename in value:
|
||||
id_table[filename] = id_count
|
||||
id_count += 1
|
||||
|
||||
# Open the output file, and output the header pieces
|
||||
output_file = open(output_filename, "wb")
|
||||
|
||||
output_file.write(b"<html><head>\n")
|
||||
output_file.write(HTML_OUTPUT_CSS)
|
||||
output_file.write(b'</head><body topmargin="0" leftmargin="0" rightmargin="0" bottommargin="0">\n')
|
||||
|
||||
# Output our table of contents
|
||||
output_file.write(b'<div class="toc">\n')
|
||||
output_file.write(b"<ul>\n")
|
||||
|
||||
# Flatten the list of lists into a single list of filenames
|
||||
sorted_filenames = sorted(itertools.chain.from_iterable(file_hash))
|
||||
|
||||
# Print out a nice table of contents
|
||||
for filename in sorted_filenames:
|
||||
stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)
|
||||
output_file.write(('<li><a href="#id%d">%s</a></li>\n' % (id_table.get(filename), stripped_filename)).encode())
|
||||
|
||||
output_file.write(b"</ul>\n")
|
||||
output_file.write(b"</div><!-- table of contents -->\n")
|
||||
# Output the individual notice file lists
|
||||
output_file.write(b'<table cellpadding="0" cellspacing="0" border="0">\n')
|
||||
for value in file_hash:
|
||||
output_file.write(('<tr id="id%d"><td class="same-license">\n' % id_table.get(value[0])).encode())
|
||||
output_file.write(b'<div class="label">Notices for file(s):</div>\n')
|
||||
output_file.write(b'<div class="file-list">\n')
|
||||
for filename in value:
|
||||
output_file.write(("%s <br/>\n" % (SRC_DIR_STRIP_RE.sub(r"\1", filename))).encode())
|
||||
output_file.write(b"</div><!-- file-list -->\n\n")
|
||||
output_file.write(b'<pre class="license-text">\n')
|
||||
with open(value[0], "rb") as notice_file:
|
||||
output_file.write(html_escape(notice_file.read()))
|
||||
output_file.write(b"\n</pre><!-- license-text -->\n")
|
||||
output_file.write(b"</td></tr><!-- same-license -->\n\n\n\n")
|
||||
|
||||
# Finish off the file output
|
||||
output_file.write(b"</table>\n")
|
||||
output_file.write(b"</body></html>\n")
|
||||
output_file.close()
|
||||
|
||||
def combine_notice_files_text(file_hash, input_dir, output_filename, file_title):
|
||||
"""Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME."""
|
||||
|
||||
SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
|
||||
output_file = open(output_filename, "wb")
|
||||
output_file.write(file_title.encode())
|
||||
output_file.write(b"\n")
|
||||
for value in file_hash:
|
||||
output_file.write(b"============================================================\n")
|
||||
output_file.write(b"Notices for file(s):\n")
|
||||
for filename in value:
|
||||
output_file.write(SRC_DIR_STRIP_RE.sub(r"\1", filename).encode())
|
||||
output_file.write(b"\n")
|
||||
output_file.write(b"------------------------------------------------------------\n")
|
||||
with open(value[0], "rb") as notice_file:
|
||||
output_file.write(notice_file.read())
|
||||
output_file.write(b"\n")
|
||||
output_file.close()
|
||||
|
||||
def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename):
|
||||
"""Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME."""
|
||||
|
||||
SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
|
||||
|
||||
# Set up a filename to row id table (anchors inside tables don't work in
|
||||
# most browsers, but href's to table row ids do)
|
||||
id_table = {}
|
||||
for file_key, files in files_with_same_hash.items():
|
||||
for filename in files:
|
||||
id_table[filename] = file_key
|
||||
|
||||
# Open the output file, and output the header pieces
|
||||
output_file = open(output_filename, "wb")
|
||||
|
||||
output_file.write(b'<?xml version="1.0" encoding="utf-8"?>\n')
|
||||
output_file.write(b"<licenses>\n")
|
||||
|
||||
# Flatten the list of lists into a single list of filenames
|
||||
sorted_filenames = sorted(list(id_table))
|
||||
|
||||
# Print out a nice table of contents
|
||||
for filename in sorted_filenames:
|
||||
stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)
|
||||
output_file.write(('<file-name contentId="%s">%s</file-name>\n' % (id_table.get(filename), stripped_filename)).encode())
|
||||
output_file.write(b"\n\n")
|
||||
|
||||
processed_file_keys = []
|
||||
# Output the individual notice file lists
|
||||
for filename in sorted_filenames:
|
||||
file_key = id_table.get(filename)
|
||||
if file_key in processed_file_keys:
|
||||
continue
|
||||
processed_file_keys.append(file_key)
|
||||
|
||||
output_file.write(('<file-content contentId="%s"><![CDATA[' % file_key).encode())
|
||||
with open(filename, "rb") as notice_file:
|
||||
output_file.write(html_escape(notice_file.read()))
|
||||
output_file.write(b"]]></file-content>\n\n")
|
||||
|
||||
# Finish off the file output
|
||||
output_file.write(b"</licenses>\n")
|
||||
output_file.close()
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--text-output', required=True,
|
||||
help='The text output file path.')
|
||||
parser.add_argument(
|
||||
'--html-output',
|
||||
help='The html output file path.')
|
||||
parser.add_argument(
|
||||
'--xml-output',
|
||||
help='The xml output file path.')
|
||||
parser.add_argument(
|
||||
'-t', '--title', required=True,
|
||||
help='The file title.')
|
||||
parser.add_argument(
|
||||
'-s', '--source-dir', required=True,
|
||||
help='The directory containing notices.')
|
||||
parser.add_argument(
|
||||
'-i', '--included-subdirs', action='append',
|
||||
help='The sub directories which should be included.')
|
||||
parser.add_argument(
|
||||
'-e', '--excluded-subdirs', action='append',
|
||||
help='The sub directories which should be excluded.')
|
||||
return parser.parse_args()
|
||||
|
||||
def main(argv):
|
||||
args = get_args()
|
||||
|
||||
txt_output_file = args.text_output
|
||||
html_output_file = args.html_output
|
||||
xml_output_file = args.xml_output
|
||||
file_title = args.title
|
||||
included_subdirs = []
|
||||
excluded_subdirs = []
|
||||
if args.included_subdirs is not None:
|
||||
included_subdirs = args.included_subdirs
|
||||
if args.excluded_subdirs is not None:
|
||||
excluded_subdirs = args.excluded_subdirs
|
||||
|
||||
# Find all the notice files and md5 them
|
||||
input_dir = os.path.normpath(args.source_dir)
|
||||
files_with_same_hash = defaultdict(list)
|
||||
for root, dir, files in os.walk(input_dir):
|
||||
for file in files:
|
||||
matched = True
|
||||
if len(included_subdirs) > 0:
|
||||
matched = False
|
||||
for subdir in included_subdirs:
|
||||
if (root == (input_dir + '/' + subdir) or
|
||||
root.startswith(input_dir + '/' + subdir + '/')):
|
||||
matched = True
|
||||
break
|
||||
elif len(excluded_subdirs) > 0:
|
||||
for subdir in excluded_subdirs:
|
||||
if (root == (input_dir + '/' + subdir) or
|
||||
root.startswith(input_dir + '/' + subdir + '/')):
|
||||
matched = False
|
||||
break
|
||||
if matched and file.endswith(".txt"):
|
||||
filename = os.path.join(root, file)
|
||||
file_md5sum = md5sum(filename)
|
||||
files_with_same_hash[file_md5sum].append(filename)
|
||||
|
||||
filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(list(files_with_same_hash))]
|
||||
|
||||
combine_notice_files_text(filesets, input_dir, txt_output_file, file_title)
|
||||
|
||||
if html_output_file is not None:
|
||||
combine_notice_files_html(filesets, input_dir, html_output_file)
|
||||
|
||||
if xml_output_file is not None:
|
||||
combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (C) 2019 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
Merges input notice files to the output file while ignoring duplicated files
|
||||
This script shouldn't be confused with build/soong/scripts/generate-notice-files.py
|
||||
which is responsible for creating the final notice file for all artifacts
|
||||
installed. This script has rather limited scope; it is meant to create a merged
|
||||
notice file for a set of modules that are packaged together, e.g. in an APEX.
|
||||
The merged notice file does not reveal the individual files in the package.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser(description='Merge notice files.')
|
||||
parser.add_argument('--output', help='output file path.')
|
||||
parser.add_argument('inputs', metavar='INPUT', nargs='+',
|
||||
help='input notice file')
|
||||
return parser.parse_args()
|
||||
|
||||
def main(argv):
|
||||
args = get_args()
|
||||
|
||||
processed = set()
|
||||
with open(args.output, 'w+') as output:
|
||||
for input in args.inputs:
|
||||
with open(input, 'r') as f:
|
||||
data = f.read().strip()
|
||||
if data not in processed:
|
||||
processed.add(data)
|
||||
output.write('%s\n\n' % data)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv)
|
||||
Loading…
Reference in a new issue