Merge changes I40a05866,Id3d6e701

* changes:
  license metadata reverse trace
  Fix copy+paste error.
This commit is contained in:
Treehugger Robot 2022-02-01 01:25:59 +00:00 committed by Gerrit Code Review
commit 6ebf02914c
8 changed files with 557 additions and 8 deletions

View file

@ -62,6 +62,13 @@ blueprint_go_binary {
testSrcs: ["cmd/htmlnotice/htmlnotice_test.go"], testSrcs: ["cmd/htmlnotice/htmlnotice_test.go"],
} }
blueprint_go_binary {
name: "rtrace",
srcs: ["cmd/rtrace/rtrace.go"],
deps: ["compliance-module"],
testSrcs: ["cmd/rtrace/rtrace_test.go"],
}
blueprint_go_binary { blueprint_go_binary {
name: "shippedlibs", name: "shippedlibs",
srcs: ["cmd/shippedlibs/shippedlibs.go"], srcs: ["cmd/shippedlibs/shippedlibs.go"],

View file

@ -919,7 +919,7 @@ func Test_plaintext(t *testing.T) {
for len(outList) > startLine && len(expectedList) > startLine && outList[startLine] == expectedList[startLine] { for len(outList) > startLine && len(expectedList) > startLine && outList[startLine] == expectedList[startLine] {
startLine++ startLine++
} }
t.Errorf("listshare: gotStdout = %v, want %v, somewhere near line %d Stdout = %v, want %v", t.Errorf("dumpresoliutions: gotStdout = %v, want %v, somewhere near line %d Stdout = %v, want %v",
out, expected, startLine+1, outList[startLine], expectedList[startLine]) out, expected, startLine+1, outList[startLine], expectedList[startLine])
} }
}) })

View file

@ -0,0 +1,185 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"android/soong/tools/compliance"
)
var (
sources = newMultiString("rtrace", "Projects or metadata files to trace back from. (required; multiple allowed)")
stripPrefix = flag.String("strip_prefix", "", "Prefix to remove from paths. i.e. path to root")
failNoneRequested = fmt.Errorf("\nNo license metadata files requested")
failNoSources = fmt.Errorf("\nNo projects or metadata files to trace back from")
failNoLicenses = fmt.Errorf("No licenses found")
)
type context struct {
sources []string
stripPrefix string
}
func init() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...}
Outputs a space-separated Target ActsOn Origin Condition tuple for each
resolution in the graph. When -dot flag given, outputs nodes and edges
in graphviz directed graph format.
If one or more '-c condition' conditions are given, outputs the
resolution for the union of the conditions. Otherwise, outputs the
resolution for all conditions.
In plain text mode, when '-label_conditions' is requested, the Target
and Origin have colon-separated license conditions appended:
i.e. target:condition1:condition2 etc.
Options:
`, filepath.Base(os.Args[0]))
flag.PrintDefaults()
}
}
// newMultiString creates a flag that allows multiple values in an array.
func newMultiString(name, usage string) *multiString {
var f multiString
flag.Var(&f, name, usage)
return &f
}
// multiString implements the flag `Value` interface for multiple strings.
type multiString []string
func (ms *multiString) String() string { return strings.Join(*ms, ", ") }
func (ms *multiString) Set(s string) error { *ms = append(*ms, s); return nil }
func main() {
flag.Parse()
// Must specify at least one root target.
if flag.NArg() == 0 {
flag.Usage()
os.Exit(2)
}
if len(*sources) == 0 {
flag.Usage()
fmt.Fprintf(os.Stderr, "\nMust specify at least 1 --rtrace source.\n")
os.Exit(2)
}
ctx := &context{
sources: *sources,
stripPrefix: *stripPrefix,
}
_, err := traceRestricted(ctx, os.Stdout, os.Stderr, flag.Args()...)
if err != nil {
if err == failNoneRequested {
flag.Usage()
}
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
os.Exit(1)
}
os.Exit(0)
}
// traceRestricted implements the rtrace utility.
func traceRestricted(ctx *context, stdout, stderr io.Writer, files ...string) (*compliance.LicenseGraph, error) {
if len(files) < 1 {
return nil, failNoneRequested
}
if len(ctx.sources) < 1 {
return nil, failNoSources
}
// Read the license graph from the license metadata files (*.meta_lic).
licenseGraph, err := compliance.ReadLicenseGraph(os.DirFS("."), stderr, files)
if err != nil {
return nil, fmt.Errorf("Unable to read license metadata file(s) %q: %v\n", files, err)
}
if licenseGraph == nil {
return nil, failNoLicenses
}
sourceMap := make(map[string]struct{})
for _, source := range ctx.sources {
sourceMap[source] = struct{}{}
}
compliance.TraceTopDownConditions(licenseGraph, func(tn *compliance.TargetNode) compliance.LicenseConditionSet {
if _, isPresent := sourceMap[tn.Name()]; isPresent {
return compliance.ImpliesRestricted
}
for _, project := range tn.Projects() {
if _, isPresent := sourceMap[project]; isPresent {
return compliance.ImpliesRestricted
}
}
return compliance.NewLicenseConditionSet()
})
// targetOut calculates the string to output for `target` adding `sep`-separated conditions as needed.
targetOut := func(target *compliance.TargetNode, sep string) string {
tOut := strings.TrimPrefix(target.Name(), ctx.stripPrefix)
return tOut
}
// outputResolution prints a resolution in the requested format to `stdout`, where one can read
// a resolution as `tname` resolves conditions named in `cnames`.
// `tname` is the name of the target the resolution traces back to.
// `cnames` is the list of conditions to resolve.
outputResolution := func(tname string, cnames []string) {
// ... one edge per line with names in a colon-separated tuple.
fmt.Fprintf(stdout, "%s %s\n", tname, strings.Join(cnames, ":"))
}
// Sort the resolutions by targetname for repeatability/stability.
actions := compliance.WalkResolutionsForCondition(licenseGraph, compliance.ImpliesShared).AllActions()
targets := make(compliance.TargetNodeList, 0, len(actions))
for tn := range actions {
if tn.LicenseConditions().MatchesAnySet(compliance.ImpliesRestricted) {
targets = append(targets, tn)
}
}
sort.Sort(targets)
// Output the sorted targets.
for _, target := range targets {
var tname string
tname = targetOut(target, ":")
// cnames accumulates the list of condition names originating at a single origin that apply to `target`.
cnames := target.LicenseConditions().Names()
// Output 1 line for each attachesTo+actsOn combination.
outputResolution(tname, cnames)
}
fmt.Fprintf(stdout, "restricted conditions trace to %d targets\n", len(targets))
if 0 == len(targets) {
fmt.Fprintln(stdout, " (check for typos in project names or metadata files)")
}
return licenseGraph, nil
}

View file

@ -0,0 +1,316 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"fmt"
"os"
"strings"
"testing"
)
func TestMain(m *testing.M) {
// Change into the parent directory before running the tests
// so they can find the testdata directory.
if err := os.Chdir(".."); err != nil {
fmt.Printf("failed to change to testdata directory: %s\n", err)
os.Exit(1)
}
os.Exit(m.Run())
}
func Test_plaintext(t *testing.T) {
tests := []struct {
condition string
name string
roots []string
ctx context
expectedOut []string
}{
{
condition: "firstparty",
name: "apex",
roots: []string{"highest.apex.meta_lic"},
expectedOut: []string{},
},
{
condition: "firstparty",
name: "apex_trimmed",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/firstparty/bin/bin1.meta_lic"},
stripPrefix: "testdata/firstparty/",
},
expectedOut: []string{},
},
{
condition: "firstparty",
name: "container",
roots: []string{"container.zip.meta_lic"},
expectedOut: []string{},
},
{
condition: "firstparty",
name: "application",
roots: []string{"application.meta_lic"},
expectedOut: []string{},
},
{
condition: "firstparty",
name: "binary",
roots: []string{"bin/bin1.meta_lic"},
expectedOut: []string{},
},
{
condition: "firstparty",
name: "library",
roots: []string{"lib/libd.so.meta_lic"},
expectedOut: []string{},
},
{
condition: "notice",
name: "apex",
roots: []string{"highest.apex.meta_lic"},
expectedOut: []string{},
},
{
condition: "notice",
name: "apex_trimmed",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/notice/bin/bin1.meta_lic"},
stripPrefix: "testdata/notice/",
},
expectedOut: []string{},
},
{
condition: "notice",
name: "container",
roots: []string{"container.zip.meta_lic"},
expectedOut: []string{},
},
{
condition: "notice",
name: "application",
roots: []string{"application.meta_lic"},
expectedOut: []string{},
},
{
condition: "notice",
name: "binary",
roots: []string{"bin/bin1.meta_lic"},
expectedOut: []string{},
},
{
condition: "notice",
name: "library",
roots: []string{"lib/libd.so.meta_lic"},
expectedOut: []string{},
},
{
condition: "reciprocal",
name: "apex",
roots: []string{"highest.apex.meta_lic"},
expectedOut: []string{},
},
{
condition: "reciprocal",
name: "apex_trimmed",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/reciprocal/bin/bin1.meta_lic"},
stripPrefix: "testdata/reciprocal/",
},
expectedOut: []string{},
},
{
condition: "reciprocal",
name: "container",
roots: []string{"container.zip.meta_lic"},
expectedOut: []string{},
},
{
condition: "reciprocal",
name: "application",
roots: []string{"application.meta_lic"},
expectedOut: []string{},
},
{
condition: "reciprocal",
name: "binary",
roots: []string{"bin/bin1.meta_lic"},
expectedOut: []string{},
},
{
condition: "reciprocal",
name: "library",
roots: []string{"lib/libd.so.meta_lic"},
expectedOut: []string{},
},
{
condition: "restricted",
name: "apex",
roots: []string{"highest.apex.meta_lic"},
expectedOut: []string{
"testdata/restricted/lib/liba.so.meta_lic restricted_allows_dynamic_linking",
"testdata/restricted/lib/libb.so.meta_lic restricted",
},
},
{
condition: "restricted",
name: "apex_trimmed_bin1",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/restricted/bin/bin1.meta_lic"},
stripPrefix: "testdata/restricted/",
},
expectedOut: []string{"lib/liba.so.meta_lic restricted_allows_dynamic_linking"},
},
{
condition: "restricted",
name: "apex_trimmed_bin2",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/restricted/bin/bin2.meta_lic"},
stripPrefix: "testdata/restricted/",
},
expectedOut: []string{"lib/libb.so.meta_lic restricted"},
},
{
condition: "restricted",
name: "container",
roots: []string{"container.zip.meta_lic"},
expectedOut: []string{
"testdata/restricted/lib/liba.so.meta_lic restricted_allows_dynamic_linking",
"testdata/restricted/lib/libb.so.meta_lic restricted",
},
},
{
condition: "restricted",
name: "application",
roots: []string{"application.meta_lic"},
expectedOut: []string{"testdata/restricted/lib/liba.so.meta_lic restricted_allows_dynamic_linking"},
},
{
condition: "restricted",
name: "binary",
roots: []string{"bin/bin1.meta_lic"},
expectedOut: []string{"testdata/restricted/lib/liba.so.meta_lic restricted_allows_dynamic_linking"},
},
{
condition: "restricted",
name: "library",
roots: []string{"lib/libd.so.meta_lic"},
expectedOut: []string{},
},
{
condition: "proprietary",
name: "apex",
roots: []string{"highest.apex.meta_lic"},
expectedOut: []string{"testdata/proprietary/lib/libb.so.meta_lic restricted"},
},
{
condition: "proprietary",
name: "apex_trimmed_bin1",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/proprietary/bin/bin1.meta_lic"},
stripPrefix: "testdata/proprietary/",
},
expectedOut: []string{},
},
{
condition: "proprietary",
name: "apex_trimmed_bin2",
roots: []string{"highest.apex.meta_lic"},
ctx: context{
sources: []string{"testdata/proprietary/bin/bin2.meta_lic"},
stripPrefix: "testdata/proprietary/",
},
expectedOut: []string{"lib/libb.so.meta_lic restricted"},
},
{
condition: "proprietary",
name: "container",
roots: []string{"container.zip.meta_lic"},
expectedOut: []string{"testdata/proprietary/lib/libb.so.meta_lic restricted"},
},
{
condition: "proprietary",
name: "application",
roots: []string{"application.meta_lic"},
expectedOut: []string{},
},
{
condition: "proprietary",
name: "binary",
roots: []string{"bin/bin1.meta_lic"},
expectedOut: []string{},
},
{
condition: "proprietary",
name: "library",
roots: []string{"lib/libd.so.meta_lic"},
expectedOut: []string{},
},
}
for _, tt := range tests {
t.Run(tt.condition+" "+tt.name, func(t *testing.T) {
expectedOut := &bytes.Buffer{}
for _, eo := range tt.expectedOut {
expectedOut.WriteString(eo)
expectedOut.WriteString("\n")
}
fmt.Fprintf(expectedOut, "restricted conditions trace to %d targets\n", len(tt.expectedOut))
if 0 == len(tt.expectedOut) {
fmt.Fprintln(expectedOut, " (check for typos in project names or metadata files)")
}
stdout := &bytes.Buffer{}
stderr := &bytes.Buffer{}
rootFiles := make([]string, 0, len(tt.roots))
for _, r := range tt.roots {
rootFiles = append(rootFiles, "testdata/"+tt.condition+"/"+r)
}
if len(tt.ctx.sources) < 1 {
tt.ctx.sources = rootFiles
}
_, err := traceRestricted(&tt.ctx, stdout, stderr, rootFiles...)
t.Logf("rtrace: stderr = %v", stderr)
t.Logf("rtrace: stdout = %v", stdout)
if err != nil {
t.Fatalf("rtrace: error = %v", err)
return
}
if stderr.Len() > 0 {
t.Errorf("rtrace: gotStderr = %v, want none", stderr)
}
out := stdout.String()
expected := expectedOut.String()
if out != expected {
outList := strings.Split(out, "\n")
expectedList := strings.Split(expected, "\n")
startLine := 0
for startLine < len(outList) && startLine < len(expectedList) && outList[startLine] == expectedList[startLine] {
startLine++
}
t.Errorf("rtrace: gotStdout = %v, want %v, somewhere near line %d Stdout = %v, want %v",
out, expected, startLine+1, outList[startLine], expectedList[startLine])
}
})
}
}

View file

@ -218,7 +218,7 @@ func depConditionsPropagatingToTarget(lg *LicenseGraph, e *TargetEdge, depCondit
// aggregation, per policy it ceases to be a pure aggregation in the context of // aggregation, per policy it ceases to be a pure aggregation in the context of
// that derivative work. The `treatAsAggregate` parameter will be false for // that derivative work. The `treatAsAggregate` parameter will be false for
// non-aggregates and for aggregates in non-aggregate contexts. // non-aggregates and for aggregates in non-aggregate contexts.
func targetConditionsPropagatingToDep(lg *LicenseGraph, e *TargetEdge, targetConditions LicenseConditionSet, treatAsAggregate bool) LicenseConditionSet { func targetConditionsPropagatingToDep(lg *LicenseGraph, e *TargetEdge, targetConditions LicenseConditionSet, treatAsAggregate bool, conditionsFn TraceConditions) LicenseConditionSet {
result := targetConditions result := targetConditions
// reverse direction -- none of these apply to things depended-on, only to targets depending-on. // reverse direction -- none of these apply to things depended-on, only to targets depending-on.
@ -232,7 +232,7 @@ func targetConditionsPropagatingToDep(lg *LicenseGraph, e *TargetEdge, targetCon
if treatAsAggregate { if treatAsAggregate {
// If the author of a pure aggregate licenses it restricted, apply restricted to immediate dependencies. // If the author of a pure aggregate licenses it restricted, apply restricted to immediate dependencies.
// Otherwise, restricted does not propagate back down to dependencies. // Otherwise, restricted does not propagate back down to dependencies.
if !LicenseConditionSetFromNames(e.target, e.target.proto.LicenseConditions...).MatchesAnySet(ImpliesRestricted) { if !conditionsFn(e.target).MatchesAnySet(ImpliesRestricted) {
result = result.Difference(ImpliesRestricted) result = result.Difference(ImpliesRestricted)
} }
return result return result

View file

@ -282,7 +282,7 @@ func TestPolicy_edgeConditions(t *testing.T) {
targetConditions = targetConditions.Union(otn.licenseConditions) targetConditions = targetConditions.Union(otn.licenseConditions)
} }
t.Logf("calculate dep conditions for edge=%s, target conditions=%v, treatAsAggregate=%v", edge.String(), targetConditions.Names(), tt.treatAsAggregate) t.Logf("calculate dep conditions for edge=%s, target conditions=%v, treatAsAggregate=%v", edge.String(), targetConditions.Names(), tt.treatAsAggregate)
cs := targetConditionsPropagatingToDep(lg, edge, targetConditions, tt.treatAsAggregate) cs := targetConditionsPropagatingToDep(lg, edge, targetConditions, tt.treatAsAggregate, AllResolutions)
t.Logf("calculated dep conditions as %v", cs.Names()) t.Logf("calculated dep conditions as %v", cs.Names())
actual := cs.Names() actual := cs.Names()
sort.Strings(actual) sort.Strings(actual)

View file

@ -18,6 +18,16 @@ import (
"sync" "sync"
) )
var (
// AllResolutions is a TraceConditions function that resolves all
// unfiltered license conditions.
AllResolutions = TraceConditions(func(tn *TargetNode) LicenseConditionSet { return tn.licenseConditions })
)
// TraceConditions is a function that returns the conditions to trace for each
// target node `tn`.
type TraceConditions func(tn *TargetNode) LicenseConditionSet
// ResolveBottomUpConditions performs a bottom-up walk of the LicenseGraph // ResolveBottomUpConditions performs a bottom-up walk of the LicenseGraph
// propagating conditions up the graph as necessary according to the properties // propagating conditions up the graph as necessary according to the properties
// of each edge and according to each license condition in question. // of each edge and according to each license condition in question.
@ -29,6 +39,14 @@ import (
// not resolve the library and its transitive closure, but the later top-down // not resolve the library and its transitive closure, but the later top-down
// walk will. // walk will.
func ResolveBottomUpConditions(lg *LicenseGraph) { func ResolveBottomUpConditions(lg *LicenseGraph) {
TraceBottomUpConditions(lg, AllResolutions)
}
// TraceBottomUpConditions performs a bottom-up walk of the LicenseGraph
// propagating trace conditions from `conditionsFn` up the graph as necessary
// according to the properties of each edge and according to each license
// condition in question.
func TraceBottomUpConditions(lg *LicenseGraph, conditionsFn TraceConditions) {
// short-cut if already walked and cached // short-cut if already walked and cached
lg.mu.Lock() lg.mu.Lock()
@ -70,7 +88,7 @@ func ResolveBottomUpConditions(lg *LicenseGraph) {
// needs to walk again in non-aggregate context // needs to walk again in non-aggregate context
delete(cmap, target) delete(cmap, target)
} else { } else {
target.resolution |= target.licenseConditions target.resolution |= conditionsFn(target)
amap[target] = struct{}{} amap[target] = struct{}{}
} }
if treatAsAggregate { if treatAsAggregate {
@ -123,6 +141,13 @@ func ResolveBottomUpConditions(lg *LicenseGraph) {
// dependency except restricted. For restricted, the policy is to share the // dependency except restricted. For restricted, the policy is to share the
// source of any libraries linked to restricted code and to provide notice. // source of any libraries linked to restricted code and to provide notice.
func ResolveTopDownConditions(lg *LicenseGraph) { func ResolveTopDownConditions(lg *LicenseGraph) {
TraceTopDownConditions(lg, AllResolutions)
}
// TraceTopDownCondtions performs a top-down walk of the LicenseGraph
// propagating trace conditions returned by `conditionsFn` from target to
// dependency.
func TraceTopDownConditions(lg *LicenseGraph, conditionsFn TraceConditions) {
// short-cut if already walked and cached // short-cut if already walked and cached
lg.mu.Lock() lg.mu.Lock()
@ -139,7 +164,7 @@ func ResolveTopDownConditions(lg *LicenseGraph) {
lg.mu.Unlock() lg.mu.Unlock()
// start with the conditions propagated up the graph // start with the conditions propagated up the graph
ResolveBottomUpConditions(lg) TraceBottomUpConditions(lg, conditionsFn)
// amap contains the set of targets already walked. (guarded by mu) // amap contains the set of targets already walked. (guarded by mu)
amap := make(map[*TargetNode]struct{}) amap := make(map[*TargetNode]struct{})
@ -156,7 +181,7 @@ func ResolveTopDownConditions(lg *LicenseGraph) {
walk = func(fnode *TargetNode, cs LicenseConditionSet, treatAsAggregate bool) { walk = func(fnode *TargetNode, cs LicenseConditionSet, treatAsAggregate bool) {
defer wg.Done() defer wg.Done()
mu.Lock() mu.Lock()
fnode.resolution |= fnode.licenseConditions fnode.resolution |= conditionsFn(fnode)
fnode.resolution |= cs fnode.resolution |= cs
amap[fnode] = struct{}{} amap[fnode] = struct{}{}
if treatAsAggregate { if treatAsAggregate {
@ -168,7 +193,7 @@ func ResolveTopDownConditions(lg *LicenseGraph) {
for _, edge := range fnode.edges { for _, edge := range fnode.edges {
func(edge *TargetEdge) { func(edge *TargetEdge) {
// dcs holds the dpendency conditions inherited from the target // dcs holds the dpendency conditions inherited from the target
dcs := targetConditionsPropagatingToDep(lg, edge, cs, treatAsAggregate) dcs := targetConditionsPropagatingToDep(lg, edge, cs, treatAsAggregate, conditionsFn)
dnode := edge.dependency dnode := edge.dependency
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()

View file

@ -86,6 +86,22 @@ func (rs ResolutionSet) Resolutions(attachesTo *TargetNode) ResolutionList {
return result return result
} }
// AllActions returns the set of actions required to resolve the set omitting
// the attachment.
func (rs ResolutionSet) AllActions() ActionSet {
result := make(ActionSet)
for _, as := range rs {
for actsOn, cs := range as {
if _, ok := result[actsOn]; ok {
result[actsOn] = cs.Union(result[actsOn])
} else {
result[actsOn] = cs
}
}
}
return result
}
// String returns a human-readable string representation of the set. // String returns a human-readable string representation of the set.
func (rs ResolutionSet) String() string { func (rs ResolutionSet) String() string {
var sb strings.Builder var sb strings.Builder