Merge changes Ie7f776a4,I3ca5dd1f,I2e911262 am: 11581cf6b3

am: 89d99337fb

Change-Id: I30312cbbacae65c29e75357e89cc554b5ee1a19d

cmd/diff_target_files/Android.bp

@@ -0,0 +1,16 @@
+blueprint_go_binary {
+    name: "diff_target_files",
+    srcs: [
+        "compare.go",
+        "diff_target_files.go",
+        "glob.go",
+        "target_files.go",
+        "whitelist.go",
+        "zip_artifact.go",
+    ],
+    testSrcs: [
+        "compare_test.go",
+        "glob_test.go",
+        "whitelist_test.go",
+    ],
+}

cmd/diff_target_files/compare.go

@@ -0,0 +1,133 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// compareTargetFiles takes two ZipArtifacts and compares the files they contain by examining
+// the path, size, and CRC of each file.
+func compareTargetFiles(priZip, refZip ZipArtifact, artifact string, whitelists []whitelist, filters []string) (zipDiff, error) {
+	priZipFiles, err := priZip.Files()
+	if err != nil {
+		return zipDiff{}, fmt.Errorf("error fetching target file lists from primary zip %v", err)
+	}
+
+	refZipFiles, err := refZip.Files()
+	if err != nil {
+		return zipDiff{}, fmt.Errorf("error fetching target file lists from reference zip %v", err)
+	}
+
+	priZipFiles, err = filterTargetZipFiles(priZipFiles, artifact, filters)
+	if err != nil {
+		return zipDiff{}, err
+	}
+
+	refZipFiles, err = filterTargetZipFiles(refZipFiles, artifact, filters)
+	if err != nil {
+		return zipDiff{}, err
+	}
+
+	// Compare the file lists from both builds
+	diff := diffTargetFilesLists(refZipFiles, priZipFiles)
+
+	return applyWhitelists(diff, whitelists)
+}
+
+// zipDiff contains the list of files that differ between two zip files.
+type zipDiff struct {
+	modified         [][2]*ZipArtifactFile
+	onlyInA, onlyInB []*ZipArtifactFile
+}
+
+// String pretty-prints the list of files that differ between two zip files.
+func (d *zipDiff) String() string {
+	buf := &bytes.Buffer{}
+
+	must := func(n int, err error) {
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	var sizeChange int64
+
+	if len(d.modified) > 0 {
+		must(fmt.Fprintln(buf, "files modified:"))
+		for _, f := range d.modified {
+			must(fmt.Fprintf(buf, "   %v (%v bytes -> %v bytes)\n", f[0].Name, f[0].UncompressedSize64, f[1].UncompressedSize64))
+			sizeChange += int64(f[1].UncompressedSize64) - int64(f[0].UncompressedSize64)
+		}
+	}
+
+	if len(d.onlyInA) > 0 {
+		must(fmt.Fprintln(buf, "files removed:"))
+		for _, f := range d.onlyInA {
+			must(fmt.Fprintf(buf, " - %v (%v bytes)\n", f.Name, f.UncompressedSize64))
+			sizeChange -= int64(f.UncompressedSize64)
+		}
+	}
+
+	if len(d.onlyInB) > 0 {
+		must(fmt.Fprintln(buf, "files added:"))
+		for _, f := range d.onlyInB {
+			must(fmt.Fprintf(buf, " + %v (%v bytes)\n", f.Name, f.UncompressedSize64))
+			sizeChange += int64(f.UncompressedSize64)
+		}
+	}
+
+	if len(d.modified) > 0 || len(d.onlyInA) > 0 || len(d.onlyInB) > 0 {
+		must(fmt.Fprintf(buf, "total size change: %v bytes\n", sizeChange))
+	}
+
+	return buf.String()
+}
+
+func diffTargetFilesLists(a, b []*ZipArtifactFile) zipDiff {
+	i := 0
+	j := 0
+
+	diff := zipDiff{}
+
+	for i < len(a) && j < len(b) {
+		if a[i].Name == b[j].Name {
+			if a[i].UncompressedSize64 != b[j].UncompressedSize64 || a[i].CRC32 != b[j].CRC32 {
+				diff.modified = append(diff.modified, [2]*ZipArtifactFile{a[i], b[j]})
+			}
+			i++
+			j++
+		} else if a[i].Name < b[j].Name {
+			// a[i] is not present in b
+			diff.onlyInA = append(diff.onlyInA, a[i])
+			i++
+		} else {
+			// b[j] is not present in a
+			diff.onlyInB = append(diff.onlyInB, b[j])
+			j++
+		}
+	}
+	for i < len(a) {
+		diff.onlyInA = append(diff.onlyInA, a[i])
+		i++
+	}
+	for j < len(b) {
+		diff.onlyInB = append(diff.onlyInB, b[j])
+		j++
+	}
+
+	return diff
+}

cmd/diff_target_files/compare_test.go

@@ -0,0 +1,131 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/zip"
+	"reflect"
+	"testing"
+)
+
+func TestDiffTargetFilesLists(t *testing.T) {
+	zipArtifactFile := func(name string, crc32 uint32, size uint64) *ZipArtifactFile {
+		return &ZipArtifactFile{
+			File: &zip.File{
+				FileHeader: zip.FileHeader{
+					Name:               name,
+					CRC32:              crc32,
+					UncompressedSize64: size,
+				},
+			},
+		}
+	}
+	x0 := zipArtifactFile("x", 0, 0)
+	x1 := zipArtifactFile("x", 1, 0)
+	x2 := zipArtifactFile("x", 0, 2)
+	y0 := zipArtifactFile("y", 0, 0)
+	//y1 := zipArtifactFile("y", 1, 0)
+	//y2 := zipArtifactFile("y", 1, 2)
+	z0 := zipArtifactFile("z", 0, 0)
+	z1 := zipArtifactFile("z", 1, 0)
+	//z2 := zipArtifactFile("z", 1, 2)
+
+	testCases := []struct {
+		name string
+		a, b []*ZipArtifactFile
+		diff zipDiff
+	}{
+		{
+			name: "same",
+			a:    []*ZipArtifactFile{x0, y0, z0},
+			b:    []*ZipArtifactFile{x0, y0, z0},
+			diff: zipDiff{nil, nil, nil},
+		},
+		{
+			name: "first only in a",
+			a:    []*ZipArtifactFile{x0, y0, z0},
+			b:    []*ZipArtifactFile{y0, z0},
+			diff: zipDiff{nil, []*ZipArtifactFile{x0}, nil},
+		},
+		{
+			name: "middle only in a",
+			a:    []*ZipArtifactFile{x0, y0, z0},
+			b:    []*ZipArtifactFile{x0, z0},
+			diff: zipDiff{nil, []*ZipArtifactFile{y0}, nil},
+		},
+		{
+			name: "last only in a",
+			a:    []*ZipArtifactFile{x0, y0, z0},
+			b:    []*ZipArtifactFile{x0, y0},
+			diff: zipDiff{nil, []*ZipArtifactFile{z0}, nil},
+		},
+
+		{
+			name: "first only in b",
+			a:    []*ZipArtifactFile{y0, z0},
+			b:    []*ZipArtifactFile{x0, y0, z0},
+			diff: zipDiff{nil, nil, []*ZipArtifactFile{x0}},
+		},
+		{
+			name: "middle only in b",
+			a:    []*ZipArtifactFile{x0, z0},
+			b:    []*ZipArtifactFile{x0, y0, z0},
+			diff: zipDiff{nil, nil, []*ZipArtifactFile{y0}},
+		},
+		{
+			name: "last only in b",
+			a:    []*ZipArtifactFile{x0, y0},
+			b:    []*ZipArtifactFile{x0, y0, z0},
+			diff: zipDiff{nil, nil, []*ZipArtifactFile{z0}},
+		},
+
+		{
+			name: "diff",
+			a:    []*ZipArtifactFile{x0},
+			b:    []*ZipArtifactFile{x1},
+			diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, nil, nil},
+		},
+		{
+			name: "diff plus unique last",
+			a:    []*ZipArtifactFile{x0, y0},
+			b:    []*ZipArtifactFile{x1, z0},
+			diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, []*ZipArtifactFile{y0}, []*ZipArtifactFile{z0}},
+		},
+		{
+			name: "diff plus unique first",
+			a:    []*ZipArtifactFile{x0, z0},
+			b:    []*ZipArtifactFile{y0, z1},
+			diff: zipDiff{[][2]*ZipArtifactFile{{z0, z1}}, []*ZipArtifactFile{x0}, []*ZipArtifactFile{y0}},
+		},
+		{
+			name: "diff size",
+			a:    []*ZipArtifactFile{x0},
+			b:    []*ZipArtifactFile{x2},
+			diff: zipDiff{[][2]*ZipArtifactFile{{x0, x2}}, nil, nil},
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			diff := diffTargetFilesLists(test.a, test.b)
+
+			if !reflect.DeepEqual(diff, test.diff) {
+
+				t.Errorf("diffTargetFilesLists = %v, %v, %v", diff.modified, diff.onlyInA, diff.onlyInB)
+				t.Errorf("                  want %v, %v, %v", test.diff.modified, test.diff.onlyInA, test.diff.onlyInB)
+			}
+		})
+	}
+}

cmd/diff_target_files/diff_target_files.go

@@ -0,0 +1,82 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+)
+
+var (
+	whitelists     = newMultiString("whitelist", "whitelist patterns in the form <pattern>[:<regex of line to ignore>]")
+	whitelistFiles = newMultiString("whitelist_file", "files containing whitelist definitions")
+
+	filters = newMultiString("filter", "filter patterns to apply to files in target-files.zip before comparing")
+)
+
+func newMultiString(name, usage string) *multiString {
+	var f multiString
+	flag.Var(&f, name, usage)
+	return &f
+}
+
+type multiString []string
+
+func (ms *multiString) String() string     { return strings.Join(*ms, ", ") }
+func (ms *multiString) Set(s string) error { *ms = append(*ms, s); return nil }
+
+func main() {
+	flag.Parse()
+
+	if flag.NArg() != 2 {
+		fmt.Fprintf(os.Stderr, "Error, exactly two arguments are required\n")
+		os.Exit(1)
+	}
+
+	whitelists, err := parseWhitelists(*whitelists, *whitelistFiles)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error parsing whitelists: %v\n", err)
+		os.Exit(1)
+	}
+
+	priZip, err := NewLocalZipArtifact(flag.Arg(0))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error opening zip file %v: %v\n", flag.Arg(0), err)
+		os.Exit(1)
+	}
+	defer priZip.Close()
+
+	refZip, err := NewLocalZipArtifact(flag.Arg(1))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error opening zip file %v: %v\n", flag.Arg(1), err)
+		os.Exit(1)
+	}
+	defer refZip.Close()
+
+	diff, err := compareTargetFiles(priZip, refZip, targetFilesPattern, whitelists, *filters)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error comparing zip files: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Print(diff.String())
+
+	if len(diff.modified) > 0 || len(diff.onlyInA) > 0 || len(diff.onlyInB) > 0 {
+		fmt.Fprintln(os.Stderr, "differences found")
+		os.Exit(1)
+	}
+}

cmd/diff_target_files/glob.go

@@ -0,0 +1,81 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+)
+
+// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
+// recursive globs (**).
+func Match(pattern, name string) (bool, error) {
+	if filepath.Base(pattern) == "**" {
+		return false, errors.New("pattern has '**' as last path element")
+	}
+
+	patternDir := pattern[len(pattern)-1] == '/'
+	nameDir := name[len(name)-1] == '/'
+
+	if patternDir != nameDir {
+		return false, nil
+	}
+
+	if nameDir {
+		name = name[:len(name)-1]
+		pattern = pattern[:len(pattern)-1]
+	}
+
+	for {
+		var patternFile, nameFile string
+		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)
+
+		if patternFile == "**" {
+			if strings.Contains(pattern, "**") {
+				return false, errors.New("pattern contains multiple '**'")
+			}
+			// Test if the any prefix of name matches the part of the pattern before **
+			for {
+				if name == "." || name == "/" {
+					return name == pattern, nil
+				}
+				if match, err := filepath.Match(pattern, name); err != nil {
+					return false, err
+				} else if match {
+					return true, nil
+				}
+				name = filepath.Dir(name)
+			}
+		} else if strings.Contains(patternFile, "**") {
+			return false, errors.New("pattern contains other characters between '**' and path separator")
+		}
+
+		name, nameFile = filepath.Dir(name), filepath.Base(name)
+
+		if nameFile == "." && patternFile == "." {
+			return true, nil
+		} else if nameFile == "/" && patternFile == "/" {
+			return true, nil
+		} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
+			return false, nil
+		}
+
+		match, err := filepath.Match(patternFile, nameFile)
+		if err != nil || !match {
+			return match, err
+		}
+	}
+}

cmd/diff_target_files/glob_test.go

@@ -0,0 +1,158 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+)
+
+func TestMatch(t *testing.T) {
+	testCases := []struct {
+		pattern, name string
+		match         bool
+	}{
+		{"a/*", "b/", false},
+		{"a/*", "b/a", false},
+		{"a/*", "b/b/", false},
+		{"a/*", "b/b/c", false},
+		{"a/**/*", "b/", false},
+		{"a/**/*", "b/a", false},
+		{"a/**/*", "b/b/", false},
+		{"a/**/*", "b/b/c", false},
+
+		{"a/*", "a/", false},
+		{"a/*", "a/a", true},
+		{"a/*", "a/b/", false},
+		{"a/*", "a/b/c", false},
+
+		{"a/*/", "a/", false},
+		{"a/*/", "a/a", false},
+		{"a/*/", "a/b/", true},
+		{"a/*/", "a/b/c", false},
+
+		{"a/**/*", "a/", false},
+		{"a/**/*", "a/a", true},
+		{"a/**/*", "a/b/", false},
+		{"a/**/*", "a/b/c", true},
+
+		{"a/**/*/", "a/", false},
+		{"a/**/*/", "a/a", false},
+		{"a/**/*/", "a/b/", true},
+		{"a/**/*/", "a/b/c", false},
+
+		{"**/*", "a/", false},
+		{"**/*", "a/a", true},
+		{"**/*", "a/b/", false},
+		{"**/*", "a/b/c", true},
+
+		{"**/*/", "a/", true},
+		{"**/*/", "a/a", false},
+		{"**/*/", "a/b/", true},
+		{"**/*/", "a/b/c", false},
+
+		{`a/\*\*/\*`, `a/**/*`, true},
+		{`a/\*\*/\*`, `a/a/*`, false},
+		{`a/\*\*/\*`, `a/**/a`, false},
+		{`a/\*\*/\*`, `a/a/a`, false},
+
+		{`a/**/\*`, `a/**/*`, true},
+		{`a/**/\*`, `a/a/*`, true},
+		{`a/**/\*`, `a/**/a`, false},
+		{`a/**/\*`, `a/a/a`, false},
+
+		{`a/\*\*/*`, `a/**/*`, true},
+		{`a/\*\*/*`, `a/a/*`, false},
+		{`a/\*\*/*`, `a/**/a`, true},
+		{`a/\*\*/*`, `a/a/a`, false},
+
+		{`*/**/a`, `a/a/a`, true},
+		{`*/**/a`, `*/a/a`, true},
+		{`*/**/a`, `a/**/a`, true},
+		{`*/**/a`, `*/**/a`, true},
+
+		{`\*/\*\*/a`, `a/a/a`, false},
+		{`\*/\*\*/a`, `*/a/a`, false},
+		{`\*/\*\*/a`, `a/**/a`, false},
+		{`\*/\*\*/a`, `*/**/a`, true},
+
+		{`a/?`, `a/?`, true},
+		{`a/?`, `a/a`, true},
+		{`a/\?`, `a/?`, true},
+		{`a/\?`, `a/a`, false},
+
+		{`a/?`, `a/?`, true},
+		{`a/?`, `a/a`, true},
+		{`a/\?`, `a/?`, true},
+		{`a/\?`, `a/a`, false},
+
+		{`a/[a-c]`, `a/b`, true},
+		{`a/[abc]`, `a/b`, true},
+
+		{`a/\[abc]`, `a/b`, false},
+		{`a/\[abc]`, `a/[abc]`, true},
+
+		{`a/\[abc\]`, `a/b`, false},
+		{`a/\[abc\]`, `a/[abc]`, true},
+
+		{`a/?`, `a/?`, true},
+		{`a/?`, `a/a`, true},
+		{`a/\?`, `a/?`, true},
+		{`a/\?`, `a/a`, false},
+
+		{"/a/*", "/a/", false},
+		{"/a/*", "/a/a", true},
+		{"/a/*", "/a/b/", false},
+		{"/a/*", "/a/b/c", false},
+
+		{"/a/*/", "/a/", false},
+		{"/a/*/", "/a/a", false},
+		{"/a/*/", "/a/b/", true},
+		{"/a/*/", "/a/b/c", false},
+
+		{"/a/**/*", "/a/", false},
+		{"/a/**/*", "/a/a", true},
+		{"/a/**/*", "/a/b/", false},
+		{"/a/**/*", "/a/b/c", true},
+
+		{"/**/*", "/a/", false},
+		{"/**/*", "/a/a", true},
+		{"/**/*", "/a/b/", false},
+		{"/**/*", "/a/b/c", true},
+
+		{"/**/*/", "/a/", true},
+		{"/**/*/", "/a/a", false},
+		{"/**/*/", "/a/b/", true},
+		{"/**/*/", "/a/b/c", false},
+
+		{`a`, `/a`, false},
+		{`/a`, `a`, false},
+		{`*`, `/a`, false},
+		{`/*`, `a`, false},
+		{`**/*`, `/a`, false},
+		{`/**/*`, `a`, false},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.pattern+","+test.name, func(t *testing.T) {
+			match, err := Match(test.pattern, test.name)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if match != test.match {
+				t.Errorf("want: %v, got %v", test.match, match)
+			}
+		})
+	}
+}

cmd/diff_target_files/known_nondeterminism.whitelist

@@ -0,0 +1,10 @@
+// List of files that are known to be non-deterministic, along with the
+// bug number to tracking fixing the non-determinism.
+[
+  {
+    "Paths": [
+       // b/120039850
+      "system/framework/oat/*/services.art"
+    ]
+  }
+]

cmd/diff_target_files/props.whitelist

@@ -0,0 +1,18 @@
+[
+  // Ignore date, version and hostname properties in build.prop and prop.default files.
+  {
+    "Paths": [
+      "**/build.prop",
+      "**/prop.default"
+    ],
+    "IgnoreMatchingLines": [
+      "ro\\..*build\\.date=.*",
+      "ro\\..*build\\.date\\.utc=.*",
+      "ro\\..*build\\.version\\.incremental=.*",
+      "ro\\..*build\\.fingerprint=.*",
+      "ro\\.build\\.display\\.id=.*",
+      "ro\\.build\\.description=.*",
+      "ro\\.build\\.host=.*"
+    ]
+  }
+]

cmd/diff_target_files/target_files.go

@@ -0,0 +1,86 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+const targetFilesPattern = "*-target_files-*.zip"
+
+var targetZipPartitions = []string{
+	"BOOT/RAMDISK/",
+	"BOOT/",
+	"DATA/",
+	"ODM/",
+	"OEM/",
+	"PRODUCT/",
+	"PRODUCT_SERVICES/",
+	"ROOT/",
+	"SYSTEM/",
+	"SYSTEM_OTHER/",
+	"VENDOR/",
+}
+
+var targetZipFilter = []string{
+	"IMAGES/",
+	"OTA/",
+	"META/",
+	"PREBUILT_IMAGES/",
+	"RADIO/",
+}
+
+func filterTargetZipFiles(files []*ZipArtifactFile, artifact string, patterns []string) ([]*ZipArtifactFile, error) {
+	var ret []*ZipArtifactFile
+outer:
+	for _, f := range files {
+		if f.FileInfo().IsDir() {
+			continue
+		}
+
+		if artifact == targetFilesPattern {
+			found := false
+			for _, p := range targetZipPartitions {
+				if strings.HasPrefix(f.Name, p) {
+					f.Name = strings.ToLower(p) + strings.TrimPrefix(f.Name, p)
+					found = true
+				}
+			}
+			for _, filter := range targetZipFilter {
+				if strings.HasPrefix(f.Name, filter) {
+					continue outer
+				}
+			}
+
+			if !found {
+				return nil, fmt.Errorf("unmatched prefix for %s", f.Name)
+			}
+		}
+
+		if patterns != nil {
+			for _, pattern := range patterns {
+				match, _ := Match(pattern, f.Name)
+				if match {
+					ret = append(ret, f)
+				}
+			}
+		} else {
+			ret = append(ret, f)
+		}
+	}
+
+	return ret, nil
+}

cmd/diff_target_files/whitelist.go

@@ -0,0 +1,251 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"io"
+	"os"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+type jsonWhitelist struct {
+	Paths               []string
+	IgnoreMatchingLines []string
+}
+
+type whitelist struct {
+	path                string
+	ignoreMatchingLines []string
+}
+
+func parseWhitelists(whitelists []string, whitelistFiles []string) ([]whitelist, error) {
+	var ret []whitelist
+
+	add := func(path string, ignoreMatchingLines []string) {
+		for _, x := range ret {
+			if x.path == path {
+				x.ignoreMatchingLines = append(x.ignoreMatchingLines, ignoreMatchingLines...)
+				return
+			}
+		}
+
+		ret = append(ret, whitelist{
+			path:                path,
+			ignoreMatchingLines: ignoreMatchingLines,
+		})
+	}
+
+	for _, file := range whitelistFiles {
+		newWhitelists, err := parseWhitelistFile(file)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, w := range newWhitelists {
+			add(w.path, w.ignoreMatchingLines)
+		}
+	}
+
+	for _, s := range whitelists {
+		colon := strings.IndexRune(s, ':')
+		var ignoreMatchingLines []string
+		if colon >= 0 {
+			ignoreMatchingLines = []string{s[colon+1:]}
+		}
+		add(s, ignoreMatchingLines)
+	}
+
+	return ret, nil
+}
+
+func parseWhitelistFile(file string) ([]whitelist, error) {
+	r, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	d := json.NewDecoder(newJSONCommentStripper(r))
+
+	var jsonWhitelists []jsonWhitelist
+
+	err = d.Decode(&jsonWhitelists)
+
+	var whitelists []whitelist
+	for _, w := range jsonWhitelists {
+		for _, p := range w.Paths {
+			whitelists = append(whitelists, whitelist{
+				path:                p,
+				ignoreMatchingLines: w.IgnoreMatchingLines,
+			})
+		}
+	}
+
+	return whitelists, err
+}
+
+func filterModifiedPaths(l [][2]*ZipArtifactFile, whitelists []whitelist) ([][2]*ZipArtifactFile, error) {
+outer:
+	for i := 0; i < len(l); i++ {
+		for _, w := range whitelists {
+			if match, err := Match(w.path, l[i][0].Name); err != nil {
+				return l, err
+			} else if match {
+				if match, err := diffIgnoringMatchingLines(l[i][0], l[i][1], w.ignoreMatchingLines); err != nil {
+					return l, err
+				} else if match || len(w.ignoreMatchingLines) == 0 {
+					l = append(l[:i], l[i+1:]...)
+					i--
+				}
+				continue outer
+			}
+		}
+	}
+
+	if len(l) == 0 {
+		l = nil
+	}
+
+	return l, nil
+}
+
+func filterNewPaths(l []*ZipArtifactFile, whitelists []whitelist) ([]*ZipArtifactFile, error) {
+outer:
+	for i := 0; i < len(l); i++ {
+		for _, w := range whitelists {
+			if match, err := Match(w.path, l[i].Name); err != nil {
+				return l, err
+			} else if match && len(w.ignoreMatchingLines) == 0 {
+				l = append(l[:i], l[i+1:]...)
+				i--
+			}
+			continue outer
+		}
+	}
+
+	if len(l) == 0 {
+		l = nil
+	}
+
+	return l, nil
+}
+
+func diffIgnoringMatchingLines(a *ZipArtifactFile, b *ZipArtifactFile, ignoreMatchingLines []string) (match bool, err error) {
+	lineMatchesIgnores := func(b []byte) (bool, error) {
+		for _, m := range ignoreMatchingLines {
+			if match, err := regexp.Match(m, b); err != nil {
+				return false, err
+			} else if match {
+				return match, nil
+			}
+		}
+		return false, nil
+	}
+
+	filter := func(z *ZipArtifactFile) ([]byte, error) {
+		var ret []byte
+
+		r, err := z.Open()
+		if err != nil {
+			return nil, err
+		}
+		s := bufio.NewScanner(r)
+
+		for s.Scan() {
+			if match, err := lineMatchesIgnores(s.Bytes()); err != nil {
+				return nil, err
+			} else if !match {
+				ret = append(ret, "\n"...)
+				ret = append(ret, s.Bytes()...)
+			}
+		}
+
+		return ret, nil
+	}
+
+	bufA, err := filter(a)
+	if err != nil {
+		return false, err
+	}
+	bufB, err := filter(b)
+	if err != nil {
+		return false, err
+	}
+
+	return bytes.Compare(bufA, bufB) == 0, nil
+}
+
+func applyWhitelists(diff zipDiff, whitelists []whitelist) (zipDiff, error) {
+	var err error
+
+	diff.modified, err = filterModifiedPaths(diff.modified, whitelists)
+	if err != nil {
+		return diff, err
+	}
+	diff.onlyInA, err = filterNewPaths(diff.onlyInA, whitelists)
+	if err != nil {
+		return diff, err
+	}
+	diff.onlyInB, err = filterNewPaths(diff.onlyInB, whitelists)
+	if err != nil {
+		return diff, err
+	}
+
+	return diff, nil
+}
+
+func newJSONCommentStripper(r io.Reader) *jsonCommentStripper {
+	return &jsonCommentStripper{
+		r: bufio.NewReader(r),
+	}
+}
+
+type jsonCommentStripper struct {
+	r   *bufio.Reader
+	b   []byte
+	err error
+}
+
+func (j *jsonCommentStripper) Read(buf []byte) (int, error) {
+	for len(j.b) == 0 {
+		if j.err != nil {
+			return 0, j.err
+		}
+
+		j.b, j.err = j.r.ReadBytes('\n')
+
+		if isComment(j.b) {
+			j.b = nil
+		}
+	}
+
+	n := copy(buf, j.b)
+	j.b = j.b[n:]
+	return n, nil
+}
+
+var commentPrefix = []byte("//")
+
+func isComment(b []byte) bool {
+	for len(b) > 0 && unicode.IsSpace(rune(b[0])) {
+		b = b[1:]
+	}
+	return bytes.HasPrefix(b, commentPrefix)
+}

cmd/diff_target_files/whitelist_test.go

@@ -0,0 +1,126 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+func bytesToZipArtifactFile(name string, data []byte) *ZipArtifactFile {
+	buf := &bytes.Buffer{}
+	w := zip.NewWriter(buf)
+	f, err := w.Create(name)
+	if err != nil {
+		panic(err)
+	}
+	_, err = f.Write(data)
+	if err != nil {
+		panic(err)
+	}
+
+	w.Close()
+
+	r, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+	if err != nil {
+		panic(err)
+	}
+
+	return &ZipArtifactFile{r.File[0]}
+}
+
+var f1a = bytesToZipArtifactFile("dir/f1", []byte(`
+a
+foo: bar
+c
+`))
+
+var f1b = bytesToZipArtifactFile("dir/f1", []byte(`
+a
+foo: baz
+c
+`))
+
+var f2 = bytesToZipArtifactFile("dir/f2", nil)
+
+func Test_applyWhitelists(t *testing.T) {
+	type args struct {
+		diff       zipDiff
+		whitelists []whitelist
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    zipDiff
+		wantErr bool
+	}{
+		{
+			name: "simple",
+			args: args{
+				diff: zipDiff{
+					onlyInA: []*ZipArtifactFile{f1a, f2},
+				},
+				whitelists: []whitelist{{path: "dir/f1"}},
+			},
+			want: zipDiff{
+				onlyInA: []*ZipArtifactFile{f2},
+			},
+		},
+		{
+			name: "glob",
+			args: args{
+				diff: zipDiff{
+					onlyInA: []*ZipArtifactFile{f1a, f2},
+				},
+				whitelists: []whitelist{{path: "dir/*"}},
+			},
+			want: zipDiff{},
+		},
+		{
+			name: "modified",
+			args: args{
+				diff: zipDiff{
+					modified: [][2]*ZipArtifactFile{{f1a, f1b}},
+				},
+				whitelists: []whitelist{{path: "dir/*"}},
+			},
+			want: zipDiff{},
+		},
+		{
+			name: "matching lines",
+			args: args{
+				diff: zipDiff{
+					modified: [][2]*ZipArtifactFile{{f1a, f1b}},
+				},
+				whitelists: []whitelist{{path: "dir/*", ignoreMatchingLines: []string{"foo: .*"}}},
+			},
+			want: zipDiff{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := applyWhitelists(tt.args.diff, tt.args.whitelists)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("applyWhitelists() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("applyWhitelists() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/diff_target_files/zip_artifact.go

@@ -0,0 +1,174 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// ZipArtifact represents a zip file that may be local or remote.
+type ZipArtifact interface {
+	// Files returns the list of files contained in the zip file.
+	Files() ([]*ZipArtifactFile, error)
+
+	// Close closes the zip file artifact.
+	Close()
+}
+
+// localZipArtifact is a handle to a local zip file artifact.
+type localZipArtifact struct {
+	zr    *zip.ReadCloser
+	files []*ZipArtifactFile
+}
+
+// NewLocalZipArtifact returns a ZipArtifact for a local zip file..
+func NewLocalZipArtifact(name string) (ZipArtifact, error) {
+	zr, err := zip.OpenReader(name)
+	if err != nil {
+		return nil, err
+	}
+
+	var files []*ZipArtifactFile
+	for _, zf := range zr.File {
+		files = append(files, &ZipArtifactFile{zf})
+	}
+
+	return &localZipArtifact{
+		zr:    zr,
+		files: files,
+	}, nil
+}
+
+// Files returns the list of files contained in the local zip file artifact.
+func (z *localZipArtifact) Files() ([]*ZipArtifactFile, error) {
+	return z.files, nil
+}
+
+// Close closes the buffered reader of the local zip file artifact.
+func (z *localZipArtifact) Close() {
+	z.zr.Close()
+}
+
+// ZipArtifactFile contains a zip.File handle to the data inside the remote *-target_files-*.zip
+// build artifact.
+type ZipArtifactFile struct {
+	*zip.File
+}
+
+// Extract begins extract a file from inside a ZipArtifact.  It returns an
+// ExtractedZipArtifactFile handle.
+func (zf *ZipArtifactFile) Extract(ctx context.Context, dir string,
+	limiter chan bool) *ExtractedZipArtifactFile {
+
+	d := &ExtractedZipArtifactFile{
+		initCh: make(chan struct{}),
+	}
+
+	go func() {
+		defer close(d.initCh)
+		limiter <- true
+		defer func() { <-limiter }()
+
+		zr, err := zf.Open()
+		if err != nil {
+			d.err = err
+			return
+		}
+		defer zr.Close()
+
+		crc := crc32.NewIEEE()
+		r := io.TeeReader(zr, crc)
+
+		if filepath.Clean(zf.Name) != zf.Name {
+			d.err = fmt.Errorf("invalid filename %q", zf.Name)
+			return
+		}
+		path := filepath.Join(dir, zf.Name)
+
+		err = os.MkdirAll(filepath.Dir(path), 0777)
+		if err != nil {
+			d.err = err
+			return
+		}
+
+		err = os.Remove(path)
+		if err != nil && !os.IsNotExist(err) {
+			d.err = err
+			return
+		}
+
+		if zf.Mode().IsRegular() {
+			w, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, zf.Mode())
+			if err != nil {
+				d.err = err
+				return
+			}
+			defer w.Close()
+
+			_, err = io.Copy(w, r)
+			if err != nil {
+				d.err = err
+				return
+			}
+		} else if zf.Mode()&os.ModeSymlink != 0 {
+			target, err := ioutil.ReadAll(r)
+			if err != nil {
+				d.err = err
+				return
+			}
+
+			err = os.Symlink(string(target), path)
+			if err != nil {
+				d.err = err
+				return
+			}
+		} else {
+			d.err = fmt.Errorf("unknown mode %q", zf.Mode())
+			return
+		}
+
+		if crc.Sum32() != zf.CRC32 {
+			d.err = fmt.Errorf("crc mismatch for %v", zf.Name)
+			return
+		}
+
+		d.path = path
+	}()
+
+	return d
+}
+
+// ExtractedZipArtifactFile is a handle to a downloaded file from a remoteZipArtifact.  The download
+// may still be in progress, and will be complete with Path() returns.
+type ExtractedZipArtifactFile struct {
+	initCh chan struct{}
+	err    error
+
+	path string
+}
+
+// Path returns the path to the downloaded file and any errors that occurred during the download.
+// It will block until the download is complete.
+func (d *ExtractedZipArtifactFile) Path() (string, error) {
+	<-d.initCh
+	return d.path, d.err
+}