diff --git a/cmd/zip2zip/Android.bp b/cmd/zip2zip/Android.bp index 64202193a..68d8bc72a 100644 --- a/cmd/zip2zip/Android.bp +++ b/cmd/zip2zip/Android.bp @@ -15,8 +15,9 @@ blueprint_go_binary { name: "zip2zip", deps: [ - "android-archive-zip", - "soong-jar", + "android-archive-zip", + "blueprint-pathtools", + "soong-jar", ], srcs: [ "zip2zip.go", diff --git a/cmd/zip2zip/zip2zip.go b/cmd/zip2zip/zip2zip.go index f48d458b8..e8ea9b949 100644 --- a/cmd/zip2zip/zip2zip.go +++ b/cmd/zip2zip/zip2zip.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "github.com/google/blueprint/pathtools" + "android/soong/jar" "android/soong/third_party/zip" ) @@ -36,8 +38,14 @@ var ( setTime = flag.Bool("t", false, "set timestamps to 2009-01-01 00:00:00") staticTime = time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC) + + excludes excludeArgs ) +func init() { + flag.Var(&excludes, "x", "exclude a filespec from the output") +} + func main() { flag.Usage = func() { fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [-s|-j] [-t] [filespec]...") @@ -45,15 +53,14 @@ func main() { fmt.Fprintln(os.Stderr, " filespec:") fmt.Fprintln(os.Stderr, " ") fmt.Fprintln(os.Stderr, " :") - fmt.Fprintln(os.Stderr, " :/") + fmt.Fprintln(os.Stderr, " [:]") fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, " uses the rules at https://golang.org/pkg/path/filepath/#Match") - fmt.Fprintln(os.Stderr, "As a special exception, '**' is supported to specify all files in the input zip.") + fmt.Fprintln(os.Stderr, " uses the rules at https://godoc.org/github.com/google/blueprint/pathtools/#Match") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to") fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments.") fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, "If no filepsec is provided all files are copied (equivalent to '**').") + fmt.Fprintln(os.Stderr, "If no filepsec is provided all files and directories are copied.") } flag.Parse() @@ -85,7 +92,9 @@ func main() { } }() - if err := zip2zip(&reader.Reader, writer, *sortGlobs, *sortJava, *setTime, flag.Args()); err != nil { + if err := zip2zip(&reader.Reader, writer, *sortGlobs, *sortJava, *setTime, + flag.Args(), excludes); err != nil { + log.Fatal(err) } } @@ -95,91 +104,126 @@ type pair struct { newName string } -func zip2zip(reader *zip.Reader, writer *zip.Writer, sortGlobs, sortJava, setTime bool, args []string) error { - if len(args) == 0 { - // If no filespec is provided, default to copying everything - args = []string{"**"} - } - for _, arg := range args { - var input string - var output string +func zip2zip(reader *zip.Reader, writer *zip.Writer, sortOutput, sortJava, setTime bool, + includes []string, excludes []string) error { + matches := []pair{} + + sortMatches := func(matches []pair) { + if sortJava { + sort.SliceStable(matches, func(i, j int) bool { + return jar.EntryNamesLess(matches[i].newName, matches[j].newName) + }) + } else if sortOutput { + sort.SliceStable(matches, func(i, j int) bool { + return matches[i].newName < matches[j].newName + }) + } + } + + for _, include := range includes { // Reserve escaping for future implementation, so make sure no // one is using \ and expecting a certain behavior. - if strings.Contains(arg, "\\") { + if strings.Contains(include, "\\") { return fmt.Errorf("\\ characters are not currently supported") } - args := strings.SplitN(arg, ":", 2) - input = args[0] - if len(args) == 2 { - output = args[1] - } + input, output := includeSplit(include) - matches := []pair{} - if strings.IndexAny(input, "*?[") >= 0 { - matchAll := input == "**" - if !matchAll && strings.Contains(input, "**") { - return fmt.Errorf("** is only supported on its own, not with other characters") - } + var includeMatches []pair - for _, file := range reader.File { - match := matchAll - - if !match { - var err error - match, err = filepath.Match(input, file.Name) - if err != nil { - return err - } - } - - if match { - var newName string - if output == "" { - newName = file.Name - } else { + for _, file := range reader.File { + var newName string + if match, err := pathtools.Match(input, file.Name); err != nil { + return err + } else if match { + if output == "" { + newName = file.Name + } else { + if pathtools.IsGlob(input) { + // If the input is a glob then the output is a directory. _, name := filepath.Split(file.Name) newName = filepath.Join(output, name) + } else { + // Otherwise it is a file. + newName = output } - matches = append(matches, pair{file, newName}) - } - } - - if sortJava { - jarSort(matches) - } else if sortGlobs { - sort.SliceStable(matches, func(i, j int) bool { - return matches[i].newName < matches[j].newName - }) - } - } else { - if output == "" { - output = input - } - for _, file := range reader.File { - if input == file.Name { - matches = append(matches, pair{file, output}) - break } + includeMatches = append(includeMatches, pair{file, newName}) } } - for _, match := range matches { - if setTime { - match.File.SetModTime(staticTime) - } - if err := writer.CopyFrom(match.File, match.newName); err != nil { + sortMatches(includeMatches) + matches = append(matches, includeMatches...) + } + + if len(includes) == 0 { + // implicitly match everything + for _, file := range reader.File { + matches = append(matches, pair{file, file.Name}) + } + sortMatches(matches) + } + + var matchesAfterExcludes []pair + seen := make(map[string]*zip.File) + + for _, match := range matches { + // Filter out matches whose original file name matches an exclude filter + excluded := false + for _, exclude := range excludes { + if excludeMatch, err := pathtools.Match(exclude, match.File.Name); err != nil { return err + } else if excludeMatch { + excluded = true + break } } + + if excluded { + continue + } + + // Check for duplicate output names, ignoring ones that come from the same input zip entry. + if prev, exists := seen[match.newName]; exists { + if prev != match.File { + return fmt.Errorf("multiple entries for %q with different contents", match.newName) + } + continue + } + seen[match.newName] = match.File + + matchesAfterExcludes = append(matchesAfterExcludes, match) + } + + for _, match := range matchesAfterExcludes { + if setTime { + match.File.SetModTime(staticTime) + } + if err := writer.CopyFrom(match.File, match.newName); err != nil { + return err + } } return nil } -func jarSort(files []pair) { - sort.SliceStable(files, func(i, j int) bool { - return jar.EntryNamesLess(files[i].newName, files[j].newName) - }) +func includeSplit(s string) (string, string) { + split := strings.SplitN(s, ":", 2) + if len(split) == 2 { + return split[0], split[1] + } else { + return split[0], "" + } +} + +type excludeArgs []string + +func (e *excludeArgs) String() string { + return strings.Join(*e, " ") +} + +func (e *excludeArgs) Set(s string) error { + *e = append(*e, s) + return nil } diff --git a/cmd/zip2zip/zip2zip_test.go b/cmd/zip2zip/zip2zip_test.go index 53c8ce2bd..212ab28e5 100644 --- a/cmd/zip2zip/zip2zip_test.go +++ b/cmd/zip2zip/zip2zip_test.go @@ -30,6 +30,7 @@ var testCases = []struct { sortGlobs bool sortJava bool args []string + excludes []string outputFiles []string err error @@ -41,13 +42,6 @@ var testCases = []struct { err: fmt.Errorf("\\ characters are not currently supported"), }, - { - name: "unsupported **", - - args: []string{"a/**:b"}, - - err: fmt.Errorf("** is only supported on its own, not with other characters"), - }, { // This is modelled after the update package build rules in build/make/core/Makefile name: "filter globs", @@ -95,16 +89,19 @@ var testCases = []struct { name: "sort all", inputFiles: []string{ + "RADIO/", "RADIO/a", + "IMAGES/", "IMAGES/system.img", "IMAGES/b.txt", "IMAGES/recovery.img", "IMAGES/vendor.img", + "OTA/", "OTA/b", "OTA/android-info.txt", }, sortGlobs: true, - args: []string{"**"}, + args: []string{"**/*"}, outputFiles: []string{ "IMAGES/b.txt", @@ -120,11 +117,14 @@ var testCases = []struct { name: "sort all implicit", inputFiles: []string{ + "RADIO/", "RADIO/a", + "IMAGES/", "IMAGES/system.img", "IMAGES/b.txt", "IMAGES/recovery.img", "IMAGES/vendor.img", + "OTA/", "OTA/b", "OTA/android-info.txt", }, @@ -132,12 +132,15 @@ var testCases = []struct { args: nil, outputFiles: []string{ + "IMAGES/", "IMAGES/b.txt", "IMAGES/recovery.img", "IMAGES/system.img", "IMAGES/vendor.img", + "OTA/", "OTA/android-info.txt", "OTA/b", + "RADIO/", "RADIO/a", }, }, @@ -177,7 +180,7 @@ var testCases = []struct { "b", "a", }, - args: []string{"a:a2", "**"}, + args: []string{"a:a2", "**/*"}, outputFiles: []string{ "a2", @@ -185,6 +188,69 @@ var testCases = []struct { "a", }, }, + { + name: "multiple matches", + + inputFiles: []string{ + "a/a", + }, + args: []string{"a/a", "a/*"}, + + outputFiles: []string{ + "a/a", + }, + }, + { + name: "multiple conflicting matches", + + inputFiles: []string{ + "a/a", + "a/b", + }, + args: []string{"a/b:a/a", "a/*"}, + + err: fmt.Errorf(`multiple entries for "a/a" with different contents`), + }, + { + name: "excludes", + + inputFiles: []string{ + "a/a", + "a/b", + }, + args: nil, + excludes: []string{"a/a"}, + + outputFiles: []string{ + "a/b", + }, + }, + { + name: "excludes with include", + + inputFiles: []string{ + "a/a", + "a/b", + }, + args: []string{"a/*"}, + excludes: []string{"a/a"}, + + outputFiles: []string{ + "a/b", + }, + }, + { + name: "excludes with glob", + + inputFiles: []string{ + "a/a", + "a/b", + }, + args: []string{"a/*"}, + excludes: []string{"a/*"}, + + outputFiles: nil, + }, } func errorString(e error) string { @@ -216,7 +282,7 @@ func TestZip2Zip(t *testing.T) { } outputWriter := zip.NewWriter(outputBuf) - err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, testCase.sortJava, false, testCase.args) + err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, testCase.sortJava, false, testCase.args, testCase.excludes) if errorString(testCase.err) != errorString(err) { t.Fatalf("Unexpected error:\n got: %q\nwant: %q", errorString(err), errorString(testCase.err)) }