tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support exclusions and Blueprint-style ** globs in zip2zip

Browse source 
am: f3831d0e08

Change-Id: I7968a56495aa661437180e43a27ae36029216ce9
This commit is contained in:
Colin Cross 2017-11-28 20:54:51 +00:00 committed by android-build-merger
commit 5579297d88
3 changed files with 192 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmd/zip2zip/Android.bp
View file

@ -15,8 +15,9 @@
blueprint_go_binary {
name: "zip2zip",
deps: [
"android-archive-zip",
"soong-jar",
"android-archive-zip",
"blueprint-pathtools",
"soong-jar",
],
srcs: [
"zip2zip.go",

182
cmd/zip2zip/zip2zip.go
View file

@ -24,6 +24,8 @@ import (
"strings"
"time"
"github.com/google/blueprint/pathtools"
"android/soong/jar"
"android/soong/third_party/zip"
)
@ -36,8 +38,14 @@ var (
setTime = flag.Bool("t", false, "set timestamps to 2009-01-01 00:00:00")
staticTime = time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
excludes excludeArgs
)
func init() {
flag.Var(&excludes, "x", "exclude a filespec from the output")
}
func main() {
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [-s|-j] [-t] [filespec]...")
@ -45,15 +53,14 @@ func main() {
fmt.Fprintln(os.Stderr, " filespec:")
fmt.Fprintln(os.Stderr, " <name>")
fmt.Fprintln(os.Stderr, " <in_name>:<out_name>")
fmt.Fprintln(os.Stderr, " <glob>:<out_dir>/")
fmt.Fprintln(os.Stderr, " <glob>[:<out_dir>]")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://golang.org/pkg/path/filepath/#Match")
fmt.Fprintln(os.Stderr, "As a special exception, '**' is supported to specify all files in the input zip.")
fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://godoc.org/github.com/google/blueprint/pathtools/#Match")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments.")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "If no filepsec is provided all files are copied (equivalent to '**').")
fmt.Fprintln(os.Stderr, "If no filepsec is provided all files and directories are copied.")
}
flag.Parse()
@ -85,7 +92,9 @@ func main() {
}
}()
if err := zip2zip(&reader.Reader, writer, *sortGlobs, *sortJava, *setTime, flag.Args()); err != nil {
if err := zip2zip(&reader.Reader, writer, *sortGlobs, *sortJava, *setTime,
flag.Args(), excludes); err != nil {
log.Fatal(err)
}
}
@ -95,91 +104,126 @@ type pair struct {
newName string
}
func zip2zip(reader *zip.Reader, writer *zip.Writer, sortGlobs, sortJava, setTime bool, args []string) error {
if len(args) == 0 {
// If no filespec is provided, default to copying everything
args = []string{"**"}
}
for _, arg := range args {
var input string
var output string
func zip2zip(reader *zip.Reader, writer *zip.Writer, sortOutput, sortJava, setTime bool,
includes []string, excludes []string) error {
matches := []pair{}
sortMatches := func(matches []pair) {
if sortJava {
sort.SliceStable(matches, func(i, j int) bool {
return jar.EntryNamesLess(matches[i].newName, matches[j].newName)
})
} else if sortOutput {
sort.SliceStable(matches, func(i, j int) bool {
return matches[i].newName < matches[j].newName
})
}
}
for _, include := range includes {
// Reserve escaping for future implementation, so make sure no
// one is using \ and expecting a certain behavior.
if strings.Contains(arg, "\\") {
if strings.Contains(include, "\\") {
return fmt.Errorf("\\ characters are not currently supported")
}
args := strings.SplitN(arg, ":", 2)
input = args[0]
if len(args) == 2 {
output = args[1]
}
input, output := includeSplit(include)
matches := []pair{}
if strings.IndexAny(input, "*?[") >= 0 {
matchAll := input == "**"
if !matchAll && strings.Contains(input, "**") {
return fmt.Errorf("** is only supported on its own, not with other characters")
}
var includeMatches []pair
for _, file := range reader.File {
match := matchAll
if !match {
var err error
match, err = filepath.Match(input, file.Name)
if err != nil {
return err
}
}
if match {
var newName string
if output == "" {
newName = file.Name
} else {
for _, file := range reader.File {
var newName string
if match, err := pathtools.Match(input, file.Name); err != nil {
return err
} else if match {
if output == "" {
newName = file.Name
} else {
if pathtools.IsGlob(input) {
// If the input is a glob then the output is a directory.
_, name := filepath.Split(file.Name)
newName = filepath.Join(output, name)
} else {
// Otherwise it is a file.
newName = output
}
matches = append(matches, pair{file, newName})
}
}
if sortJava {
jarSort(matches)
} else if sortGlobs {
sort.SliceStable(matches, func(i, j int) bool {
return matches[i].newName < matches[j].newName
})
}
} else {
if output == "" {
output = input
}
for _, file := range reader.File {
if input == file.Name {
matches = append(matches, pair{file, output})
break
}
includeMatches = append(includeMatches, pair{file, newName})
}
}
for _, match := range matches {
if setTime {
match.File.SetModTime(staticTime)
}
if err := writer.CopyFrom(match.File, match.newName); err != nil {
sortMatches(includeMatches)
matches = append(matches, includeMatches...)
}
if len(includes) == 0 {
// implicitly match everything
for _, file := range reader.File {
matches = append(matches, pair{file, file.Name})
}
sortMatches(matches)
}
var matchesAfterExcludes []pair
seen := make(map[string]*zip.File)
for _, match := range matches {
// Filter out matches whose original file name matches an exclude filter
excluded := false
for _, exclude := range excludes {
if excludeMatch, err := pathtools.Match(exclude, match.File.Name); err != nil {
return err
} else if excludeMatch {
excluded = true
break
}
}
if excluded {
continue
}
// Check for duplicate output names, ignoring ones that come from the same input zip entry.
if prev, exists := seen[match.newName]; exists {
if prev != match.File {
return fmt.Errorf("multiple entries for %q with different contents", match.newName)
}
continue
}
seen[match.newName] = match.File
matchesAfterExcludes = append(matchesAfterExcludes, match)
}
for _, match := range matchesAfterExcludes {
if setTime {
match.File.SetModTime(staticTime)
}
if err := writer.CopyFrom(match.File, match.newName); err != nil {
return err
}
}
return nil
}
func jarSort(files []pair) {
sort.SliceStable(files, func(i, j int) bool {
return jar.EntryNamesLess(files[i].newName, files[j].newName)
})
func includeSplit(s string) (string, string) {
split := strings.SplitN(s, ":", 2)
if len(split) == 2 {
return split[0], split[1]
} else {
return split[0], ""
}
}
type excludeArgs []string
func (e *excludeArgs) String() string {
return strings.Join(*e, " ")
}
func (e *excludeArgs) Set(s string) error {
*e = append(*e, s)
return nil
}

86
cmd/zip2zip/zip2zip_test.go
View file

@ -30,6 +30,7 @@ var testCases = []struct {
sortGlobs bool
sortJava bool
args []string
excludes []string
outputFiles []string
err error
@ -41,13 +42,6 @@ var testCases = []struct {
err: fmt.Errorf("\\ characters are not currently supported"),
},
{
name: "unsupported **",
args: []string{"a/**:b"},
err: fmt.Errorf("** is only supported on its own, not with other characters"),
},
{ // This is modelled after the update package build rules in build/make/core/Makefile
name: "filter globs",
@ -95,16 +89,19 @@ var testCases = []struct {
name: "sort all",
inputFiles: []string{
"RADIO/",
"RADIO/a",
"IMAGES/",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
"OTA/",
"OTA/b",
"OTA/android-info.txt",
},
sortGlobs: true,
args: []string{"**"},
args: []string{"**/*"},
outputFiles: []string{
"IMAGES/b.txt",
@ -120,11 +117,14 @@ var testCases = []struct {
name: "sort all implicit",
inputFiles: []string{
"RADIO/",
"RADIO/a",
"IMAGES/",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
"OTA/",
"OTA/b",
"OTA/android-info.txt",
},
@ -132,12 +132,15 @@ var testCases = []struct {
args: nil,
outputFiles: []string{
"IMAGES/",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/system.img",
"IMAGES/vendor.img",
"OTA/",
"OTA/android-info.txt",
"OTA/b",
"RADIO/",
"RADIO/a",
},
},
@ -177,7 +180,7 @@ var testCases = []struct {
"b",
"a",
},
args: []string{"a:a2", "**"},
args: []string{"a:a2", "**/*"},
outputFiles: []string{
"a2",
@ -185,6 +188,69 @@ var testCases = []struct {
"a",
},
},
{
name: "multiple matches",
inputFiles: []string{
"a/a",
},
args: []string{"a/a", "a/*"},
outputFiles: []string{
"a/a",
},
},
{
name: "multiple conflicting matches",
inputFiles: []string{
"a/a",
"a/b",
},
args: []string{"a/b:a/a", "a/*"},
err: fmt.Errorf(`multiple entries for "a/a" with different contents`),
},
{
name: "excludes",
inputFiles: []string{
"a/a",
"a/b",
},
args: nil,
excludes: []string{"a/a"},
outputFiles: []string{
"a/b",
},
},
{
name: "excludes with include",
inputFiles: []string{
"a/a",
"a/b",
},
args: []string{"a/*"},
excludes: []string{"a/a"},
outputFiles: []string{
"a/b",
},
},
{
name: "excludes with glob",
inputFiles: []string{
"a/a",
"a/b",
},
args: []string{"a/*"},
excludes: []string{"a/*"},
outputFiles: nil,
},
}
func errorString(e error) string {
@ -216,7 +282,7 @@ func TestZip2Zip(t *testing.T) {
}
outputWriter := zip.NewWriter(outputBuf)
err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, testCase.sortJava, false, testCase.args)
err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, testCase.sortJava, false, testCase.args, testCase.excludes)
if errorString(testCase.err) != errorString(err) {
t.Fatalf("Unexpected error:\n got: %q\nwant: %q", errorString(err), errorString(testCase.err))
}