437e9470c0
A map iteration was causing a non-deterministic order in output entries. The default behaviour is to preserve the same order of the input zips, which is necessary to ensure a deterministic output for deterministic inputs. Bug: b/322788229 Test: Ran a couple of builds and confirmed no cache misses from the output of merge_zips. Change-Id: I3217e6887ab108d213a290b59da5b33d51b8241f
461 lines
11 KiB
Go
461 lines
11 KiB
Go
// Copyright 2018 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"android/soong/jar"
|
|
"android/soong/third_party/zip"
|
|
)
|
|
|
|
type testZipEntry struct {
|
|
name string
|
|
mode os.FileMode
|
|
data []byte
|
|
method uint16
|
|
timestamp time.Time
|
|
}
|
|
|
|
var (
|
|
A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
|
|
a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
|
|
a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate, jar.DefaultTime}
|
|
a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate, jar.DefaultTime}
|
|
bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
|
|
bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
|
|
bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate, jar.DefaultTime}
|
|
ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
|
|
bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate, jar.DefaultTime}
|
|
bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate, jar.DefaultTime}
|
|
be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate, jar.DefaultTime}
|
|
|
|
withTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime.Add(time.Hour)}
|
|
withoutTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime}
|
|
|
|
service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store, jar.DefaultTime}
|
|
service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate, jar.DefaultTime}
|
|
service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store, jar.DefaultTime}
|
|
service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate, jar.DefaultTime}
|
|
|
|
metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
|
|
manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate, jar.DefaultTime}
|
|
manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate, jar.DefaultTime}
|
|
moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate, jar.DefaultTime}
|
|
)
|
|
|
|
type testInputZip struct {
|
|
name string
|
|
entries []testZipEntry
|
|
reader *zip.Reader
|
|
}
|
|
|
|
func (tiz *testInputZip) Name() string {
|
|
return tiz.name
|
|
}
|
|
|
|
func (tiz *testInputZip) Open() error {
|
|
if tiz.reader == nil {
|
|
tiz.reader = testZipEntriesToZipReader(tiz.entries)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (tiz *testInputZip) Close() error {
|
|
tiz.reader = nil
|
|
return nil
|
|
}
|
|
|
|
func (tiz *testInputZip) Entries() []*zip.File {
|
|
if tiz.reader == nil {
|
|
panic(fmt.Errorf("%s: should be open to get entries", tiz.Name()))
|
|
}
|
|
return tiz.reader.File
|
|
}
|
|
|
|
func (tiz *testInputZip) IsOpen() bool {
|
|
return tiz.reader != nil
|
|
}
|
|
|
|
func TestMergeZips(t *testing.T) {
|
|
testCases := []struct {
|
|
name string
|
|
in [][]testZipEntry
|
|
stripFiles []string
|
|
stripDirs []string
|
|
jar bool
|
|
par bool
|
|
sort bool
|
|
ignoreDuplicates bool
|
|
stripDirEntries bool
|
|
zipsToNotStrip map[string]bool
|
|
|
|
out []testZipEntry
|
|
err string
|
|
}{
|
|
{
|
|
name: "duplicates error",
|
|
in: [][]testZipEntry{
|
|
{a},
|
|
{a2},
|
|
{a3},
|
|
},
|
|
out: []testZipEntry{a},
|
|
err: "duplicate",
|
|
},
|
|
{
|
|
name: "duplicates take first",
|
|
in: [][]testZipEntry{
|
|
{a},
|
|
{a2},
|
|
{a3},
|
|
},
|
|
out: []testZipEntry{a},
|
|
|
|
ignoreDuplicates: true,
|
|
},
|
|
{
|
|
name: "duplicates identical",
|
|
in: [][]testZipEntry{
|
|
{a},
|
|
{a},
|
|
},
|
|
out: []testZipEntry{a},
|
|
},
|
|
{
|
|
name: "sort",
|
|
in: [][]testZipEntry{
|
|
{be, bc, bDir, bbDir, bbb, A, metainfDir, manifestFile},
|
|
},
|
|
out: []testZipEntry{A, metainfDir, manifestFile, bDir, bbDir, bbb, bc, be},
|
|
|
|
sort: true,
|
|
},
|
|
{
|
|
name: "jar sort",
|
|
in: [][]testZipEntry{
|
|
{be, bc, bDir, A, metainfDir, manifestFile},
|
|
},
|
|
out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
|
|
|
|
jar: true,
|
|
},
|
|
{
|
|
name: "jar merge",
|
|
in: [][]testZipEntry{
|
|
{metainfDir, manifestFile, bDir, be},
|
|
{metainfDir, manifestFile2, bDir, bc},
|
|
{metainfDir, manifestFile2, A},
|
|
},
|
|
out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
|
|
|
|
jar: true,
|
|
},
|
|
{
|
|
name: "merge",
|
|
in: [][]testZipEntry{
|
|
{bDir, be},
|
|
{bDir, bc},
|
|
{A},
|
|
},
|
|
out: []testZipEntry{bDir, be, bc, A},
|
|
},
|
|
{
|
|
name: "strip dir entries",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, bbDir, bbb, bc, bd, be},
|
|
},
|
|
out: []testZipEntry{a, bbb, bc, bd, be},
|
|
|
|
stripDirEntries: true,
|
|
},
|
|
{
|
|
name: "strip files",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, bbDir, bbb, bc, bd, be},
|
|
},
|
|
out: []testZipEntry{a, bDir, bbDir, bbb, bc},
|
|
|
|
stripFiles: []string{"b/d", "b/e"},
|
|
},
|
|
{
|
|
// merge_zips used to treat -stripFile a as stripping any file named a, it now only strips a in the
|
|
// root of the zip.
|
|
name: "strip file name",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, ba},
|
|
},
|
|
out: []testZipEntry{bDir, ba},
|
|
|
|
stripFiles: []string{"a"},
|
|
},
|
|
{
|
|
name: "strip files glob",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, ba},
|
|
},
|
|
out: []testZipEntry{bDir},
|
|
|
|
stripFiles: []string{"**/a"},
|
|
},
|
|
{
|
|
name: "strip dirs",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, bbDir, bbb, bc, bd, be},
|
|
},
|
|
out: []testZipEntry{a},
|
|
|
|
stripDirs: []string{"b"},
|
|
},
|
|
{
|
|
name: "strip dirs glob",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, bbDir, bbb, bc, bd, be},
|
|
},
|
|
out: []testZipEntry{a, bDir, bc, bd, be},
|
|
|
|
stripDirs: []string{"b/*"},
|
|
},
|
|
{
|
|
name: "zips to not strip",
|
|
in: [][]testZipEntry{
|
|
{a, bDir, bc},
|
|
{bDir, bd},
|
|
{bDir, be},
|
|
},
|
|
out: []testZipEntry{a, bDir, bd},
|
|
|
|
stripDirs: []string{"b"},
|
|
zipsToNotStrip: map[string]bool{
|
|
"in1": true,
|
|
},
|
|
},
|
|
{
|
|
name: "services",
|
|
in: [][]testZipEntry{
|
|
{service1a, service2},
|
|
{service1b},
|
|
},
|
|
jar: true,
|
|
out: []testZipEntry{service1combined, service2},
|
|
},
|
|
{
|
|
name: "strip timestamps",
|
|
in: [][]testZipEntry{
|
|
{withTimestamp},
|
|
{a},
|
|
},
|
|
out: []testZipEntry{withoutTimestamp, a},
|
|
},
|
|
{
|
|
name: "emulate par",
|
|
in: [][]testZipEntry{
|
|
{
|
|
testZipEntry{name: "3/main.py"},
|
|
testZipEntry{name: "c/main.py"},
|
|
testZipEntry{name: "a/main.py"},
|
|
testZipEntry{name: "2/main.py"},
|
|
testZipEntry{name: "b/main.py"},
|
|
testZipEntry{name: "1/main.py"},
|
|
},
|
|
},
|
|
out: []testZipEntry{
|
|
testZipEntry{name: "3/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "c/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "a/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "2/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "b/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "1/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "3/main.py", timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "c/main.py", timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "a/main.py", timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "2/main.py", timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "b/main.py", timestamp: jar.DefaultTime},
|
|
testZipEntry{name: "1/main.py", timestamp: jar.DefaultTime},
|
|
},
|
|
par: true,
|
|
},
|
|
}
|
|
|
|
for _, test := range testCases {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
inputZips := make([]InputZip, len(test.in))
|
|
for i, in := range test.in {
|
|
inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in}
|
|
}
|
|
|
|
want := testZipEntriesToBuf(test.out)
|
|
|
|
out := &bytes.Buffer{}
|
|
writer := zip.NewWriter(out)
|
|
|
|
err := mergeZips(inputZips, writer, "", "",
|
|
test.sort, test.jar, test.par, test.stripDirEntries, test.ignoreDuplicates,
|
|
test.stripFiles, test.stripDirs, test.zipsToNotStrip)
|
|
|
|
closeErr := writer.Close()
|
|
if closeErr != nil {
|
|
t.Fatal(closeErr)
|
|
}
|
|
|
|
if test.err != "" {
|
|
if err == nil {
|
|
t.Fatal("missing err, expected: ", test.err)
|
|
} else if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.err)) {
|
|
t.Fatal("incorrect err, want:", test.err, "got:", err)
|
|
}
|
|
return
|
|
} else if err != nil {
|
|
t.Fatal("unexpected err: ", err)
|
|
}
|
|
|
|
if !bytes.Equal(want, out.Bytes()) {
|
|
t.Error("incorrect zip output")
|
|
t.Errorf("want:\n%s", dumpZip(want))
|
|
t.Errorf("got:\n%s", dumpZip(out.Bytes()))
|
|
os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
|
|
os.WriteFile("/tmp/want.zip", want, 0755)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func testZipEntriesToBuf(entries []testZipEntry) []byte {
|
|
b := &bytes.Buffer{}
|
|
zw := zip.NewWriter(b)
|
|
|
|
for _, e := range entries {
|
|
fh := zip.FileHeader{
|
|
Name: e.name,
|
|
}
|
|
fh.SetMode(e.mode)
|
|
fh.Method = e.method
|
|
fh.SetModTime(e.timestamp)
|
|
fh.UncompressedSize64 = uint64(len(e.data))
|
|
fh.CRC32 = crc32.ChecksumIEEE(e.data)
|
|
if fh.Method == zip.Store {
|
|
fh.CompressedSize64 = fh.UncompressedSize64
|
|
}
|
|
|
|
w, err := zw.CreateHeaderAndroid(&fh)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
_, err = w.Write(e.data)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
err := zw.Close()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
return b.Bytes()
|
|
}
|
|
|
|
func testZipEntriesToZipReader(entries []testZipEntry) *zip.Reader {
|
|
b := testZipEntriesToBuf(entries)
|
|
r := bytes.NewReader(b)
|
|
|
|
zr, err := zip.NewReader(r, int64(len(b)))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
return zr
|
|
}
|
|
|
|
func dumpZip(buf []byte) string {
|
|
r := bytes.NewReader(buf)
|
|
zr, err := zip.NewReader(r, int64(len(buf)))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
var ret string
|
|
|
|
for _, f := range zr.File {
|
|
ret += fmt.Sprintf("%v: %v %v %08x %s\n", f.Name, f.Mode(), f.UncompressedSize64, f.CRC32, f.ModTime())
|
|
}
|
|
|
|
return ret
|
|
}
|
|
|
|
type DummyInpuZip struct {
|
|
isOpen bool
|
|
}
|
|
|
|
func (diz *DummyInpuZip) Name() string {
|
|
return "dummy"
|
|
}
|
|
|
|
func (diz *DummyInpuZip) Open() error {
|
|
diz.isOpen = true
|
|
return nil
|
|
}
|
|
|
|
func (diz *DummyInpuZip) Close() error {
|
|
diz.isOpen = false
|
|
return nil
|
|
}
|
|
|
|
func (DummyInpuZip) Entries() []*zip.File {
|
|
panic("implement me")
|
|
}
|
|
|
|
func (diz *DummyInpuZip) IsOpen() bool {
|
|
return diz.isOpen
|
|
}
|
|
|
|
func TestInputZipsManager(t *testing.T) {
|
|
const nInputZips = 20
|
|
const nMaxOpenZips = 10
|
|
izm := NewInputZipsManager(20, 10)
|
|
managedZips := make([]InputZip, nInputZips)
|
|
for i := 0; i < nInputZips; i++ {
|
|
managedZips[i] = izm.Manage(&DummyInpuZip{})
|
|
}
|
|
|
|
t.Run("InputZipsManager", func(t *testing.T) {
|
|
for i, iz := range managedZips {
|
|
if err := iz.Open(); err != nil {
|
|
t.Fatalf("Step %d: open failed: %s", i, err)
|
|
return
|
|
}
|
|
if izm.nOpenZips > nMaxOpenZips {
|
|
t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips)
|
|
}
|
|
}
|
|
if !managedZips[nInputZips-1].IsOpen() {
|
|
t.Error("The last input should stay open")
|
|
}
|
|
for _, iz := range managedZips {
|
|
iz.Close()
|
|
}
|
|
if izm.nOpenZips > 0 {
|
|
t.Error("Some input zips are still open")
|
|
}
|
|
})
|
|
}
|