tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

soong_zip: Add tests

Browse source 
am: 05518bc13b

Change-Id: I08bd473131439f46a962f3b33dc387806362cbfb
This commit is contained in:
Colin Cross 2018-09-28 17:32:51 -07:00 committed by android-build-merger
commit d843c25e24
7 changed files with 445 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/merge_zips/merge_zips.go
View file

@ -250,7 +250,12 @@ func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoin
addMapping(jar.MetaDir, dirSource)
}
fh, buf, err := jar.ManifestFileContents(manifest)
contents, err := ioutil.ReadFile(manifest)
if err != nil {
return err
}
fh, buf, err := jar.ManifestFileContents(contents)
if err != nil {
return err
}

4
cmd/multiproduct_kati/main.go
View file

@ -328,7 +328,7 @@ func main() {
NumParallelJobs: runtime.NumCPU(),
CompressionLevel: 5,
}
if err := zip.Run(args); err != nil {
if err := zip.Zip(args); err != nil {
log.Fatalf("Error zipping logs: %v", err)
}
}
@ -409,7 +409,7 @@ func buildProduct(mpctx *mpContext, product string) {
NumParallelJobs: runtime.NumCPU(),
CompressionLevel: 5,
}
if err := zip.Run(args); err != nil {
if err := zip.Zip(args); err != nil {
log.Fatalf("Error zipping artifacts: %v", err)
}
}

28
jar/jar.go
View file

@ -17,7 +17,6 @@ package jar
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"strings"
"time"
@ -81,10 +80,9 @@ func MetaDirFileHeader() *zip.FileHeader {
return dirHeader
}
// Convert manifest source path to zip header and contents. If path is empty uses a default
// manifest.
func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) {
b, err := manifestContents(src)
// Create a manifest zip header and contents using the provided contents if any.
func ManifestFileContents(contents []byte) (*zip.FileHeader, []byte, error) {
b, err := manifestContents(contents)
if err != nil {
return nil, nil, err
}
@ -100,26 +98,16 @@ func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) {
return fh, b, nil
}
// Convert manifest source path to contents. If path is empty uses a default manifest.
func manifestContents(src string) ([]byte, error) {
var givenBytes []byte
var err error
if src != "" {
givenBytes, err = ioutil.ReadFile(src)
if err != nil {
return nil, err
}
}
// Create manifest contents, using the provided contents if any.
func manifestContents(contents []byte) ([]byte, error) {
manifestMarker := []byte("Manifest-Version:")
header := append(manifestMarker, []byte(" 1.0\nCreated-By: soong_zip\n")...)
var finalBytes []byte
if !bytes.Contains(givenBytes, manifestMarker) {
finalBytes = append(append(header, givenBytes...), byte('\n'))
if !bytes.Contains(contents, manifestMarker) {
finalBytes = append(append(header, contents...), byte('\n'))
} else {
finalBytes = givenBytes
finalBytes = contents
}
return finalBytes, nil

3
java/java.go
View file

@ -95,6 +95,9 @@ type CompilerProperties struct {
// list of java libraries that will be compiled into the resulting jar
Static_libs []string `android:"arch_variant"`
// list of native libraries that will be provided in or alongside the resulting jar
Jni_libs []string `android:"arch_variant"`
// manifest file to be included in resulting jar
Manifest *string

2
zip/cmd/main.go
View file

@ -187,7 +187,7 @@ func main() {
os.Exit(1)
}
err := zip.Run(zip.ZipArgs{
err := zip.Zip(zip.ZipArgs{
FileArgs: fileArgsBuilder.FileArgs(),
OutputFilePath: *out,
EmulateJar: *emulateJar,

61
zip/zip.go
View file

@ -22,7 +22,6 @@ import (
"hash/crc32"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
@ -178,6 +177,8 @@ type ZipWriter struct {
compressorPool sync.Pool
compLevel int
fs pathtools.FileSystem
}
type zipEntry struct {
@ -201,6 +202,7 @@ type ZipArgs struct {
NumParallelJobs int
NonDeflatedFiles map[string]bool
WriteIfChanged bool
Filesystem pathtools.FileSystem
}
const NOQUOTE = '\x00'
@ -246,22 +248,24 @@ func ReadRespFile(bytes []byte) []string {
return args
}
func Run(args ZipArgs) (err error) {
if args.OutputFilePath == "" {
return fmt.Errorf("output file path must be nonempty")
}
func ZipTo(args ZipArgs, w io.Writer) error {
if args.EmulateJar {
args.AddDirectoryEntriesToZip = true
}
w := &ZipWriter{
z := &ZipWriter{
time: jar.DefaultTime,
createdDirs: make(map[string]string),
createdFiles: make(map[string]string),
directories: args.AddDirectoryEntriesToZip,
compLevel: args.CompressionLevel,
fs: args.Filesystem,
}
if z.fs == nil {
z.fs = pathtools.OsFs
}
pathMappings := []pathMapping{}
noCompression := args.CompressionLevel == 0
@ -274,11 +278,19 @@ func Run(args ZipArgs) (err error) {
for _, src := range srcs {
err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
if err != nil {
log.Fatal(err)
return err
}
}
}
return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
}
func Zip(args ZipArgs) error {
if args.OutputFilePath == "" {
return fmt.Errorf("output file path must be nonempty")
}
buf := &bytes.Buffer{}
var out io.Writer = buf
@ -298,7 +310,7 @@ func Run(args ZipArgs) (err error) {
out = f
}
err = w.write(out, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
err := ZipTo(args, out)
if err != nil {
return err
}
@ -351,13 +363,6 @@ func jarSort(mappings []pathMapping) {
sort.SliceStable(mappings, less)
}
type readerSeekerCloser interface {
io.Reader
io.ReaderAt
io.Closer
io.Seeker
}
func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
z.errors = make(chan error)
defer close(z.errors)
@ -504,7 +509,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
var fileSize int64
var executable bool
if s, err := os.Lstat(src); err != nil {
if s, err := z.fs.Lstat(src); err != nil {
return err
} else if s.IsDir() {
if z.directories {
@ -535,7 +540,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
executable = s.Mode()&0100 != 0
}
r, err := os.Open(src)
r, err := z.fs.Open(src)
if err != nil {
return err
}
@ -565,7 +570,21 @@ func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
return err
}
fh, buf, err := jar.ManifestFileContents(src)
var contents []byte
if src != "" {
f, err := z.fs.Open(src)
if err != nil {
return err
}
contents, err = ioutil.ReadAll(f)
f.Close()
if err != nil {
return err
}
}
fh, buf, err := jar.ManifestFileContents(contents)
if err != nil {
return err
}
@ -575,7 +594,7 @@ func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
return z.writeFileContents(fh, reader)
}
func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r readerSeekerCloser) (err error) {
func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
header.SetModTime(z.time)
@ -845,7 +864,7 @@ func (z *ZipWriter) writeSymlink(rel, file string) error {
fileHeader.SetModTime(z.time)
fileHeader.SetMode(0777 | os.ModeSymlink)
dest, err := os.Readlink(file)
dest, err := z.fs.Readlink(file)
if err != nil {
return err
}

385
zip/zip_test.go
View file

@ -15,10 +15,395 @@
package zip
import (
"bytes"
"hash/crc32"
"io"
"os"
"reflect"
"syscall"
"testing"
"android/soong/third_party/zip"
"github.com/google/blueprint/pathtools"
)
var (
fileA = []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
fileB = []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")
fileC = []byte("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")
fileEmpty = []byte("")
fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n")
fileCustomManifest = []byte("Custom manifest: true\n")
customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n")
)
var mockFs = pathtools.MockFs(map[string][]byte{
"a/a/a": fileA,
"a/a/b": fileB,
"a/a/c -> ../../c": nil,
"a/a/d -> b": nil,
"c": fileC,
"l": []byte("a/a/a\na/a/b\nc\n"),
"l2": []byte("missing\n"),
"manifest.txt": fileCustomManifest,
})
func fh(name string, contents []byte, method uint16) zip.FileHeader {
return zip.FileHeader{
Name: name,
Method: method,
CRC32: crc32.ChecksumIEEE(contents),
UncompressedSize64: uint64(len(contents)),
ExternalAttrs: 0,
}
}
func fhManifest(contents []byte) zip.FileHeader {
return zip.FileHeader{
Name: "META-INF/MANIFEST.MF",
Method: zip.Store,
CRC32: crc32.ChecksumIEEE(contents),
UncompressedSize64: uint64(len(contents)),
ExternalAttrs: (syscall.S_IFREG | 0700) << 16,
}
}
func fhLink(name string, to string) zip.FileHeader {
return zip.FileHeader{
Name: name,
Method: zip.Store,
CRC32: crc32.ChecksumIEEE([]byte(to)),
UncompressedSize64: uint64(len(to)),
ExternalAttrs: (syscall.S_IFLNK | 0777) << 16,
}
}
func fhDir(name string) zip.FileHeader {
return zip.FileHeader{
Name: name,
Method: zip.Store,
CRC32: crc32.ChecksumIEEE(nil),
UncompressedSize64: 0,
ExternalAttrs: (syscall.S_IFDIR|0700)<<16 | 0x10,
}
}
func fileArgsBuilder() *FileArgsBuilder {
return &FileArgsBuilder{
fs: mockFs,
}
}
func TestZip(t *testing.T) {
testCases := []struct {
name string
args *FileArgsBuilder
compressionLevel int
emulateJar bool
nonDeflatedFiles map[string]bool
dirEntries bool
manifest string
files []zip.FileHeader
err error
}{
{
name: "empty args",
args: fileArgsBuilder(),
files: []zip.FileHeader{},
},
{
name: "files",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b").
File("c"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
fh("c", fileC, zip.Deflate),
},
},
{
name: "stored files",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b").
File("c"),
compressionLevel: 0,
files: []zip.FileHeader{
fh("a/a/a", fileA, zip.Store),
fh("a/a/b", fileB, zip.Store),
fh("c", fileC, zip.Store),
},
},
{
name: "symlinks in zip",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b").
File("a/a/c").
File("a/a/d"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
fhLink("a/a/c", "../../c"),
fhLink("a/a/d", "b"),
},
},
{
name: "list",
args: fileArgsBuilder().
List("l"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
fh("c", fileC, zip.Deflate),
},
},
{
name: "prefix in zip",
args: fileArgsBuilder().
PathPrefixInZip("foo").
File("a/a/a").
File("a/a/b").
File("c"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("foo/a/a/a", fileA, zip.Deflate),
fh("foo/a/a/b", fileB, zip.Deflate),
fh("foo/c", fileC, zip.Deflate),
},
},
{
name: "relative root",
args: fileArgsBuilder().
SourcePrefixToStrip("a").
File("a/a/a").
File("a/a/b"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("a/a", fileA, zip.Deflate),
fh("a/b", fileB, zip.Deflate),
},
},
{
name: "multiple relative root",
args: fileArgsBuilder().
SourcePrefixToStrip("a").
File("a/a/a").
SourcePrefixToStrip("a/a").
File("a/a/b"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("a/a", fileA, zip.Deflate),
fh("b", fileB, zip.Deflate),
},
},
{
name: "emulate jar",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b"),
compressionLevel: 9,
emulateJar: true,
files: []zip.FileHeader{
fhDir("META-INF/"),
fhManifest(fileManifest),
fhDir("a/"),
fhDir("a/a/"),
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
},
},
{
name: "emulate jar with manifest",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b"),
compressionLevel: 9,
emulateJar: true,
manifest: "manifest.txt",
files: []zip.FileHeader{
fhDir("META-INF/"),
fhManifest(customManifestAfter),
fhDir("a/"),
fhDir("a/a/"),
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
},
},
{
name: "dir entries",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b"),
compressionLevel: 9,
dirEntries: true,
files: []zip.FileHeader{
fhDir("a/"),
fhDir("a/a/"),
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
},
},
{
name: "junk paths",
args: fileArgsBuilder().
JunkPaths(true).
File("a/a/a").
File("a/a/b"),
compressionLevel: 9,
files: []zip.FileHeader{
fh("a", fileA, zip.Deflate),
fh("b", fileB, zip.Deflate),
},
},
{
name: "non deflated files",
args: fileArgsBuilder().
File("a/a/a").
File("a/a/b"),
compressionLevel: 9,
nonDeflatedFiles: map[string]bool{"a/a/a": true},
files: []zip.FileHeader{
fh("a/a/a", fileA, zip.Store),
fh("a/a/b", fileB, zip.Deflate),
},
},
// errors
{
name: "error missing file",
args: fileArgsBuilder().
File("missing"),
err: os.ErrNotExist,
},
{
name: "error missing file in list",
args: fileArgsBuilder().
List("l2"),
err: os.ErrNotExist,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
if test.args.Error() != nil {
t.Fatal(test.args.Error())
}
args := ZipArgs{}
args.FileArgs = test.args.FileArgs()
args.CompressionLevel = test.compressionLevel
args.EmulateJar = test.emulateJar
args.AddDirectoryEntriesToZip = test.dirEntries
args.NonDeflatedFiles = test.nonDeflatedFiles
args.ManifestSourcePath = test.manifest
args.Filesystem = mockFs
buf := &bytes.Buffer{}
err := ZipTo(args, buf)
if (err != nil) != (test.err != nil) {
t.Fatalf("want error %v, got %v", test.err, err)
} else if test.err != nil {
if os.IsNotExist(test.err) {
if !os.IsNotExist(test.err) {
t.Fatalf("want error %v, got %v", test.err, err)
}
} else {
t.Fatalf("want error %v, got %v", test.err, err)
}
return
}
br := bytes.NewReader(buf.Bytes())
zr, err := zip.NewReader(br, int64(br.Len()))
if err != nil {
t.Fatal(err)
}
var files []zip.FileHeader
for _, f := range zr.File {
r, err := f.Open()
if err != nil {
t.Fatalf("error when opening %s: %s", f.Name, err)
}
crc := crc32.NewIEEE()
len, err := io.Copy(crc, r)
r.Close()
if err != nil {
t.Fatalf("error when reading %s: %s", f.Name, err)
}
if uint64(len) != f.UncompressedSize64 {
t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len)
}
if crc.Sum32() != f.CRC32 {
t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc)
}
files = append(files, f.FileHeader)
}
if len(files) != len(test.files) {
t.Fatalf("want %d files, got %d", len(test.files), len(files))
}
for i := range files {
want := test.files[i]
got := files[i]
if want.Name != got.Name {
t.Errorf("incorrect file %d want %q got %q", i, want.Name, got.Name)
continue
}
if want.UncompressedSize64 != got.UncompressedSize64 {
t.Errorf("incorrect file %s length want %v got %v", want.Name,
want.UncompressedSize64, got.UncompressedSize64)
}
if want.ExternalAttrs != got.ExternalAttrs {
t.Errorf("incorrect file %s attrs want %x got %x", want.Name,
want.ExternalAttrs, got.ExternalAttrs)
}
if want.CRC32 != got.CRC32 {
t.Errorf("incorrect file %s crc want %v got %v", want.Name,
want.CRC32, got.CRC32)
}
if want.Method != got.Method {
t.Errorf("incorrect file %s method want %v got %v", want.Name,
want.Method, got.Method)
}
}
})
}
}
func TestReadRespFile(t *testing.T) {
testCases := []struct {
name, in string