soong_zip: Add tests

am: 05518bc13b

Change-Id: I08bd473131439f46a962f3b33dc387806362cbfb

cmd/merge_zips/merge_zips.go

@@ -250,7 +250,12 @@ func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoin
 			addMapping(jar.MetaDir, dirSource)
 		}
 
-		fh, buf, err := jar.ManifestFileContents(manifest)
+		contents, err := ioutil.ReadFile(manifest)
+		if err != nil {
+			return err
+		}
+
+		fh, buf, err := jar.ManifestFileContents(contents)
 		if err != nil {
 			return err
 		}

cmd/multiproduct_kati/main.go

@@ -328,7 +328,7 @@ func main() {
 			NumParallelJobs:  runtime.NumCPU(),
 			CompressionLevel: 5,
 		}
-		if err := zip.Run(args); err != nil {
+		if err := zip.Zip(args); err != nil {
 			log.Fatalf("Error zipping logs: %v", err)
 		}
 	}
@@ -409,7 +409,7 @@ func buildProduct(mpctx *mpContext, product string) {
 				NumParallelJobs:  runtime.NumCPU(),
 				CompressionLevel: 5,
 			}
-			if err := zip.Run(args); err != nil {
+			if err := zip.Zip(args); err != nil {
 				log.Fatalf("Error zipping artifacts: %v", err)
 			}
 		}

jar/jar.go

@@ -17,7 +17,6 @@ package jar
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 	"time"
@@ -81,10 +80,9 @@ func MetaDirFileHeader() *zip.FileHeader {
 	return dirHeader
 }
 
-// Convert manifest source path to zip header and contents.  If path is empty uses a default
+// Create a manifest zip header and contents using the provided contents if any.
-// manifest.
+func ManifestFileContents(contents []byte) (*zip.FileHeader, []byte, error) {
-func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) {
+	b, err := manifestContents(contents)
-	b, err := manifestContents(src)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -100,26 +98,16 @@ func ManifestFileContents(src string) (*zip.FileHeader, []byte, error) {
 	return fh, b, nil
 }
 
-// Convert manifest source path to contents.  If path is empty uses a default manifest.
+// Create manifest contents, using the provided contents if any.
-func manifestContents(src string) ([]byte, error) {
+func manifestContents(contents []byte) ([]byte, error) {
-	var givenBytes []byte
-	var err error
-
-	if src != "" {
-		givenBytes, err = ioutil.ReadFile(src)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	manifestMarker := []byte("Manifest-Version:")
 	header := append(manifestMarker, []byte(" 1.0\nCreated-By: soong_zip\n")...)
 
 	var finalBytes []byte
-	if !bytes.Contains(givenBytes, manifestMarker) {
+	if !bytes.Contains(contents, manifestMarker) {
-		finalBytes = append(append(header, givenBytes...), byte('\n'))
+		finalBytes = append(append(header, contents...), byte('\n'))
 	} else {
-		finalBytes = givenBytes
+		finalBytes = contents
 	}
 
 	return finalBytes, nil

java/java.go

@@ -95,6 +95,9 @@ type CompilerProperties struct {
 	// list of java libraries that will be compiled into the resulting jar
 	Static_libs []string `android:"arch_variant"`
 
+	// list of native libraries that will be provided in or alongside the resulting jar
+	Jni_libs []string `android:"arch_variant"`
+
 	// manifest file to be included in resulting jar
 	Manifest *string
 

zip/cmd/main.go

@@ -187,7 +187,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err := zip.Run(zip.ZipArgs{
+	err := zip.Zip(zip.ZipArgs{
 		FileArgs:                 fileArgsBuilder.FileArgs(),
 		OutputFilePath:           *out,
 		EmulateJar:               *emulateJar,

zip/zip.go

@@ -22,7 +22,6 @@ import (
 	"hash/crc32"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"sort"
@@ -178,6 +177,8 @@ type ZipWriter struct {
 
 	compressorPool sync.Pool
 	compLevel      int
+
+	fs pathtools.FileSystem
 }
 
 type zipEntry struct {
@@ -201,6 +202,7 @@ type ZipArgs struct {
 	NumParallelJobs          int
 	NonDeflatedFiles         map[string]bool
 	WriteIfChanged           bool
+	Filesystem               pathtools.FileSystem
 }
 
 const NOQUOTE = '\x00'
@@ -246,22 +248,24 @@ func ReadRespFile(bytes []byte) []string {
 	return args
 }
 
-func Run(args ZipArgs) (err error) {
+func ZipTo(args ZipArgs, w io.Writer) error {
-	if args.OutputFilePath == "" {
-		return fmt.Errorf("output file path must be nonempty")
-	}
-
 	if args.EmulateJar {
 		args.AddDirectoryEntriesToZip = true
 	}
 
-	w := &ZipWriter{
+	z := &ZipWriter{
 		time:         jar.DefaultTime,
 		createdDirs:  make(map[string]string),
 		createdFiles: make(map[string]string),
 		directories:  args.AddDirectoryEntriesToZip,
 		compLevel:    args.CompressionLevel,
+		fs:           args.Filesystem,
 	}
+
+	if z.fs == nil {
+		z.fs = pathtools.OsFs
+	}
+
 	pathMappings := []pathMapping{}
 
 	noCompression := args.CompressionLevel == 0
@@ -274,11 +278,19 @@ func Run(args ZipArgs) (err error) {
 		for _, src := range srcs {
 			err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
 			if err != nil {
-				log.Fatal(err)
+				return err
 			}
 		}
 	}
 
+	return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
+}
+
+func Zip(args ZipArgs) error {
+	if args.OutputFilePath == "" {
+		return fmt.Errorf("output file path must be nonempty")
+	}
+
 	buf := &bytes.Buffer{}
 	var out io.Writer = buf
 
@@ -298,7 +310,7 @@ func Run(args ZipArgs) (err error) {
 		out = f
 	}
 
-	err = w.write(out, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
+	err := ZipTo(args, out)
 	if err != nil {
 		return err
 	}
@@ -351,13 +363,6 @@ func jarSort(mappings []pathMapping) {
 	sort.SliceStable(mappings, less)
 }
 
-type readerSeekerCloser interface {
-	io.Reader
-	io.ReaderAt
-	io.Closer
-	io.Seeker
-}
-
 func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
 	z.errors = make(chan error)
 	defer close(z.errors)
@@ -504,7 +509,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
 	var fileSize int64
 	var executable bool
 
-	if s, err := os.Lstat(src); err != nil {
+	if s, err := z.fs.Lstat(src); err != nil {
 		return err
 	} else if s.IsDir() {
 		if z.directories {
@@ -535,7 +540,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
 		executable = s.Mode()&0100 != 0
 	}
 
-	r, err := os.Open(src)
+	r, err := z.fs.Open(src)
 	if err != nil {
 		return err
 	}
@@ -565,7 +570,21 @@ func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
 		return err
 	}
 
-	fh, buf, err := jar.ManifestFileContents(src)
+	var contents []byte
+	if src != "" {
+		f, err := z.fs.Open(src)
+		if err != nil {
+			return err
+		}
+
+		contents, err = ioutil.ReadAll(f)
+		f.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	fh, buf, err := jar.ManifestFileContents(contents)
 	if err != nil {
 		return err
 	}
@@ -575,7 +594,7 @@ func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
 	return z.writeFileContents(fh, reader)
 }
 
-func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r readerSeekerCloser) (err error) {
+func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
 
 	header.SetModTime(z.time)
 
@@ -845,7 +864,7 @@ func (z *ZipWriter) writeSymlink(rel, file string) error {
 	fileHeader.SetModTime(z.time)
 	fileHeader.SetMode(0777 | os.ModeSymlink)
 
-	dest, err := os.Readlink(file)
+	dest, err := z.fs.Readlink(file)
 	if err != nil {
 		return err
 	}

zip/zip_test.go

@@ -15,10 +15,395 @@
 package zip
 
 import (
+	"bytes"
+	"hash/crc32"
+	"io"
+	"os"
 	"reflect"
+	"syscall"
 	"testing"
+
+	"android/soong/third_party/zip"
+
+	"github.com/google/blueprint/pathtools"
 )
 
+var (
+	fileA        = []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+	fileB        = []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")
+	fileC        = []byte("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")
+	fileEmpty    = []byte("")
+	fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n")
+
+	fileCustomManifest  = []byte("Custom manifest: true\n")
+	customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n")
+)
+
+var mockFs = pathtools.MockFs(map[string][]byte{
+	"a/a/a":            fileA,
+	"a/a/b":            fileB,
+	"a/a/c -> ../../c": nil,
+	"a/a/d -> b":       nil,
+	"c":                fileC,
+	"l":                []byte("a/a/a\na/a/b\nc\n"),
+	"l2":               []byte("missing\n"),
+	"manifest.txt":     fileCustomManifest,
+})
+
+func fh(name string, contents []byte, method uint16) zip.FileHeader {
+	return zip.FileHeader{
+		Name:               name,
+		Method:             method,
+		CRC32:              crc32.ChecksumIEEE(contents),
+		UncompressedSize64: uint64(len(contents)),
+		ExternalAttrs:      0,
+	}
+}
+
+func fhManifest(contents []byte) zip.FileHeader {
+	return zip.FileHeader{
+		Name:               "META-INF/MANIFEST.MF",
+		Method:             zip.Store,
+		CRC32:              crc32.ChecksumIEEE(contents),
+		UncompressedSize64: uint64(len(contents)),
+		ExternalAttrs:      (syscall.S_IFREG | 0700) << 16,
+	}
+}
+
+func fhLink(name string, to string) zip.FileHeader {
+	return zip.FileHeader{
+		Name:               name,
+		Method:             zip.Store,
+		CRC32:              crc32.ChecksumIEEE([]byte(to)),
+		UncompressedSize64: uint64(len(to)),
+		ExternalAttrs:      (syscall.S_IFLNK | 0777) << 16,
+	}
+}
+
+func fhDir(name string) zip.FileHeader {
+	return zip.FileHeader{
+		Name:               name,
+		Method:             zip.Store,
+		CRC32:              crc32.ChecksumIEEE(nil),
+		UncompressedSize64: 0,
+		ExternalAttrs:      (syscall.S_IFDIR|0700)<<16 | 0x10,
+	}
+}
+
+func fileArgsBuilder() *FileArgsBuilder {
+	return &FileArgsBuilder{
+		fs: mockFs,
+	}
+}
+
+func TestZip(t *testing.T) {
+	testCases := []struct {
+		name             string
+		args             *FileArgsBuilder
+		compressionLevel int
+		emulateJar       bool
+		nonDeflatedFiles map[string]bool
+		dirEntries       bool
+		manifest         string
+
+		files []zip.FileHeader
+		err   error
+	}{
+		{
+			name: "empty args",
+			args: fileArgsBuilder(),
+
+			files: []zip.FileHeader{},
+		},
+		{
+			name: "files",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b").
+				File("c"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+				fh("c", fileC, zip.Deflate),
+			},
+		},
+		{
+			name: "stored files",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b").
+				File("c"),
+			compressionLevel: 0,
+
+			files: []zip.FileHeader{
+				fh("a/a/a", fileA, zip.Store),
+				fh("a/a/b", fileB, zip.Store),
+				fh("c", fileC, zip.Store),
+			},
+		},
+		{
+			name: "symlinks in zip",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b").
+				File("a/a/c").
+				File("a/a/d"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+				fhLink("a/a/c", "../../c"),
+				fhLink("a/a/d", "b"),
+			},
+		},
+		{
+			name: "list",
+			args: fileArgsBuilder().
+				List("l"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+				fh("c", fileC, zip.Deflate),
+			},
+		},
+		{
+			name: "prefix in zip",
+			args: fileArgsBuilder().
+				PathPrefixInZip("foo").
+				File("a/a/a").
+				File("a/a/b").
+				File("c"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("foo/a/a/a", fileA, zip.Deflate),
+				fh("foo/a/a/b", fileB, zip.Deflate),
+				fh("foo/c", fileC, zip.Deflate),
+			},
+		},
+		{
+			name: "relative root",
+			args: fileArgsBuilder().
+				SourcePrefixToStrip("a").
+				File("a/a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("a/a", fileA, zip.Deflate),
+				fh("a/b", fileB, zip.Deflate),
+			},
+		},
+		{
+			name: "multiple relative root",
+			args: fileArgsBuilder().
+				SourcePrefixToStrip("a").
+				File("a/a/a").
+				SourcePrefixToStrip("a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("a/a", fileA, zip.Deflate),
+				fh("b", fileB, zip.Deflate),
+			},
+		},
+		{
+			name: "emulate jar",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+			emulateJar:       true,
+
+			files: []zip.FileHeader{
+				fhDir("META-INF/"),
+				fhManifest(fileManifest),
+				fhDir("a/"),
+				fhDir("a/a/"),
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+			},
+		},
+		{
+			name: "emulate jar with manifest",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+			emulateJar:       true,
+			manifest:         "manifest.txt",
+
+			files: []zip.FileHeader{
+				fhDir("META-INF/"),
+				fhManifest(customManifestAfter),
+				fhDir("a/"),
+				fhDir("a/a/"),
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+			},
+		},
+		{
+			name: "dir entries",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+			dirEntries:       true,
+
+			files: []zip.FileHeader{
+				fhDir("a/"),
+				fhDir("a/a/"),
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+			},
+		},
+		{
+			name: "junk paths",
+			args: fileArgsBuilder().
+				JunkPaths(true).
+				File("a/a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+
+			files: []zip.FileHeader{
+				fh("a", fileA, zip.Deflate),
+				fh("b", fileB, zip.Deflate),
+			},
+		},
+		{
+			name: "non deflated files",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b"),
+			compressionLevel: 9,
+			nonDeflatedFiles: map[string]bool{"a/a/a": true},
+
+			files: []zip.FileHeader{
+				fh("a/a/a", fileA, zip.Store),
+				fh("a/a/b", fileB, zip.Deflate),
+			},
+		},
+
+		// errors
+		{
+			name: "error missing file",
+			args: fileArgsBuilder().
+				File("missing"),
+			err: os.ErrNotExist,
+		},
+		{
+			name: "error missing file in list",
+			args: fileArgsBuilder().
+				List("l2"),
+			err: os.ErrNotExist,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			if test.args.Error() != nil {
+				t.Fatal(test.args.Error())
+			}
+
+			args := ZipArgs{}
+			args.FileArgs = test.args.FileArgs()
+			args.CompressionLevel = test.compressionLevel
+			args.EmulateJar = test.emulateJar
+			args.AddDirectoryEntriesToZip = test.dirEntries
+			args.NonDeflatedFiles = test.nonDeflatedFiles
+			args.ManifestSourcePath = test.manifest
+			args.Filesystem = mockFs
+
+			buf := &bytes.Buffer{}
+			err := ZipTo(args, buf)
+
+			if (err != nil) != (test.err != nil) {
+				t.Fatalf("want error %v, got %v", test.err, err)
+			} else if test.err != nil {
+				if os.IsNotExist(test.err) {
+					if !os.IsNotExist(test.err) {
+						t.Fatalf("want error %v, got %v", test.err, err)
+					}
+				} else {
+					t.Fatalf("want error %v, got %v", test.err, err)
+				}
+				return
+			}
+
+			br := bytes.NewReader(buf.Bytes())
+			zr, err := zip.NewReader(br, int64(br.Len()))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var files []zip.FileHeader
+			for _, f := range zr.File {
+				r, err := f.Open()
+				if err != nil {
+					t.Fatalf("error when opening %s: %s", f.Name, err)
+				}
+
+				crc := crc32.NewIEEE()
+				len, err := io.Copy(crc, r)
+				r.Close()
+				if err != nil {
+					t.Fatalf("error when reading %s: %s", f.Name, err)
+				}
+
+				if uint64(len) != f.UncompressedSize64 {
+					t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len)
+				}
+
+				if crc.Sum32() != f.CRC32 {
+					t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc)
+				}
+
+				files = append(files, f.FileHeader)
+			}
+
+			if len(files) != len(test.files) {
+				t.Fatalf("want %d files, got %d", len(test.files), len(files))
+			}
+
+			for i := range files {
+				want := test.files[i]
+				got := files[i]
+
+				if want.Name != got.Name {
+					t.Errorf("incorrect file %d want %q got %q", i, want.Name, got.Name)
+					continue
+				}
+
+				if want.UncompressedSize64 != got.UncompressedSize64 {
+					t.Errorf("incorrect file %s length want %v got %v", want.Name,
+						want.UncompressedSize64, got.UncompressedSize64)
+				}
+
+				if want.ExternalAttrs != got.ExternalAttrs {
+					t.Errorf("incorrect file %s attrs want %x got %x", want.Name,
+						want.ExternalAttrs, got.ExternalAttrs)
+				}
+
+				if want.CRC32 != got.CRC32 {
+					t.Errorf("incorrect file %s crc want %v got %v", want.Name,
+						want.CRC32, got.CRC32)
+				}
+
+				if want.Method != got.Method {
+					t.Errorf("incorrect file %s method want %v got %v", want.Name,
+						want.Method, got.Method)
+				}
+			}
+		})
+	}
+}
+
 func TestReadRespFile(t *testing.T) {
 	testCases := []struct {
 		name, in string