Merge META-INF/services/* files in merge_zips -jar

kotlinx_coroutines_test and kotlinx_coroutine_android each provide a
META-INF/services/kotlinx.coroutines.CoroutineExceptionHandler with
different contents, and the final contents needs to be the combination
of the two files.  Implement service merging in merge_zips when the
-jar argument is provided.

Bug: 290933559
Test: TestMergeZips
Change-Id: I69f80d1265c64c671d308ef4cdccfa1564abe056

cmd/merge_zips/merge_zips.go

@@ -122,7 +122,7 @@ func (be ZipEntryFromBuffer) Size() uint64 {
 }
 
 func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error {
-	w, err := zw.CreateHeader(be.fh)
+	w, err := zw.CreateHeaderAndroid(be.fh)
 	if err != nil {
 		return err
 	}
@@ -562,6 +562,8 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
 		}
 	}
 
+	var jarServices jar.Services
+
 	// Finally, add entries from all the input zips.
 	for _, inputZip := range inputZips {
 		_, copyFully := zipsToNotStrip[inputZip.Name()]
@@ -570,6 +572,14 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
 		}
 
 		for i, entry := range inputZip.Entries() {
+			if emulateJar && jarServices.IsServiceFile(entry) {
+				// If this is a jar, collect service files to combine  instead of adding them to the zip.
+				err := jarServices.AddServiceFile(entry)
+				if err != nil {
+					return err
+				}
+				continue
+			}
 			if copyFully || !out.isEntryExcluded(entry.Name) {
 				if err := out.copyEntry(inputZip, i); err != nil {
 					return err
@@ -585,6 +595,16 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
 	}
 
 	if emulateJar {
+		// Combine all the service files into a single list of combined service files and add them to the zip.
+		for _, serviceFile := range jarServices.ServiceFiles() {
+			_, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{
+				fh:      serviceFile.FileHeader,
+				content: serviceFile.Contents,
+			})
+			if err != nil {
+				return err
+			}
+		}
 		return out.writeEntries(out.jarSorted())
 	} else if sortEntries {
 		return out.writeEntries(out.alphanumericSorted())

cmd/merge_zips/merge_zips_test.go

@@ -17,6 +17,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"hash/crc32"
 	"os"
 	"strconv"
 	"strings"
@@ -30,25 +31,31 @@ type testZipEntry struct {
 	name   string
 	mode   os.FileMode
 	data   []byte
+	method uint16
 }
 
 var (
-	A     = testZipEntry{"A", 0755, []byte("foo")}
+	A     = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate}
-	a     = testZipEntry{"a", 0755, []byte("foo")}
+	a     = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate}
-	a2    = testZipEntry{"a", 0755, []byte("FOO2")}
+	a2    = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate}
-	a3    = testZipEntry{"a", 0755, []byte("Foo3")}
+	a3    = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate}
-	bDir  = testZipEntry{"b/", os.ModeDir | 0755, nil}
+	bDir  = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate}
-	bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil}
+	bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate}
-	bbb   = testZipEntry{"b/b/b", 0755, nil}
+	bbb   = testZipEntry{"b/b/b", 0755, nil, zip.Deflate}
-	ba    = testZipEntry{"b/a", 0755, []byte("foob")}
+	ba    = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate}
-	bc    = testZipEntry{"b/c", 0755, []byte("bar")}
+	bc    = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate}
-	bd    = testZipEntry{"b/d", 0700, []byte("baz")}
+	bd    = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate}
-	be    = testZipEntry{"b/e", 0700, []byte("")}
+	be    = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate}
 
-	metainfDir     = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil}
+	service1a        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store}
-	manifestFile   = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")}
+	service1b        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate}
-	manifestFile2  = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")}
+	service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store}
-	moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")}
+	service2         = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate}
+
+	metainfDir     = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate}
+	manifestFile   = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate}
+	manifestFile2  = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate}
+	moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate}
 )
 
 type testInputZip struct {
@@ -236,6 +243,15 @@ func TestMergeZips(t *testing.T) {
 				"in1": true,
 			},
 		},
+		{
+			name: "services",
+			in: [][]testZipEntry{
+				{service1a, service2},
+				{service1b},
+			},
+			jar: true,
+			out: []testZipEntry{service1combined, service2},
+		},
 	}
 
 	for _, test := range testCases {
@@ -256,7 +272,7 @@ func TestMergeZips(t *testing.T) {
 
 			closeErr := writer.Close()
 			if closeErr != nil {
-				t.Fatal(err)
+				t.Fatal(closeErr)
 			}
 
 			if test.err != "" {
@@ -266,12 +282,16 @@ func TestMergeZips(t *testing.T) {
 					t.Fatal("incorrect err, want:", test.err, "got:", err)
 				}
 				return
+			} else if err != nil {
+				t.Fatal("unexpected err: ", err)
 			}
 
 			if !bytes.Equal(want, out.Bytes()) {
 				t.Error("incorrect zip output")
 				t.Errorf("want:\n%s", dumpZip(want))
 				t.Errorf("got:\n%s", dumpZip(out.Bytes()))
+				os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
+				os.WriteFile("/tmp/want.zip", want, 0755)
 			}
 		})
 	}
@@ -286,8 +306,14 @@ func testZipEntriesToBuf(entries []testZipEntry) []byte {
 			Name: e.name,
 		}
 		fh.SetMode(e.mode)
+		fh.Method = e.method
+		fh.UncompressedSize64 = uint64(len(e.data))
+		fh.CRC32 = crc32.ChecksumIEEE(e.data)
+		if fh.Method == zip.Store {
+			fh.CompressedSize64 = fh.UncompressedSize64
+		}
 
-		w, err := zw.CreateHeader(&fh)
+		w, err := zw.CreateHeaderAndroid(&fh)
 		if err != nil {
 			panic(err)
 		}

jar/Android.bp

@@ -21,6 +21,7 @@ bootstrap_go_package {
     pkgPath: "android/soong/jar",
     srcs: [
         "jar.go",
+        "services.go",
     ],
     testSrcs: [
         "jar_test.go",

jar/services.go

@@ -0,0 +1,128 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jar
+
+import (
+	"android/soong/third_party/zip"
+	"bufio"
+	"hash/crc32"
+	"sort"
+	"strings"
+)
+
+const servicesPrefix = "META-INF/services/"
+
+// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing
+// the unique lines from all the input zip entries with the same name.
+type Services struct {
+	services map[string]*ServiceFile
+}
+
+// ServiceFile contains the combined contents of all input zip entries with a single name.
+type ServiceFile struct {
+	Name       string
+	FileHeader *zip.FileHeader
+	Contents   []byte
+	Lines      []string
+}
+
+// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory.
+func (Services) IsServiceFile(entry *zip.File) bool {
+	return strings.HasPrefix(entry.Name, servicesPrefix)
+}
+
+// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need
+// to be combined.
+func (j *Services) AddServiceFile(entry *zip.File) error {
+	if j.services == nil {
+		j.services = map[string]*ServiceFile{}
+	}
+
+	service := entry.Name
+	serviceFile := j.services[service]
+	fh := entry.FileHeader
+	if serviceFile == nil {
+		serviceFile = &ServiceFile{
+			Name:       service,
+			FileHeader: &fh,
+		}
+		j.services[service] = serviceFile
+	}
+
+	f, err := entry.Open()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line != "" {
+			serviceFile.Lines = append(serviceFile.Lines, line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ServiceFiles returns the list of combined service files, each containing all the unique lines from the
+// corresponding service files in the input zip entries.
+func (j *Services) ServiceFiles() []ServiceFile {
+	services := make([]ServiceFile, 0, len(j.services))
+
+	for _, serviceFile := range j.services {
+		serviceFile.Lines = dedupServicesLines(serviceFile.Lines)
+		serviceFile.Lines = append(serviceFile.Lines, "")
+		serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n"))
+
+		serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents))
+		serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents)
+		if serviceFile.FileHeader.Method == zip.Store {
+			serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64
+		}
+
+		services = append(services, *serviceFile)
+	}
+
+	sort.Slice(services, func(i, j int) bool {
+		return services[i].Name < services[j].Name
+	})
+
+	return services
+}
+
+func dedupServicesLines(in []string) []string {
+	writeIndex := 0
+outer:
+	for readIndex := 0; readIndex < len(in); readIndex++ {
+		for compareIndex := 0; compareIndex < writeIndex; compareIndex++ {
+			if interface{}(in[readIndex]) == interface{}(in[compareIndex]) {
+				// The value at readIndex already exists somewhere in the output region
+				// of the slice before writeIndex, skip it.
+				continue outer
+			}
+		}
+		if readIndex != writeIndex {
+			in[writeIndex] = in[readIndex]
+		}
+		writeIndex++
+	}
+	return in[0:writeIndex]
+}

third_party/zip/android.go

@@ -170,7 +170,7 @@ func (w *Writer) CreateCompressedHeader(fh *FileHeader) (io.WriteCloser, error)
 func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) {
 	writeDataDescriptor := fh.Method != Store
 	if writeDataDescriptor {
-		fh.Flags &= DataDescriptorFlag
+		fh.Flags |= DataDescriptorFlag
 	} else {
 		fh.Flags &= ^uint16(DataDescriptorFlag)
 	}