Merge "Add metadata generator tool for test spec metadata generation. Bug: 296873595 Test: Manual test (use go test inside tools/metadata/testdata)" into main

tools/metadata/Android.bp

@@ -0,0 +1,14 @@
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+blueprint_go_binary {
+    name: "metadata",
+    deps: [
+            "soong-testing-test_spec_proto",
+            "golang-protobuf-proto",
+        ],
+    srcs: [
+        "generator.go",
+    ]
+}

tools/metadata/generator.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+
+	"android/soong/testing/test_spec_proto"
+	"google.golang.org/protobuf/proto"
+)
+
+type keyToLocksMap struct {
+	locks sync.Map
+}
+
+func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex {
+	mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{})
+	return mutex.(*sync.Mutex)
+}
+
+func getSortedKeys(syncMap *sync.Map) []string {
+	var allKeys []string
+	syncMap.Range(
+		func(key, _ interface{}) bool {
+			allKeys = append(allKeys, key.(string))
+			return true
+		},
+	)
+
+	sort.Strings(allKeys)
+	return allKeys
+}
+
+func writeOutput(
+	outputFile string,
+	allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata,
+) {
+	testSpec := &test_spec_proto.TestSpec{
+		OwnershipMetadataList: allMetadata,
+	}
+	data, err := proto.Marshal(testSpec)
+	if err != nil {
+		log.Fatal(err)
+	}
+	file, err := os.Create(outputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	_, err = file.Write(data)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func readFileToString(filePath string) string {
+	file, err := os.Open(filePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return string(data)
+}
+
+func processProtobuf(
+	filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap,
+	errCh chan error, wg *sync.WaitGroup,
+) {
+	defer wg.Done()
+
+	fileContent := strings.TrimRight(readFileToString(filePath), "\n")
+	testData := test_spec_proto.TestSpec{}
+	err := proto.Unmarshal([]byte(fileContent), &testData)
+	if err != nil {
+		errCh <- err
+		return
+	}
+
+	ownershipMetadata := testData.GetOwnershipMetadataList()
+	for _, metadata := range ownershipMetadata {
+		key := metadata.GetTargetName()
+		lock := keyLocks.GetLockForKey(key)
+		lock.Lock()
+
+		value, loaded := ownershipMetadataMap.LoadOrStore(
+			key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata},
+		)
+		if loaded {
+			existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
+			isDuplicate := false
+			for _, existing := range existingMetadata {
+				if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() {
+					errCh <- fmt.Errorf(
+						"Conflicting trendy team IDs found for %s at:\n%s with teamId"+
+							": %s,\n%s with teamId: %s",
+						key,
+						metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(),
+						existing.GetTrendyTeamId(),
+					)
+
+					lock.Unlock()
+					return
+				}
+				if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() {
+					isDuplicate = true
+					break
+				}
+			}
+			if !isDuplicate {
+				existingMetadata = append(existingMetadata, metadata)
+				ownershipMetadataMap.Store(key, existingMetadata)
+			}
+		}
+
+		lock.Unlock()
+	}
+}
+
+func main() {
+	inputFile := flag.String("inputFile", "", "Input file path")
+	outputFile := flag.String("outputFile", "", "Output file path")
+	flag.Parse()
+
+	if *inputFile == "" || *outputFile == "" {
+		fmt.Println("Usage: metadata -inputFile <input file path> -outputFile <output file path>")
+		os.Exit(1)
+	}
+
+	inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n")
+	filePaths := strings.Split(inputFileData, "\n")
+	ownershipMetadataMap := &sync.Map{}
+	keyLocks := &keyToLocksMap{}
+	errCh := make(chan error, len(filePaths))
+	var wg sync.WaitGroup
+
+	for _, filePath := range filePaths {
+		wg.Add(1)
+		go processProtobuf(filePath, ownershipMetadataMap, keyLocks, errCh, &wg)
+	}
+
+	wg.Wait()
+	close(errCh)
+
+	for err := range errCh {
+		log.Fatal(err)
+	}
+
+	allKeys := getSortedKeys(ownershipMetadataMap)
+	var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata
+
+	for _, key := range allKeys {
+		value, _ := ownershipMetadataMap.Load(key)
+		metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
+		allMetadata = append(allMetadata, metadataList...)
+	}
+
+	writeOutput(*outputFile, allMetadata)
+}

tools/metadata/go.mod

@@ -0,0 +1,7 @@
+module android/soong/tools/metadata
+
+require google.golang.org/protobuf v0.0.0
+
+replace google.golang.org/protobuf v0.0.0 => ../../../external/golang-protobuf
+
+go 1.18

tools/metadata/go.work

@@ -0,0 +1,10 @@
+go 1.18
+
+use (
+	.
+	../../../../external/golang-protobuf
+	../../../soong/testing/test_spec_proto
+
+)
+
+replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf

tools/metadata/testdata/expectedOutputFile.txt

@@ -0,0 +1,22 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345

tools/metadata/testdata/file1.txt

@@ -0,0 +1,13 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345

tools/metadata/testdata/file2.txt

@@ -0,0 +1,25 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346

tools/metadata/testdata/file3.txt

@@ -0,0 +1,13 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345

tools/metadata/testdata/file4.txt

@@ -0,0 +1,25 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-one
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346

tools/metadata/testdata/generatedOutputFile.txt

@@ -0,0 +1,22 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345

tools/metadata/testdata/inputFiles.txt

@@ -0,0 +1,2 @@
+file1.txt
+file2.txt

tools/metadata/testdata/inputFilesNegativeCase.txt

@@ -0,0 +1,2 @@
+file3.txt
+file4.txt

tools/metadata/testdata/metadata_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestMetadata(t *testing.T) {
+	cmd := exec.Command(
+		"metadata", "-inputFile", "./inputFiles.txt", "-outputFile",
+		"./generatedOutputFile.txt",
+	)
+	stderr, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err)
+	}
+
+	// Read the contents of the expected output file
+	expectedOutput, err := ioutil.ReadFile("./expectedOutputFile.txt")
+	if err != nil {
+		t.Fatalf("Error reading expected output file: %s", err)
+	}
+
+	// Read the contents of the generated output file
+	generatedOutput, err := ioutil.ReadFile("./generatedOutputFile.txt")
+	if err != nil {
+		t.Fatalf("Error reading generated output file: %s", err)
+	}
+
+	fmt.Println()
+
+	// Compare the contents
+	if string(expectedOutput) != string(generatedOutput) {
+		t.Errorf("Generated file contents do not match the expected output")
+	}
+}
+
+func TestMetadataNegativeCase(t *testing.T) {
+	cmd := exec.Command(
+		"metadata", "-inputFile", "./inputFilesNegativeCase.txt", "-outputFile",
+		"./generatedOutputFileNegativeCase.txt",
+	)
+	stderr, err := cmd.CombinedOutput()
+	if err == nil {
+		t.Fatalf(
+			"Expected an error, but the metadata command executed successfully. Output: %s",
+			stderr,
+		)
+	}
+
+	expectedError := "Conflicting trendy team IDs found for java-test-module" +
+		"-name-one at:\nAndroid.bp with teamId: 12346," +
+		"\nAndroid.bp with teamId: 12345"
+	if !strings.Contains(
+		strings.TrimSpace(string(stderr)), strings.TrimSpace(expectedError),
+	) {
+		t.Errorf(
+			"Unexpected error message. Expected to contain: %s, Got: %s",
+			expectedError, stderr,
+		)
+	}
+}

tools/metadata/testdata/outputFile.txt

@@ -0,0 +1,22 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345