Dist build.ninja and Android.bp.list

Debugging issues on the build servers can be difficult because the
intermediate files are not visible.  Gzip ninja file and Makefiles
generated by Soong and the ninja files generated by Kati to the dist
directory, and also copy all of the finder output files.

Bug: 157656545
Test: m dist nothing
Change-Id: I48d75305e551ccae81c7a55721981cf58acd838b

ui/build/build.go

@@ -243,6 +243,8 @@ func Build(ctx Context, config Config, what int) {
 	// Write combined ninja file
 	createCombinedBuildNinjaFile(ctx, config)
 
+	distGzipFile(ctx, config, config.CombinedNinjaFile())
+
 	if what&RunBuildTests != 0 {
 		testForDanglingRules(ctx, config)
 	}
@@ -256,3 +258,47 @@ func Build(ctx Context, config Config, what int) {
 		runNinja(ctx, config)
 	}
 }
+
+// distGzipFile writes a compressed copy of src to the distDir if dist is enabled.  Failures
+// are printed but non-fatal.
+func distGzipFile(ctx Context, config Config, src string, subDirs ...string) {
+	if !config.Dist() {
+		return
+	}
+
+	subDir := filepath.Join(subDirs...)
+	destDir := filepath.Join(config.DistDir(), "soong_ui", subDir)
+
+	err := os.MkdirAll(destDir, 0777)
+	if err != nil {
+		ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
+
+	}
+
+	err = gzipFileToDir(src, destDir)
+	if err != nil {
+		ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
+	}
+}
+
+// distFile writes a copy of src to the distDir if dist is enabled.  Failures are printed but
+// non-fatal.
+func distFile(ctx Context, config Config, src string, subDirs ...string) {
+	if !config.Dist() {
+		return
+	}
+
+	subDir := filepath.Join(subDirs...)
+	destDir := filepath.Join(config.DistDir(), "soong_ui", subDir)
+
+	err := os.MkdirAll(destDir, 0777)
+	if err != nil {
+		ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
+
+	}
+
+	_, err = copyFile(src, filepath.Join(destDir, filepath.Base(src)))
+	if err != nil {
+		ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
+	}
+}

ui/build/finder.go

@@ -86,7 +86,7 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
 	os.MkdirAll(dumpDir, 0777)
 
 	androidMks := f.FindFirstNamedAt(".", "Android.mk")
-	err := dumpListToFile(androidMks, filepath.Join(dumpDir, "Android.mk.list"))
+	err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list"))
 	if err != nil {
 		ctx.Fatalf("Could not export module list: %v", err)
 	}
@@ -94,25 +94,25 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
 	androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk")
 	androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...)
 	androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...)
-	err = dumpListToFile(androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list"))
+	err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list"))
 	if err != nil {
 		ctx.Fatalf("Could not export product list: %v", err)
 	}
 
 	cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
-	err = dumpListToFile(cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
+	err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
 	if err != nil {
 		ctx.Fatalf("Could not export module list: %v", err)
 	}
 
 	owners := f.FindNamedAt(".", "OWNERS")
-	err = dumpListToFile(owners, filepath.Join(dumpDir, "OWNERS.list"))
+	err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list"))
 	if err != nil {
 		ctx.Fatalf("Could not find OWNERS: %v", err)
 	}
 
 	testMappings := f.FindNamedAt(".", "TEST_MAPPING")
-	err = dumpListToFile(testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list"))
+	err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list"))
 	if err != nil {
 		ctx.Fatalf("Could not find TEST_MAPPING: %v", err)
 	}
@@ -122,18 +122,24 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
 	if len(androidBps) == 0 {
 		ctx.Fatalf("No Android.bp found")
 	}
-	err = dumpListToFile(androidBps, filepath.Join(dumpDir, "Android.bp.list"))
+	err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list"))
 	if err != nil {
 		ctx.Fatalf("Could not find modules: %v", err)
 	}
 }
 
-func dumpListToFile(list []string, filePath string) (err error) {
+func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) {
 	desiredText := strings.Join(list, "\n")
 	desiredBytes := []byte(desiredText)
 	actualBytes, readErr := ioutil.ReadFile(filePath)
 	if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
 		err = ioutil.WriteFile(filePath, desiredBytes, 0777)
+		if err != nil {
+			return err
+		}
 	}
-	return err
+
+	distFile(ctx, config, filePath, "module_paths")
+
+	return nil
 }

ui/build/kati.go

@@ -156,6 +156,8 @@ func runKatiBuild(ctx Context, config Config) {
 
 	runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {})
 
+	distGzipFile(ctx, config, config.KatiBuildNinjaFile())
+
 	cleanCopyHeaders(ctx, config)
 	cleanOldInstalledFiles(ctx, config)
 }
@@ -251,6 +253,8 @@ func runKatiPackage(ctx Context, config Config) {
 			env.Set("DIST_DIR", config.DistDir())
 		}
 	})
+
+	distGzipFile(ctx, config, config.KatiPackageNinjaFile())
 }
 
 func runKatiCleanSpec(ctx Context, config Config) {

ui/build/soong.go

@@ -139,6 +139,13 @@ func runSoong(ctx Context, config Config) {
 	soongBuildMetrics := loadSoongBuildMetrics(ctx, config)
 	logSoongBuildMetrics(ctx, soongBuildMetrics)
 
+	distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
+
+	if !config.SkipMake() {
+		distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
+		distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
+	}
+
 	if ctx.Metrics != nil {
 		ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
 	}

ui/build/util.go

@@ -15,6 +15,8 @@
 package build
 
 import (
+	"compress/gzip"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -142,3 +144,29 @@ func copyFile(src, dst string) (int64, error) {
 
 	return io.Copy(destination, source)
 }
+
+// gzipFileToDir writes a compressed copy of src to destDir with the suffix ".gz".
+func gzipFileToDir(src, destDir string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open %s: %s", src, err.Error())
+	}
+	defer in.Close()
+
+	dest := filepath.Join(destDir, filepath.Base(src)+".gz")
+
+	out, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to open %s: %s", dest, err.Error())
+	}
+	defer out.Close()
+	gz := gzip.NewWriter(out)
+	defer gz.Close()
+
+	_, err = io.Copy(gz, in)
+	if err != nil {
+		return fmt.Errorf("failed to gzip %s: %s", dest, err.Error())
+	}
+
+	return nil
+}