diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 7a8fca999..b79754cbe 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -50,6 +50,7 @@ bootstrap_go_package { "cleanbuild.go", "config.go", "context.go", + "staging_snapshot.go", "dumpvars.go", "environment.go", "exec.go", @@ -70,10 +71,11 @@ bootstrap_go_package { "cleanbuild_test.go", "config_test.go", "environment_test.go", + "proc_sync_test.go", "rbe_test.go", + "staging_snapshot_test.go", "upload_test.go", "util_test.go", - "proc_sync_test.go", ], darwin: { srcs: [ diff --git a/ui/build/build.go b/ui/build/build.go index d49a7543e..edc595d16 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -102,9 +102,9 @@ const ( // Whether to include the kati-generated ninja file in the combined ninja. RunKatiNinja = 1 << iota // Whether to run ninja on the combined ninja. - RunNinja = 1 << iota - RunBuildTests = 1 << iota - RunAll = RunProductConfig | RunSoong | RunKati | RunKatiNinja | RunNinja + RunNinja = 1 << iota + RunDistActions = 1 << iota + RunBuildTests = 1 << iota ) // checkBazelMode fails the build if there are conflicting arguments for which bazel @@ -322,34 +322,42 @@ func Build(ctx Context, config Config) { runNinjaForBuild(ctx, config) } + + if what&RunDistActions != 0 { + runDistActions(ctx, config) + } } func evaluateWhatToRun(config Config, verboseln func(v ...interface{})) int { //evaluate what to run - what := RunAll + what := 0 if config.Checkbuild() { what |= RunBuildTests } - if config.SkipConfig() { + if !config.SkipConfig() { + what |= RunProductConfig + } else { verboseln("Skipping Config as requested") - what = what &^ RunProductConfig } - if config.SkipKati() { - verboseln("Skipping Kati as requested") - what = what &^ RunKati - } - if config.SkipKatiNinja() { - verboseln("Skipping use of Kati ninja as requested") - what = what &^ RunKatiNinja - } - if config.SkipSoong() { + if !config.SkipSoong() { + what |= RunSoong + } else { verboseln("Skipping use of Soong as requested") - what = what &^ RunSoong } - - if config.SkipNinja() { + if !config.SkipKati() { + what |= RunKati + } else { + verboseln("Skipping Kati as requested") + } + if !config.SkipKatiNinja() { + what |= RunKatiNinja + } else { + verboseln("Skipping use of Kati ninja as requested") + } + if !config.SkipNinja() { + what |= RunNinja + } else { verboseln("Skipping Ninja as requested") - what = what &^ RunNinja } if !config.SoongBuildInvocationNeeded() { @@ -361,6 +369,11 @@ func evaluateWhatToRun(config Config, verboseln func(v ...interface{})) int { what = what &^ RunNinja what = what &^ RunKati } + + if config.Dist() { + what |= RunDistActions + } + return what } @@ -419,3 +432,9 @@ func distFile(ctx Context, config Config, src string, subDirs ...string) { } }() } + +// Actions to run on every build where 'dist' is in the actions. +// Be careful, anything added here slows down EVERY CI build +func runDistActions(ctx Context, config Config) { + runStagingSnapshot(ctx, config) +} diff --git a/ui/build/staging_snapshot.go b/ui/build/staging_snapshot.go new file mode 100644 index 000000000..377aa64cd --- /dev/null +++ b/ui/build/staging_snapshot.go @@ -0,0 +1,246 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "crypto/sha1" + "encoding/hex" + "encoding/json" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "android/soong/shared" + "android/soong/ui/metrics" +) + +// Metadata about a staged file +type fileEntry struct { + Name string `json:"name"` + Mode fs.FileMode `json:"mode"` + Size int64 `json:"size"` + Sha1 string `json:"sha1"` +} + +func fileEntryEqual(a fileEntry, b fileEntry) bool { + return a.Name == b.Name && a.Mode == b.Mode && a.Size == b.Size && a.Sha1 == b.Sha1 +} + +func sha1_hash(filename string) (string, error) { + f, err := os.Open(filename) + if err != nil { + return "", err + } + defer f.Close() + + h := sha1.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +// Subdirs of PRODUCT_OUT to scan +var stagingSubdirs = []string{ + "apex", + "cache", + "coverage", + "data", + "debug_ramdisk", + "fake_packages", + "installer", + "oem", + "product", + "ramdisk", + "recovery", + "root", + "sysloader", + "system", + "system_dlkm", + "system_ext", + "system_other", + "testcases", + "test_harness_ramdisk", + "vendor", + "vendor_debug_ramdisk", + "vendor_kernel_ramdisk", + "vendor_ramdisk", +} + +// Return an array of stagedFileEntrys, one for each file in the staging directories inside +// productOut +func takeStagingSnapshot(ctx Context, productOut string, subdirs []string) ([]fileEntry, error) { + var outer_err error + if !strings.HasSuffix(productOut, "/") { + productOut += "/" + } + result := []fileEntry{} + for _, subdir := range subdirs { + filepath.WalkDir(productOut+subdir, + func(filename string, dirent fs.DirEntry, err error) error { + // Ignore errors. The most common one is that one of the subdirectories + // hasn't been built, in which case we just report it as empty. + if err != nil { + ctx.Verbosef("scanModifiedStagingOutputs error: %s", err) + return nil + } + if dirent.Type().IsRegular() { + fileInfo, _ := dirent.Info() + relative := strings.TrimPrefix(filename, productOut) + sha, err := sha1_hash(filename) + if err != nil { + outer_err = err + } + result = append(result, fileEntry{ + Name: relative, + Mode: fileInfo.Mode(), + Size: fileInfo.Size(), + Sha1: sha, + }) + } + return nil + }) + } + + sort.Slice(result, func(l, r int) bool { return result[l].Name < result[r].Name }) + + return result, outer_err +} + +// Read json into an array of fileEntry. On error return empty array. +func readJson(filename string) ([]fileEntry, error) { + buf, err := os.ReadFile(filename) + if err != nil { + // Not an error, just missing, which is empty. + return []fileEntry{}, nil + } + + var result []fileEntry + err = json.Unmarshal(buf, &result) + if err != nil { + // Bad formatting. This is an error + return []fileEntry{}, err + } + + return result, nil +} + +// Write obj to filename. +func writeJson(filename string, obj interface{}) error { + buf, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return err + } + + return os.WriteFile(filename, buf, 0660) +} + +type snapshotDiff struct { + Added []string `json:"added"` + Changed []string `json:"changed"` + Removed []string `json:"removed"` +} + +// Diff the two snapshots, returning a snapshotDiff. +func diffSnapshots(previous []fileEntry, current []fileEntry) snapshotDiff { + result := snapshotDiff{ + Added: []string{}, + Changed: []string{}, + Removed: []string{}, + } + + found := make(map[string]bool) + + prev := make(map[string]fileEntry) + for _, pre := range previous { + prev[pre.Name] = pre + } + + for _, cur := range current { + pre, ok := prev[cur.Name] + found[cur.Name] = true + // Added + if !ok { + result.Added = append(result.Added, cur.Name) + continue + } + // Changed + if !fileEntryEqual(pre, cur) { + result.Changed = append(result.Changed, cur.Name) + } + } + + // Removed + for _, pre := range previous { + if !found[pre.Name] { + result.Removed = append(result.Removed, pre.Name) + } + } + + // Sort the results + sort.Strings(result.Added) + sort.Strings(result.Changed) + sort.Strings(result.Removed) + + return result +} + +// Write a json files to dist: +// - A list of which files have changed in this build. +// +// And record in out/soong: +// - A list of all files in the staging directories, including their hashes. +func runStagingSnapshot(ctx Context, config Config) { + ctx.BeginTrace(metrics.RunSoong, "runStagingSnapshot") + defer ctx.EndTrace() + + snapshotFilename := shared.JoinPath(config.SoongOutDir(), "staged_files.json") + + // Read the existing snapshot file. If it doesn't exist, this is a full + // build, so all files will be treated as new. + previous, err := readJson(snapshotFilename) + if err != nil { + ctx.Fatal(err) + return + } + + // Take a snapshot of the current out directory + current, err := takeStagingSnapshot(ctx, config.ProductOut(), stagingSubdirs) + if err != nil { + ctx.Fatal(err) + return + } + + // Diff the snapshots + diff := diffSnapshots(previous, current) + + // Write the diff (use RealDistDir, not one that might have been faked for bazel) + err = writeJson(shared.JoinPath(config.RealDistDir(), "modified_files.json"), diff) + if err != nil { + ctx.Fatal(err) + return + } + + // Update the snapshot + err = writeJson(snapshotFilename, current) + if err != nil { + ctx.Fatal(err) + return + } +} diff --git a/ui/build/staging_snapshot_test.go b/ui/build/staging_snapshot_test.go new file mode 100644 index 000000000..7ac544364 --- /dev/null +++ b/ui/build/staging_snapshot_test.go @@ -0,0 +1,188 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func assertDeepEqual(t *testing.T, expected interface{}, actual interface{}) { + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("expected:\n %#v\n actual:\n %#v", expected, actual) + } +} + +// Make a temp directory containing the supplied contents +func makeTempDir(files []string, directories []string, symlinks []string) string { + temp, _ := os.MkdirTemp("", "soon_staging_snapshot_test_") + + for _, file := range files { + os.MkdirAll(temp+"/"+filepath.Dir(file), 0700) + os.WriteFile(temp+"/"+file, []byte(file), 0600) + } + + for _, dir := range directories { + os.MkdirAll(temp+"/"+dir, 0770) + } + + for _, symlink := range symlinks { + os.MkdirAll(temp+"/"+filepath.Dir(symlink), 0770) + os.Symlink(temp, temp+"/"+symlink) + } + + return temp +} + +// If this is a clean build, we won't have any preexisting files, make sure we get back an empty +// list and not errors. +func TestEmptyOut(t *testing.T) { + ctx := testContext() + + temp := makeTempDir(nil, nil, nil) + defer os.RemoveAll(temp) + + actual, _ := takeStagingSnapshot(ctx, temp, []string{"a", "e", "g"}) + + expected := []fileEntry{} + + assertDeepEqual(t, expected, actual) +} + +// Make sure only the listed directories are picked up, and only regular files +func TestNoExtraSubdirs(t *testing.T) { + ctx := testContext() + + temp := makeTempDir([]string{"a/b", "a/c", "d", "e/f"}, []string{"g/h"}, []string{"e/symlink"}) + defer os.RemoveAll(temp) + + actual, _ := takeStagingSnapshot(ctx, temp, []string{"a", "e", "g"}) + + expected := []fileEntry{ + {"a/b", 0600, 3, "3ec69c85a4ff96830024afeef2d4e512181c8f7b"}, + {"a/c", 0600, 3, "592d70e4e03ee6f6780c71b0bf3b9608dbf1e201"}, + {"e/f", 0600, 3, "9e164bef74aceede0974b857170100409efe67f1"}, + } + + assertDeepEqual(t, expected, actual) +} + +// Make sure diff handles empty lists +func TestDiffEmpty(t *testing.T) { + actual := diffSnapshots(nil, []fileEntry{}) + + expected := snapshotDiff{ + Added: []string{}, + Changed: []string{}, + Removed: []string{}, + } + + assertDeepEqual(t, expected, actual) +} + +// Make sure diff handles adding +func TestDiffAdd(t *testing.T) { + actual := diffSnapshots([]fileEntry{ + {"a", 0600, 1, "1234"}, + }, []fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 2, "5678"}, + }) + + expected := snapshotDiff{ + Added: []string{"b"}, + Changed: []string{}, + Removed: []string{}, + } + + assertDeepEqual(t, expected, actual) +} + +// Make sure diff handles changing mode +func TestDiffChangeMode(t *testing.T) { + actual := diffSnapshots([]fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 2, "5678"}, + }, []fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0600, 2, "5678"}, + }) + + expected := snapshotDiff{ + Added: []string{}, + Changed: []string{"b"}, + Removed: []string{}, + } + + assertDeepEqual(t, expected, actual) +} + +// Make sure diff handles changing size +func TestDiffChangeSize(t *testing.T) { + actual := diffSnapshots([]fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 2, "5678"}, + }, []fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 3, "5678"}, + }) + + expected := snapshotDiff{ + Added: []string{}, + Changed: []string{"b"}, + Removed: []string{}, + } + + assertDeepEqual(t, expected, actual) +} + +// Make sure diff handles changing contents +func TestDiffChangeContents(t *testing.T) { + actual := diffSnapshots([]fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 2, "5678"}, + }, []fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 2, "aaaa"}, + }) + + expected := snapshotDiff{ + Added: []string{}, + Changed: []string{"b"}, + Removed: []string{}, + } + + assertDeepEqual(t, expected, actual) +} + +// Make sure diff handles removing +func TestDiffRemove(t *testing.T) { + actual := diffSnapshots([]fileEntry{ + {"a", 0600, 1, "1234"}, + {"b", 0700, 2, "5678"}, + }, []fileEntry{ + {"a", 0600, 1, "1234"}, + }) + + expected := snapshotDiff{ + Added: []string{}, + Changed: []string{}, + Removed: []string{"b"}, + } + + assertDeepEqual(t, expected, actual) +}