Remove files that should no longer be installed

Make now exports (partial) lists of what should be installed. Those
lists are incomplete, but if something is removed from the list, we can
remove it from the filesystem so that incremental builds work better.

Test: built-in tests (m blueprint_tools)
Test: remove a module from PRODUCT_PACKAGES, see the print and file removed
Test: change the name of a cts test, see the old one removed from cts
Change-Id: I6bc14c6a5af63383ea265e97ed8b875fa51af0ed

ui/build/Android.bp

@@ -59,6 +59,7 @@ bootstrap_go_package {
         "util.go",
     ],
     testSrcs: [
+        "cleanbuild_test.go",
         "config_test.go",
         "environment_test.go",
         "util_test.go",

ui/build/cleanbuild.go

@@ -15,10 +15,12 @@
 package build
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"android/soong/ui/metrics"
@@ -177,3 +179,78 @@ func installCleanIfNecessary(ctx Context, config Config) {
 
 	writeConfig()
 }
+
+// cleanOldFiles takes an input file (with all paths relative to basePath), and removes files from
+// the filesystem if they were removed from the input file since the last execution.
+func cleanOldFiles(ctx Context, basePath, file string) {
+	file = filepath.Join(basePath, file)
+	oldFile := file + ".previous"
+
+	if _, err := os.Stat(file); err != nil {
+		ctx.Fatalf("Expected %q to be readable", file)
+	}
+
+	if _, err := os.Stat(oldFile); os.IsNotExist(err) {
+		if err := os.Rename(file, oldFile); err != nil {
+			ctx.Fatalf("Failed to rename file list (%q->%q): %v", file, oldFile, err)
+		}
+		return
+	}
+
+	var newPaths, oldPaths []string
+	if newData, err := ioutil.ReadFile(file); err == nil {
+		if oldData, err := ioutil.ReadFile(oldFile); err == nil {
+			// Common case: nothing has changed
+			if bytes.Equal(newData, oldData) {
+				return
+			}
+			newPaths = strings.Fields(string(newData))
+			oldPaths = strings.Fields(string(oldData))
+		} else {
+			ctx.Fatalf("Failed to read list of installable files (%q): %v", oldFile, err)
+		}
+	} else {
+		ctx.Fatalf("Failed to read list of installable files (%q): %v", file, err)
+	}
+
+	// These should be mostly sorted by make already, but better make sure Go concurs
+	sort.Strings(newPaths)
+	sort.Strings(oldPaths)
+
+	for len(oldPaths) > 0 {
+		if len(newPaths) > 0 {
+			if oldPaths[0] == newPaths[0] {
+				// Same file; continue
+				newPaths = newPaths[1:]
+				oldPaths = oldPaths[1:]
+				continue
+			} else if oldPaths[0] > newPaths[0] {
+				// New file; ignore
+				newPaths = newPaths[1:]
+				continue
+			}
+		}
+		// File only exists in the old list; remove if it exists
+		old := filepath.Join(basePath, oldPaths[0])
+		oldPaths = oldPaths[1:]
+		if fi, err := os.Stat(old); err == nil {
+			if fi.IsDir() {
+				if err := os.Remove(old); err == nil {
+					ctx.Println("Removed directory that is no longer installed: ", old)
+				} else {
+					ctx.Println("Failed to remove directory that is no longer installed (%q): %v", old, err)
+					ctx.Println("It's recommended to run `m installclean`")
+				}
+			} else {
+				if err := os.Remove(old); err == nil {
+					ctx.Println("Removed file that is no longer installed: ", old)
+				} else if !os.IsNotExist(err) {
+					ctx.Fatalf("Failed to remove file that is no longer installed (%q): %v", old, err)
+				}
+			}
+		}
+	}
+
+	// Use the new list as the base for the next build
+	os.Rename(file, oldFile)
+}

ui/build/cleanbuild_test.go

@@ -0,0 +1,100 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"android/soong/ui/logger"
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestCleanOldFiles(t *testing.T) {
+	dir, err := ioutil.TempDir("", "testcleanoldfiles")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	ctx := testContext()
+	logBuf := &bytes.Buffer{}
+	ctx.Logger = logger.New(logBuf)
+
+	touch := func(names ...string) {
+		for _, name := range names {
+			if f, err := os.Create(filepath.Join(dir, name)); err != nil {
+				t.Fatal(err)
+			} else {
+				f.Close()
+			}
+		}
+	}
+	runCleanOldFiles := func(names ...string) {
+		data := []byte(strings.Join(names, " "))
+		if err := ioutil.WriteFile(filepath.Join(dir, ".installed"), data, 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		cleanOldFiles(ctx, dir, ".installed")
+	}
+
+	assertFileList := func(names ...string) {
+		t.Helper()
+
+		sort.Strings(names)
+
+		var foundNames []string
+		if foundFiles, err := ioutil.ReadDir(dir); err == nil {
+			for _, fi := range foundFiles {
+				foundNames = append(foundNames, fi.Name())
+			}
+		} else {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(names, foundNames) {
+			t.Errorf("Expected a different list of files:\nwant: %v\n got: %v", names, foundNames)
+			t.Error("Log: ", logBuf.String())
+			logBuf.Reset()
+		}
+	}
+
+	// Initial list of potential files
+	runCleanOldFiles("foo", "bar")
+	touch("foo", "bar", "baz")
+	assertFileList("foo", "bar", "baz", ".installed.previous")
+
+	// This should be a no-op, as the list hasn't changed
+	runCleanOldFiles("foo", "bar")
+	assertFileList("foo", "bar", "baz", ".installed", ".installed.previous")
+
+	// This should be a no-op, as only a file was added
+	runCleanOldFiles("foo", "bar", "foo2")
+	assertFileList("foo", "bar", "baz", ".installed.previous")
+
+	// "bar" should be removed, foo2 should be ignored as it was never there
+	runCleanOldFiles("foo")
+	assertFileList("foo", "baz", ".installed.previous")
+
+	// Recreate bar, and create foo2. Ensure that they aren't removed
+	touch("bar", "foo2")
+	runCleanOldFiles("foo", "baz")
+	assertFileList("foo", "bar", "baz", "foo2", ".installed.previous")
+}

ui/build/kati.go

@@ -153,6 +153,7 @@ func runKatiBuild(ctx Context, config Config) {
 	runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {})
 
 	cleanCopyHeaders(ctx, config)
+	cleanOldInstalledFiles(ctx, config)
 }
 
 func cleanCopyHeaders(ctx Context, config Config) {
@@ -192,6 +193,23 @@ func cleanCopyHeaders(ctx Context, config Config) {
 		})
 }
 
+func cleanOldInstalledFiles(ctx Context, config Config) {
+	ctx.BeginTrace("clean", "clean old installed files")
+	defer ctx.EndTrace()
+
+	// We shouldn't be removing files from one side of the two-step asan builds
+	var suffix string
+	if v, ok := config.Environment().Get("SANITIZE_TARGET"); ok {
+		if sanitize := strings.Fields(v); inList("address", sanitize) {
+			suffix = "_asan"
+		}
+	}
+
+	cleanOldFiles(ctx, config.ProductOut(), ".installable_files"+suffix)
+
+	cleanOldFiles(ctx, config.HostOut(), ".installable_test_files")
+}
+
 func runKatiPackage(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunKati, "kati package")
 	defer ctx.EndTrace()