Cacheable, multithreaded finder.

It can find every Android.bp in internal master in
about 2.5 sec the first time and 0.3 sec subsequent times

Bug: 62455338

Test: m -j blueprint_tools # which runs the unit tests

Test: m -j blueprint_tools && \
  out/soong/host/linux-x86/bin/finder \
    -v
    --db /tmp/mydb \
    --names Android.mk \
    --prune-files .android-out-dir \
    --exclude-dirs .git,.repo \
    . \
    >/tmp/finder-log 2>&1

Change-Id: I5ab2650459a1dae0d5d076faf411ec2d053c743d

Android.bp

@@ -2,6 +2,8 @@ subdirs = [
     "androidmk",
     "bpfix",
     "cmd/*",
+    "fs",
+    "finder",
     "third_party/zip",
     "ui/*",
 ]

finder/Android.bp

@@ -0,0 +1,37 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// fast, parallel, caching implementation of `find`
+//
+
+subdirs = [
+    "cmd"
+]
+
+bootstrap_go_package {
+    name: "soong-finder",
+    pkgPath: "android/soong/finder",
+    srcs: [
+        "finder.go",
+    ],
+    testSrcs: [
+        "finder_test.go",
+    ],
+    deps: [
+      "soong-fs",
+    ],
+}
+
+

finder/cmd/Android.bp

@@ -0,0 +1,29 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// fast, parallel, caching implementation of `find`
+//
+
+blueprint_go_binary {
+    name: "finder",
+    srcs: [
+        "finder.go",
+    ],
+    deps: [
+        "soong-finder"
+    ],
+}
+
+

finder/cmd/finder.go

@@ -0,0 +1,149 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"runtime/pprof"
+	"sort"
+	"strings"
+	"time"
+
+	"android/soong/finder"
+	"android/soong/fs"
+)
+
+var (
+	// configuration of what to find
+	excludeDirs     string
+	filenamesToFind string
+	pruneFiles      string
+
+	// other configuration
+	cpuprofile    string
+	verbose       bool
+	dbPath        string
+	numIterations int
+)
+
+func init() {
+	flag.StringVar(&cpuprofile, "cpuprofile", "",
+		"filepath of profile file to write (optional)")
+	flag.BoolVar(&verbose, "v", false, "log additional information")
+	flag.StringVar(&dbPath, "db", "", "filepath of cache db")
+
+	flag.StringVar(&excludeDirs, "exclude-dirs", "",
+		"comma-separated list of directory names to exclude from search")
+	flag.StringVar(&filenamesToFind, "names", "",
+		"comma-separated list of filenames to find")
+	flag.StringVar(&pruneFiles, "prune-files", "",
+		"filenames that if discovered will exclude their entire directory "+
+			"(including sibling files and directories)")
+	flag.IntVar(&numIterations, "count", 1,
+		"number of times to run. This is intended for use with --cpuprofile"+
+			" , to increase profile accuracy")
+}
+
+var usage = func() {
+	fmt.Printf("usage: finder -name <fileName> --db <dbPath> <searchDirectory> [<searchDirectory>...]\n")
+	flag.PrintDefaults()
+}
+
+func main() {
+	err := run()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err.Error())
+		os.Exit(1)
+	}
+}
+
+func stringToList(input string) []string {
+	return strings.Split(input, ",")
+}
+
+func run() error {
+	startTime := time.Now()
+	flag.Parse()
+
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
+		if err != nil {
+			return fmt.Errorf("Error opening cpuprofile: %s", err)
+		}
+		pprof.StartCPUProfile(f)
+		defer f.Close()
+		defer pprof.StopCPUProfile()
+	}
+
+	var writer io.Writer
+	if verbose {
+		writer = os.Stderr
+	} else {
+		writer = ioutil.Discard
+	}
+
+	// TODO: replace Lshortfile with Llongfile when bug 63821638 is done
+	logger := log.New(writer, "", log.Ldate|log.Lmicroseconds|log.Lshortfile)
+
+	logger.Printf("Finder starting at %v\n", startTime)
+
+	rootPaths := flag.Args()
+	if len(rootPaths) < 1 {
+		usage()
+		return fmt.Errorf(
+			"Must give at least one <searchDirectory>")
+	}
+
+	workingDir, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	params := finder.CacheParams{
+		WorkingDirectory: workingDir,
+		RootDirs:         rootPaths,
+		ExcludeDirs:      stringToList(excludeDirs),
+		PruneFiles:       stringToList(pruneFiles),
+		IncludeFiles:     stringToList(filenamesToFind),
+	}
+	if dbPath == "" {
+		usage()
+		return errors.New("Param 'db' must be nonempty")
+	}
+	matches := []string{}
+	for i := 0; i < numIterations; i++ {
+		matches = runFind(params, logger)
+	}
+	findDuration := time.Since(startTime)
+	logger.Printf("Found these %v inodes in %v :\n", len(matches), findDuration)
+	sort.Strings(matches)
+	for _, match := range matches {
+		fmt.Println(match)
+	}
+	logger.Printf("End of %v inodes\n", len(matches))
+	logger.Printf("Finder completed in %v\n", time.Since(startTime))
+	return nil
+}
+
+func runFind(params finder.CacheParams, logger *log.Logger) (paths []string) {
+	service := finder.New(params, fs.OsFs, logger, dbPath)
+	defer service.Shutdown()
+	return service.FindAll()
+}

fs/Android.bp

@@ -0,0 +1,27 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// mock filesystem
+//
+
+bootstrap_go_package {
+    name: "soong-fs",
+    pkgPath: "android/soong/fs",
+    srcs: [
+        "fs.go",
+    ],
+}
+
+

fs/fs.go

@@ -0,0 +1,957 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"sync"
+	"syscall"
+	"time"
+)
+
+var OsFs FileSystem = osFs{}
+
+func NewMockFs(files map[string][]byte) *MockFs {
+	workDir := "/cwd"
+	fs := &MockFs{
+		Clock:   NewClock(time.Unix(2, 2)),
+		workDir: workDir,
+	}
+	fs.root = *fs.newDir()
+	fs.MkDirs(workDir)
+
+	for path, bytes := range files {
+		dir := filepath.Dir(path)
+		fs.MkDirs(dir)
+		fs.WriteFile(path, bytes, 0777)
+	}
+
+	return fs
+}
+
+type FileSystem interface {
+	// getting information about files
+	Open(name string) (file io.ReadCloser, err error)
+	Lstat(path string) (stats os.FileInfo, err error)
+	ReadDir(path string) (contents []os.FileInfo, err error)
+
+	InodeNumber(info os.FileInfo) (number uint64, err error)
+	DeviceNumber(info os.FileInfo) (number uint64, err error)
+	PermTime(info os.FileInfo) (time time.Time, err error)
+
+	// changing contents of the filesystem
+	Rename(oldPath string, newPath string) (err error)
+	WriteFile(path string, data []byte, perm os.FileMode) (err error)
+	Remove(path string) (err error)
+	RemoveAll(path string) (err error)
+
+	// metadata about the filesystem
+	ViewId() (id string) // Some unique id of the user accessing the filesystem
+}
+
+// osFs implements FileSystem using the local disk.
+type osFs struct{}
+
+func (osFs) Open(name string) (io.ReadCloser, error) { return os.Open(name) }
+
+func (osFs) Lstat(path string) (stats os.FileInfo, err error) {
+	return os.Lstat(path)
+}
+
+func (osFs) InodeNumber(info os.FileInfo) (number uint64, err error) {
+	sys := info.Sys()
+	linuxStats, ok := sys.(*syscall.Stat_t)
+	if ok {
+		return linuxStats.Ino, nil
+	}
+	return 0, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) DeviceNumber(info os.FileInfo) (number uint64, err error) {
+	sys := info.Sys()
+	linuxStats, ok := sys.(*syscall.Stat_t)
+	if ok {
+		return linuxStats.Dev, nil
+	}
+	return 0, fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) PermTime(info os.FileInfo) (when time.Time, err error) {
+	sys := info.Sys()
+	linuxStats, ok := sys.(*syscall.Stat_t)
+	if ok {
+		return time.Unix(linuxStats.Ctim.Sec, linuxStats.Ctim.Nsec), nil
+	}
+	return time.Date(0, 0, 0, 0, 0, 0, 0, nil), fmt.Errorf("%v is not a *syscall.Stat_t", sys)
+}
+
+func (osFs) ReadDir(path string) (contents []os.FileInfo, err error) {
+	return ioutil.ReadDir(path)
+}
+func (osFs) Rename(oldPath string, newPath string) error {
+	return os.Rename(oldPath, newPath)
+}
+
+func (osFs) WriteFile(path string, data []byte, perm os.FileMode) error {
+	return ioutil.WriteFile(path, data, perm)
+}
+
+func (osFs) Remove(path string) error {
+	return os.Remove(path)
+}
+
+func (osFs) RemoveAll(path string) error {
+	return os.RemoveAll(path)
+}
+
+func (osFs) ViewId() (id string) {
+	user, err := user.Current()
+	if err != nil {
+		return ""
+	}
+	username := user.Username
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		return ""
+	}
+
+	return username + "@" + hostname
+}
+
+type Clock struct {
+	time time.Time
+}
+
+func NewClock(startTime time.Time) *Clock {
+	return &Clock{time: startTime}
+
+}
+
+func (c *Clock) Tick() {
+	c.time = c.time.Add(time.Microsecond)
+}
+
+func (c *Clock) Time() time.Time {
+	return c.time
+}
+
+// given "/a/b/c/d", pathSplit returns ("/a/b/c", "d")
+func pathSplit(path string) (dir string, leaf string) {
+	dir, leaf = filepath.Split(path)
+	if dir != "/" && len(dir) > 0 {
+		dir = dir[:len(dir)-1]
+	}
+	return dir, leaf
+}
+
+// MockFs supports singlethreaded writes and multithreaded reads
+type MockFs struct {
+	// configuration
+	viewId       string //
+	deviceNumber uint64
+
+	// implementation
+	root            mockDir
+	Clock           *Clock
+	workDir         string
+	nextInodeNumber uint64
+
+	// history of requests, for tests to check
+	StatCalls      []string
+	ReadDirCalls   []string
+	aggregatesLock sync.Mutex
+}
+
+type mockInode struct {
+	modTime     time.Time
+	permTime    time.Time
+	sys         interface{}
+	inodeNumber uint64
+	readable    bool
+}
+
+func (m mockInode) ModTime() time.Time {
+	return m.modTime
+}
+
+func (m mockInode) Sys() interface{} {
+	return m.sys
+}
+
+type mockFile struct {
+	bytes []byte
+
+	mockInode
+}
+
+type mockLink struct {
+	target string
+
+	mockInode
+}
+
+type mockDir struct {
+	mockInode
+
+	subdirs  map[string]*mockDir
+	files    map[string]*mockFile
+	symlinks map[string]*mockLink
+}
+
+func (m *MockFs) resolve(path string, followLastLink bool) (result string, err error) {
+	if !filepath.IsAbs(path) {
+		path = filepath.Join(m.workDir, path)
+	}
+	path = filepath.Clean(path)
+
+	return m.followLinks(path, followLastLink, 10)
+}
+
+// note that followLinks can return a file path that doesn't exist
+func (m *MockFs) followLinks(path string, followLastLink bool, count int) (canonicalPath string, err error) {
+	if path == "/" {
+		return path, nil
+	}
+
+	parentPath, leaf := pathSplit(path)
+	if parentPath == path {
+		err = fmt.Errorf("Internal error: %v yields itself as a parent", path)
+		panic(err.Error())
+		return "", fmt.Errorf("Internal error: %v yields itself as a parent", path)
+	}
+
+	parentPath, err = m.followLinks(parentPath, true, count)
+	if err != nil {
+		return "", err
+	}
+
+	parentNode, err := m.getDir(parentPath, false)
+	if err != nil {
+		return "", err
+	}
+	if !parentNode.readable {
+		return "", &os.PathError{
+			Op:   "read",
+			Path: path,
+			Err:  os.ErrPermission,
+		}
+	}
+
+	link, isLink := parentNode.symlinks[leaf]
+	if isLink && followLastLink {
+		if count <= 0 {
+			// probably a loop
+			return "", &os.PathError{
+				Op:   "read",
+				Path: path,
+				Err:  fmt.Errorf("too many levels of symbolic links"),
+			}
+		}
+
+		if !link.readable {
+			return "", &os.PathError{
+				Op:   "read",
+				Path: path,
+				Err:  os.ErrPermission,
+			}
+		}
+
+		target := m.followLink(link, parentPath)
+		return m.followLinks(target, followLastLink, count-1)
+	}
+	return path, nil
+}
+
+func (m *MockFs) followLink(link *mockLink, parentPath string) (result string) {
+	return filepath.Clean(filepath.Join(parentPath, link.target))
+}
+
+func (m *MockFs) getFile(parentDir *mockDir, fileName string) (file *mockFile, err error) {
+	file, isFile := parentDir.files[fileName]
+	if !isFile {
+		_, isDir := parentDir.subdirs[fileName]
+		_, isLink := parentDir.symlinks[fileName]
+		if isDir || isLink {
+			return nil, &os.PathError{
+				Op:   "open",
+				Path: fileName,
+				Err:  os.ErrInvalid,
+			}
+		}
+
+		return nil, &os.PathError{
+			Op:   "open",
+			Path: fileName,
+			Err:  os.ErrNotExist,
+		}
+	}
+	if !file.readable {
+		return nil, &os.PathError{
+			Op:   "open",
+			Path: fileName,
+			Err:  os.ErrPermission,
+		}
+	}
+	return file, nil
+}
+
+func (m *MockFs) getInode(parentDir *mockDir, name string) (inode *mockInode, err error) {
+	file, isFile := parentDir.files[name]
+	if isFile {
+		return &file.mockInode, nil
+	}
+	link, isLink := parentDir.symlinks[name]
+	if isLink {
+		return &link.mockInode, nil
+	}
+	dir, isDir := parentDir.subdirs[name]
+	if isDir {
+		return &dir.mockInode, nil
+	}
+	return nil, &os.PathError{
+		Op:   "stat",
+		Path: name,
+		Err:  os.ErrNotExist,
+	}
+
+}
+
+func (m *MockFs) Open(path string) (io.ReadCloser, error) {
+	path, err := m.resolve(path, true)
+	if err != nil {
+		return nil, err
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	parentPath, base := pathSplit(path)
+	parentDir, err := m.getDir(parentPath, false)
+	if err != nil {
+		return nil, err
+	}
+	file, err := m.getFile(parentDir, base)
+	if err != nil {
+		return nil, err
+	}
+	return struct {
+		io.Closer
+		*bytes.Reader
+	}{
+		ioutil.NopCloser(nil),
+		bytes.NewReader(file.bytes),
+	}, nil
+
+}
+
+// a mockFileInfo is for exporting file stats in a way that satisfies the FileInfo interface
+type mockFileInfo struct {
+	path         string
+	size         int64
+	modTime      time.Time // time at which the inode's contents were modified
+	permTime     time.Time // time at which the inode's permissions were modified
+	isDir        bool
+	inodeNumber  uint64
+	deviceNumber uint64
+}
+
+func (m *mockFileInfo) Name() string {
+	return m.path
+}
+
+func (m *mockFileInfo) Size() int64 {
+	return m.size
+}
+
+func (m *mockFileInfo) Mode() os.FileMode {
+	return 0
+}
+
+func (m *mockFileInfo) ModTime() time.Time {
+	return m.modTime
+}
+
+func (m *mockFileInfo) IsDir() bool {
+	return m.isDir
+}
+
+func (m *mockFileInfo) Sys() interface{} {
+	return nil
+}
+
+func (m *MockFs) dirToFileInfo(d *mockDir, path string) (info *mockFileInfo) {
+	return &mockFileInfo{
+		path:         path,
+		size:         1,
+		modTime:      d.modTime,
+		permTime:     d.permTime,
+		isDir:        true,
+		inodeNumber:  d.inodeNumber,
+		deviceNumber: m.deviceNumber,
+	}
+
+}
+
+func (m *MockFs) fileToFileInfo(f *mockFile, path string) (info *mockFileInfo) {
+	return &mockFileInfo{
+		path:         path,
+		size:         1,
+		modTime:      f.modTime,
+		permTime:     f.permTime,
+		isDir:        false,
+		inodeNumber:  f.inodeNumber,
+		deviceNumber: m.deviceNumber,
+	}
+}
+
+func (m *MockFs) linkToFileInfo(l *mockLink, path string) (info *mockFileInfo) {
+	return &mockFileInfo{
+		path:         path,
+		size:         1,
+		modTime:      l.modTime,
+		permTime:     l.permTime,
+		isDir:        false,
+		inodeNumber:  l.inodeNumber,
+		deviceNumber: m.deviceNumber,
+	}
+}
+
+func (m *MockFs) Lstat(path string) (stats os.FileInfo, err error) {
+	// update aggregates
+	m.aggregatesLock.Lock()
+	m.StatCalls = append(m.StatCalls, path)
+	m.aggregatesLock.Unlock()
+
+	// resolve symlinks
+	path, err = m.resolve(path, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// special case for root dir
+	if path == "/" {
+		return m.dirToFileInfo(&m.root, "/"), nil
+	}
+
+	// determine type and handle appropriately
+	parentPath, baseName := pathSplit(path)
+	dir, err := m.getDir(parentPath, false)
+	if err != nil {
+		return nil, err
+	}
+	subdir, subdirExists := dir.subdirs[baseName]
+	if subdirExists {
+		return m.dirToFileInfo(subdir, path), nil
+	}
+	file, fileExists := dir.files[baseName]
+	if fileExists {
+		return m.fileToFileInfo(file, path), nil
+	}
+	link, linkExists := dir.symlinks[baseName]
+	if linkExists {
+		return m.linkToFileInfo(link, path), nil
+	}
+	// not found
+	return nil, &os.PathError{
+		Op:   "stat",
+		Path: path,
+		Err:  os.ErrNotExist,
+	}
+}
+
+func (m *MockFs) InodeNumber(info os.FileInfo) (number uint64, err error) {
+	mockInfo, ok := info.(*mockFileInfo)
+	if ok {
+		return mockInfo.inodeNumber, nil
+	}
+	return 0, fmt.Errorf("%v is not a mockFileInfo", info)
+}
+func (m *MockFs) DeviceNumber(info os.FileInfo) (number uint64, err error) {
+	mockInfo, ok := info.(*mockFileInfo)
+	if ok {
+		return mockInfo.deviceNumber, nil
+	}
+	return 0, fmt.Errorf("%v is not a mockFileInfo", info)
+}
+func (m *MockFs) PermTime(info os.FileInfo) (when time.Time, err error) {
+	mockInfo, ok := info.(*mockFileInfo)
+	if ok {
+		return mockInfo.permTime, nil
+	}
+	return time.Date(0, 0, 0, 0, 0, 0, 0, nil),
+		fmt.Errorf("%v is not a mockFileInfo", info)
+}
+
+func (m *MockFs) ReadDir(path string) (contents []os.FileInfo, err error) {
+	// update aggregates
+	m.aggregatesLock.Lock()
+	m.ReadDirCalls = append(m.ReadDirCalls, path)
+	m.aggregatesLock.Unlock()
+
+	// locate directory
+	path, err = m.resolve(path, true)
+	if err != nil {
+		return nil, err
+	}
+	results := []os.FileInfo{}
+	dir, err := m.getDir(path, false)
+	if err != nil {
+		return nil, err
+	}
+	if !dir.readable {
+		return nil, &os.PathError{
+			Op:   "read",
+			Path: path,
+			Err:  os.ErrPermission,
+		}
+	}
+	// describe its contents
+	for name, subdir := range dir.subdirs {
+		dirInfo := m.dirToFileInfo(subdir, name)
+		results = append(results, dirInfo)
+	}
+	for name, file := range dir.files {
+		info := m.fileToFileInfo(file, name)
+		results = append(results, info)
+	}
+	for name, link := range dir.symlinks {
+		info := m.linkToFileInfo(link, name)
+		results = append(results, info)
+	}
+	return results, nil
+}
+
+func (m *MockFs) Rename(sourcePath string, destPath string) error {
+	// validate source parent exists
+	sourcePath, err := m.resolve(sourcePath, false)
+	if err != nil {
+		return err
+	}
+	sourceParentPath := filepath.Dir(sourcePath)
+	sourceParentDir, err := m.getDir(sourceParentPath, false)
+	if err != nil {
+		return err
+	}
+	if sourceParentDir == nil {
+		return &os.PathError{
+			Op:   "move",
+			Path: sourcePath,
+			Err:  os.ErrNotExist,
+		}
+	}
+	if !sourceParentDir.readable {
+		return &os.PathError{
+			Op:   "move",
+			Path: sourcePath,
+			Err:  os.ErrPermission,
+		}
+	}
+
+	// validate dest parent exists
+	destPath, err = m.resolve(destPath, false)
+	destParentPath := filepath.Dir(destPath)
+	destParentDir, err := m.getDir(destParentPath, false)
+	if err != nil {
+		return err
+	}
+	if destParentDir == nil {
+		return &os.PathError{
+			Op:   "move",
+			Path: destParentPath,
+			Err:  os.ErrNotExist,
+		}
+	}
+	if !destParentDir.readable {
+		return &os.PathError{
+			Op:   "move",
+			Path: destParentPath,
+			Err:  os.ErrPermission,
+		}
+	}
+	// check the source and dest themselves
+	sourceBase := filepath.Base(sourcePath)
+	destBase := filepath.Base(destPath)
+
+	file, sourceIsFile := sourceParentDir.files[sourceBase]
+	dir, sourceIsDir := sourceParentDir.subdirs[sourceBase]
+	link, sourceIsLink := sourceParentDir.symlinks[sourceBase]
+
+	// validate that the source exists
+	if !sourceIsFile && !sourceIsDir && !sourceIsLink {
+		return &os.PathError{
+			Op:   "move",
+			Path: sourcePath,
+			Err:  os.ErrNotExist,
+		}
+
+	}
+
+	// validate the destination doesn't already exist as an incompatible type
+	_, destWasFile := destParentDir.files[destBase]
+	_, destWasDir := destParentDir.subdirs[destBase]
+	_, destWasLink := destParentDir.symlinks[destBase]
+
+	if destWasDir {
+		return &os.PathError{
+			Op:   "move",
+			Path: destPath,
+			Err:  errors.New("destination exists as a directory"),
+		}
+	}
+
+	if sourceIsDir && (destWasFile || destWasLink) {
+		return &os.PathError{
+			Op:   "move",
+			Path: destPath,
+			Err:  errors.New("destination exists as a file"),
+		}
+	}
+
+	if destWasFile {
+		delete(destParentDir.files, destBase)
+	}
+	if destWasDir {
+		delete(destParentDir.subdirs, destBase)
+	}
+	if destWasLink {
+		delete(destParentDir.symlinks, destBase)
+	}
+
+	if sourceIsFile {
+		destParentDir.files[destBase] = file
+		delete(sourceParentDir.files, sourceBase)
+	}
+	if sourceIsDir {
+		destParentDir.subdirs[destBase] = dir
+		delete(sourceParentDir.subdirs, sourceBase)
+	}
+	if sourceIsLink {
+		destParentDir.symlinks[destBase] = link
+		delete(destParentDir.symlinks, sourceBase)
+	}
+
+	destParentDir.modTime = m.Clock.Time()
+	sourceParentDir.modTime = m.Clock.Time()
+	return nil
+}
+
+func (m *MockFs) newInodeNumber() uint64 {
+	result := m.nextInodeNumber
+	m.nextInodeNumber++
+	return result
+}
+
+func (m *MockFs) WriteFile(filePath string, data []byte, perm os.FileMode) error {
+	filePath, err := m.resolve(filePath, true)
+	if err != nil {
+		return err
+	}
+	parentPath := filepath.Dir(filePath)
+	parentDir, err := m.getDir(parentPath, false)
+	if err != nil || parentDir == nil {
+		return &os.PathError{
+			Op:   "write",
+			Path: parentPath,
+			Err:  os.ErrNotExist,
+		}
+	}
+	if !parentDir.readable {
+		return &os.PathError{
+			Op:   "write",
+			Path: parentPath,
+			Err:  os.ErrPermission,
+		}
+	}
+
+	baseName := filepath.Base(filePath)
+	_, exists := parentDir.files[baseName]
+	if !exists {
+		parentDir.modTime = m.Clock.Time()
+		parentDir.files[baseName] = m.newFile()
+	} else {
+		if !parentDir.files[baseName].readable {
+			return &os.PathError{
+				Op:   "write",
+				Path: filePath,
+				Err:  os.ErrPermission,
+			}
+		}
+	}
+	file := parentDir.files[baseName]
+	file.bytes = data
+	file.modTime = m.Clock.Time()
+	return nil
+}
+
+func (m *MockFs) newFile() *mockFile {
+	newFile := &mockFile{}
+	newFile.inodeNumber = m.newInodeNumber()
+	newFile.modTime = m.Clock.Time()
+	newFile.permTime = newFile.modTime
+	newFile.readable = true
+	return newFile
+}
+
+func (m *MockFs) newDir() *mockDir {
+	newDir := &mockDir{
+		subdirs:  make(map[string]*mockDir, 0),
+		files:    make(map[string]*mockFile, 0),
+		symlinks: make(map[string]*mockLink, 0),
+	}
+	newDir.inodeNumber = m.newInodeNumber()
+	newDir.modTime = m.Clock.Time()
+	newDir.permTime = newDir.modTime
+	newDir.readable = true
+	return newDir
+}
+
+func (m *MockFs) newLink(target string) *mockLink {
+	newLink := &mockLink{
+		target: target,
+	}
+	newLink.inodeNumber = m.newInodeNumber()
+	newLink.modTime = m.Clock.Time()
+	newLink.permTime = newLink.modTime
+	newLink.readable = true
+
+	return newLink
+}
+func (m *MockFs) MkDirs(path string) error {
+	_, err := m.getDir(path, true)
+	return err
+}
+
+// getDir doesn't support symlinks
+func (m *MockFs) getDir(path string, createIfMissing bool) (dir *mockDir, err error) {
+	cleanedPath := filepath.Clean(path)
+	if cleanedPath == "/" {
+		return &m.root, nil
+	}
+
+	parentPath, leaf := pathSplit(cleanedPath)
+	if len(parentPath) >= len(path) {
+		return &m.root, nil
+	}
+	parent, err := m.getDir(parentPath, createIfMissing)
+	if err != nil {
+		return nil, err
+	}
+	if !parent.readable {
+		return nil, &os.PathError{
+			Op:   "stat",
+			Path: path,
+			Err:  os.ErrPermission,
+		}
+	}
+	childDir, dirExists := parent.subdirs[leaf]
+	if !dirExists {
+		if createIfMissing {
+			// confirm that a file with the same name doesn't already exist
+			_, fileExists := parent.files[leaf]
+			if fileExists {
+				return nil, &os.PathError{
+					Op:   "mkdir",
+					Path: path,
+					Err:  os.ErrExist,
+				}
+			}
+			// create this directory
+			childDir = m.newDir()
+			parent.subdirs[leaf] = childDir
+			parent.modTime = m.Clock.Time()
+		} else {
+			return nil, &os.PathError{
+				Op:   "stat",
+				Path: path,
+				Err:  os.ErrNotExist,
+			}
+		}
+	}
+	return childDir, nil
+
+}
+
+func (m *MockFs) Remove(path string) (err error) {
+	path, err = m.resolve(path, false)
+	parentPath, leaf := pathSplit(path)
+	if len(leaf) == 0 {
+		return fmt.Errorf("Cannot remove %v\n", path)
+	}
+	parentDir, err := m.getDir(parentPath, false)
+	if err != nil {
+		return err
+	}
+	if parentDir == nil {
+		return &os.PathError{
+			Op:   "remove",
+			Path: path,
+			Err:  os.ErrNotExist,
+		}
+	}
+	if !parentDir.readable {
+		return &os.PathError{
+			Op:   "remove",
+			Path: path,
+			Err:  os.ErrPermission,
+		}
+	}
+	_, isDir := parentDir.subdirs[leaf]
+	if isDir {
+		return &os.PathError{
+			Op:   "remove",
+			Path: path,
+			Err:  os.ErrInvalid,
+		}
+	}
+	_, isLink := parentDir.symlinks[leaf]
+	if isLink {
+		delete(parentDir.symlinks, leaf)
+	} else {
+		_, isFile := parentDir.files[leaf]
+		if !isFile {
+			return &os.PathError{
+				Op:   "remove",
+				Path: path,
+				Err:  os.ErrNotExist,
+			}
+		}
+		delete(parentDir.files, leaf)
+	}
+	parentDir.modTime = m.Clock.Time()
+	return nil
+}
+
+func (m *MockFs) Symlink(oldPath string, newPath string) (err error) {
+	newPath, err = m.resolve(newPath, false)
+	if err != nil {
+		return err
+	}
+
+	newParentPath, leaf := pathSplit(newPath)
+	newParentDir, err := m.getDir(newParentPath, false)
+	if !newParentDir.readable {
+		return &os.PathError{
+			Op:   "link",
+			Path: newPath,
+			Err:  os.ErrPermission,
+		}
+	}
+	if err != nil {
+		return err
+	}
+	newParentDir.symlinks[leaf] = m.newLink(oldPath)
+	return nil
+}
+
+func (m *MockFs) RemoveAll(path string) (err error) {
+	path, err = m.resolve(path, false)
+	if err != nil {
+		return err
+	}
+	parentPath, leaf := pathSplit(path)
+	if len(leaf) == 0 {
+		return fmt.Errorf("Cannot remove %v\n", path)
+	}
+	parentDir, err := m.getDir(parentPath, false)
+	if err != nil {
+		return err
+	}
+	if parentDir == nil {
+		return &os.PathError{
+			Op:   "removeAll",
+			Path: path,
+			Err:  os.ErrNotExist,
+		}
+	}
+	if !parentDir.readable {
+		return &os.PathError{
+			Op:   "removeAll",
+			Path: path,
+			Err:  os.ErrPermission,
+		}
+
+	}
+	_, isFile := parentDir.files[leaf]
+	_, isLink := parentDir.symlinks[leaf]
+	if isFile || isLink {
+		return m.Remove(path)
+	}
+	_, isDir := parentDir.subdirs[leaf]
+	if !isDir {
+		if !isDir {
+			return &os.PathError{
+				Op:   "removeAll",
+				Path: path,
+				Err:  os.ErrNotExist,
+			}
+		}
+	}
+
+	delete(parentDir.subdirs, leaf)
+	parentDir.modTime = m.Clock.Time()
+	return nil
+}
+
+func (m *MockFs) SetReadable(path string, readable bool) error {
+	path, err := m.resolve(path, false)
+	if err != nil {
+		return err
+	}
+	parentPath, leaf := filepath.Split(path)
+	parentDir, err := m.getDir(parentPath, false)
+	if err != nil {
+		return err
+	}
+	if !parentDir.readable {
+		return &os.PathError{
+			Op:   "chmod",
+			Path: parentPath,
+			Err:  os.ErrPermission,
+		}
+	}
+
+	inode, err := m.getInode(parentDir, leaf)
+	if err != nil {
+		return err
+	}
+	inode.readable = readable
+	inode.permTime = m.Clock.Time()
+	return nil
+}
+
+func (m *MockFs) ClearMetrics() {
+	m.ReadDirCalls = []string{}
+	m.StatCalls = []string{}
+}
+
+func (m *MockFs) ViewId() (id string) {
+	return m.viewId
+}
+
+func (m *MockFs) SetViewId(id string) {
+	m.viewId = id
+}
+func (m *MockFs) SetDeviceNumber(deviceNumber uint64) {
+	m.deviceNumber = deviceNumber
+}

ui/logger/logger.go

@@ -145,13 +145,14 @@ func New(out io.Writer) *stdLogger {
 
 // SetVerbose controls whether Verbose[f|ln] logs to stderr as well as the
 // file-backed log.
-func (s *stdLogger) SetVerbose(v bool) {
+func (s *stdLogger) SetVerbose(v bool) *stdLogger {
 	s.verbose = v
+	return s
 }
 
 // SetOutput controls where the file-backed log will be saved. It will keep
 // some number of backups of old log files.
-func (s *stdLogger) SetOutput(path string) {
+func (s *stdLogger) SetOutput(path string) *stdLogger {
 	if f, err := CreateFileWithRotation(path, 5); err == nil {
 		s.mutex.Lock()
 		defer s.mutex.Unlock()
@@ -164,6 +165,7 @@ func (s *stdLogger) SetOutput(path string) {
 	} else {
 		s.Fatal(err.Error())
 	}
+	return s
 }
 
 // Close disables logging to the file and closes the file handle.