diff --git a/Blueprints b/Blueprints index 702d0de..582a0aa 100644 --- a/Blueprints +++ b/Blueprints @@ -8,7 +8,6 @@ bootstrap_go_package( pkgPath = "github.com/google/blueprint", srcs = [ "context.go", - "fs.go", "glob.go", "live_tracker.go", "mangle.go", @@ -61,6 +60,7 @@ bootstrap_go_package( ], srcs = [ "pathtools/lists.go", + "pathtools/fs.go", "pathtools/glob.go", ], testSrcs = [ diff --git a/build.ninja.in b/build.ninja.in index ec32fcf..1de3dbb 100644 --- a/build.ninja.in +++ b/build.ninja.in @@ -58,9 +58,8 @@ rule g.bootstrap.link build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint/pkg/github.com/google/blueprint.a $ : g.bootstrap.compile ${g.bootstrap.srcDir}/context.go $ - ${g.bootstrap.srcDir}/fs.go ${g.bootstrap.srcDir}/glob.go $ - ${g.bootstrap.srcDir}/live_tracker.go ${g.bootstrap.srcDir}/mangle.go $ - ${g.bootstrap.srcDir}/module_ctx.go $ + ${g.bootstrap.srcDir}/glob.go ${g.bootstrap.srcDir}/live_tracker.go $ + ${g.bootstrap.srcDir}/mangle.go ${g.bootstrap.srcDir}/module_ctx.go $ ${g.bootstrap.srcDir}/ninja_defs.go $ ${g.bootstrap.srcDir}/ninja_strings.go $ ${g.bootstrap.srcDir}/ninja_writer.go $ @@ -130,7 +129,7 @@ default $ # Variant: # Type: bootstrap_go_package # Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1 -# Defined: Blueprints:50:1 +# Defined: Blueprints:49:1 build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a $ @@ -145,7 +144,7 @@ default $ # Variant: # Type: bootstrap_go_package # Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1 -# Defined: Blueprints:34:1 +# Defined: Blueprints:33:1 build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-parser/pkg/github.com/google/blueprint/parser.a $ @@ -163,11 +162,12 @@ default $ # Variant: # Type: bootstrap_go_package # Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1 -# Defined: Blueprints:56:1 +# Defined: Blueprints:55:1 build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-pathtools/pkg/github.com/google/blueprint/pathtools.a $ : g.bootstrap.compile ${g.bootstrap.srcDir}/pathtools/lists.go $ + ${g.bootstrap.srcDir}/pathtools/fs.go $ ${g.bootstrap.srcDir}/pathtools/glob.go | ${g.bootstrap.compileCmd} $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a incFlags = -I ${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg diff --git a/context.go b/context.go index c28215d..f63b1af 100644 --- a/context.go +++ b/context.go @@ -31,6 +31,7 @@ import ( "text/template" "github.com/google/blueprint/parser" + "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" ) @@ -104,7 +105,7 @@ type Context struct { globs map[string]GlobPath globLock sync.Mutex - fs fileSystem + fs pathtools.FileSystem } // An Error describes a problem that was encountered that is related to a @@ -270,7 +271,7 @@ func NewContext() *Context { moduleInfo: make(map[Module]*moduleInfo), moduleNinjaNames: make(map[string]*moduleGroup), globs: make(map[string]GlobPath), - fs: fs, + fs: pathtools.OsFs, } ctx.RegisterBottomUpMutator("blueprint_deps", blueprintDepsMutator) @@ -768,9 +769,7 @@ loop: // MockFileSystem causes the Context to replace all reads with accesses to the provided map of // filenames to contents stored as a byte slice. func (c *Context) MockFileSystem(files map[string][]byte) { - c.fs = &mockFS{ - files: files, - } + c.fs = pathtools.MockFs(files) } // parseBlueprintFile parses a single Blueprints file, returning any errors through diff --git a/fs.go b/fs.go deleted file mode 100644 index 653d6d6..0000000 --- a/fs.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2016 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package blueprint - -import ( - "bytes" - "io" - "io/ioutil" - "os" -) - -// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go" - -var fs fileSystem = osFS{} - -type fileSystem interface { - Open(name string) (io.ReadCloser, error) - Exists(name string) (bool, bool, error) -} - -// osFS implements fileSystem using the local disk. -type osFS struct{} - -func (osFS) Open(name string) (io.ReadCloser, error) { return os.Open(name) } -func (osFS) Exists(name string) (bool, bool, error) { - stat, err := os.Stat(name) - if err == nil { - return true, stat.IsDir(), nil - } else if os.IsNotExist(err) { - return false, false, nil - } else { - return false, false, err - } -} - -type mockFS struct { - files map[string][]byte -} - -func (m mockFS) Open(name string) (io.ReadCloser, error) { - if f, ok := m.files[name]; ok { - return struct { - io.Closer - *bytes.Reader - }{ - ioutil.NopCloser(nil), - bytes.NewReader(f), - }, nil - } - - return nil, &os.PathError{ - Op: "open", - Path: name, - Err: os.ErrNotExist, - } -} - -func (m mockFS) Exists(name string) (bool, bool, error) { - _, ok := m.files[name] - return ok, false, nil -} diff --git a/glob.go b/glob.go index dad5edf..3497236 100644 --- a/glob.go +++ b/glob.go @@ -18,8 +18,6 @@ import ( "fmt" "reflect" "sort" - - "github.com/google/blueprint/pathtools" ) type GlobPath struct { @@ -54,7 +52,7 @@ func (c *Context) glob(pattern string, excludes []string) ([]string, error) { } // Get a globbed file list - files, deps, err := pathtools.GlobWithExcludes(pattern, excludes) + files, deps, err := c.fs.Glob(pattern, excludes) if err != nil { return nil, err } diff --git a/module_ctx.go b/module_ctx.go index abd87c1..a2fd9c9 100644 --- a/module_ctx.go +++ b/module_ctx.go @@ -18,6 +18,8 @@ import ( "fmt" "path/filepath" "text/scanner" + + "github.com/google/blueprint/pathtools" ) // A Module handles generating all of the Ninja build actions needed to build a @@ -134,6 +136,8 @@ type BaseModuleContext interface { // file that does not match the pattern is added to a searched directory. GlobWithDeps(pattern string, excludes []string) ([]string, error) + Fs() pathtools.FileSystem + moduleInfo() *moduleInfo error(err error) } @@ -260,6 +264,10 @@ func (d *baseModuleContext) GlobWithDeps(pattern string, return d.context.glob(pattern, excludes) } +func (d *baseModuleContext) Fs() pathtools.FileSystem { + return d.context.fs +} + var _ ModuleContext = (*moduleContext)(nil) type moduleContext struct { diff --git a/pathtools/fs.go b/pathtools/fs.go new file mode 100644 index 0000000..4ed91ff --- /dev/null +++ b/pathtools/fs.go @@ -0,0 +1,151 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pathtools + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go" + +var OsFs FileSystem = osFs{} + +func MockFs(files map[string][]byte) FileSystem { + fs := &mockFs{ + files: make(map[string][]byte, len(files)), + dirs: make(map[string]bool), + all: []string(nil), + } + + for f, b := range files { + fs.files[filepath.Clean(f)] = b + dir := filepath.Dir(f) + for dir != "." && dir != "/" { + fs.dirs[dir] = true + dir = filepath.Dir(dir) + } + } + + for f := range fs.files { + fs.all = append(fs.all, f) + } + + for d := range fs.dirs { + fs.all = append(fs.all, d) + } + + return fs +} + +type FileSystem interface { + Open(name string) (io.ReadCloser, error) + Exists(name string) (bool, bool, error) + Glob(pattern string, excludes []string) (matches, dirs []string, err error) + glob(pattern string) (matches []string, err error) + IsDir(name string) (bool, error) +} + +// osFs implements FileSystem using the local disk. +type osFs struct{} + +func (osFs) Open(name string) (io.ReadCloser, error) { return os.Open(name) } +func (osFs) Exists(name string) (bool, bool, error) { + stat, err := os.Stat(name) + if err == nil { + return true, stat.IsDir(), nil + } else if os.IsNotExist(err) { + return false, false, nil + } else { + return false, false, err + } +} + +func (osFs) IsDir(name string) (bool, error) { + info, err := os.Stat(name) + if err != nil { + return false, fmt.Errorf("unexpected error after glob: %s", err) + } + return info.IsDir(), nil +} + +func (fs osFs) Glob(pattern string, excludes []string) (matches, dirs []string, err error) { + return startGlob(fs, pattern, excludes) +} + +func (osFs) glob(pattern string) ([]string, error) { + return filepath.Glob(pattern) +} + +type mockFs struct { + files map[string][]byte + dirs map[string]bool + all []string +} + +func (m *mockFs) Open(name string) (io.ReadCloser, error) { + if f, ok := m.files[name]; ok { + return struct { + io.Closer + *bytes.Reader + }{ + ioutil.NopCloser(nil), + bytes.NewReader(f), + }, nil + } + + return nil, &os.PathError{ + Op: "open", + Path: name, + Err: os.ErrNotExist, + } +} + +func (m *mockFs) Exists(name string) (bool, bool, error) { + name = filepath.Clean(name) + if _, ok := m.files[name]; ok { + return ok, false, nil + } + if _, ok := m.dirs[name]; ok { + return ok, true, nil + } + return false, false, nil +} + +func (m *mockFs) IsDir(name string) (bool, error) { + return m.dirs[filepath.Clean(name)], nil +} + +func (m *mockFs) Glob(pattern string, excludes []string) (matches, dirs []string, err error) { + return startGlob(m, pattern, excludes) +} + +func (m *mockFs) glob(pattern string) ([]string, error) { + var matches []string + for _, f := range m.all { + match, err := filepath.Match(pattern, f) + if err != nil { + return nil, err + } + if match { + matches = append(matches, f) + } + } + return matches, nil +} diff --git a/pathtools/glob.go b/pathtools/glob.go index 286d933..a842a82 100644 --- a/pathtools/glob.go +++ b/pathtools/glob.go @@ -28,35 +28,25 @@ import ( var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **") var GlobLastRecursiveErr = errors.New("pattern ** as last path element") -// Glob returns the list of files that match the given pattern along with the -// list of directories that were searched to construct the file list. -// The supported glob patterns are equivalent to filepath.Glob, with an -// extension that recursive glob (** matching zero or more complete path -// entries) is supported. Glob also returns a list of directories that were -// searched. -// -// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps -// should be used instead, as they will automatically set up dependencies -// to rerun the primary builder when the list of matching files changes. -func Glob(pattern string) (matches, dirs []string, err error) { - return GlobWithExcludes(pattern, nil) -} - -// GlobWithExcludes returns the list of files that match the given pattern but +// Glob returns the list of files that match the given pattern but // do not match the given exclude patterns, along with the list of directories // that were searched to construct the file list. The supported glob and // exclude patterns are equivalent to filepath.Glob, with an extension that // recursive glob (** matching zero or more complete path entries) is supported. -// GlobWithExcludes also returns a list of directories that were searched. +// Glob also returns a list of directories that were searched. // // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps // should be used instead, as they will automatically set up dependencies // to rerun the primary builder when the list of matching files changes. -func GlobWithExcludes(pattern string, excludes []string) (matches, dirs []string, err error) { +func Glob(pattern string, excludes []string) (matches, dirs []string, err error) { + return startGlob(OsFs, pattern, excludes) +} + +func startGlob(fs FileSystem, pattern string, excludes []string) (matches, dirs []string, err error) { if filepath.Base(pattern) == "**" { return nil, nil, GlobLastRecursiveErr } else { - matches, dirs, err = glob(pattern, false) + matches, dirs, err = glob(fs, pattern, false) } if err != nil { @@ -73,12 +63,12 @@ func GlobWithExcludes(pattern string, excludes []string) (matches, dirs []string // glob is a recursive helper function to handle globbing each level of the pattern individually, // allowing searched directories to be tracked. Also handles the recursive glob pattern, **. -func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error) { +func glob(fs FileSystem, pattern string, hasRecursive bool) (matches, dirs []string, err error) { if !isWild(pattern) { // If there are no wilds in the pattern, check whether the file exists or not. // Uses filepath.Glob instead of manually statting to get consistent results. pattern = filepath.Clean(pattern) - matches, err = filepath.Glob(pattern) + matches, err = fs.glob(pattern) if err != nil { return matches, dirs, err } @@ -89,7 +79,7 @@ func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error) var matchDirs []string for len(matchDirs) == 0 { pattern, _ = saneSplit(pattern) - matchDirs, err = filepath.Glob(pattern) + matchDirs, err = fs.glob(pattern) if err != nil { return matches, dirs, err } @@ -108,17 +98,15 @@ func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error) hasRecursive = true } - dirMatches, dirs, err := glob(dir, hasRecursive) + dirMatches, dirs, err := glob(fs, dir, hasRecursive) if err != nil { return nil, nil, err } for _, m := range dirMatches { - info, err := os.Stat(m) - if err != nil { + if isDir, err := fs.IsDir(m); err != nil { return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) - } - if info.IsDir() { + } else if isDir { if file == "**" { recurseDirs, err := walkAllDirs(m) if err != nil { @@ -127,7 +115,7 @@ func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error) matches = append(matches, recurseDirs...) } else { dirs = append(dirs, m) - newMatches, err := filepath.Glob(filepath.Join(m, file)) + newMatches, err := fs.glob(filepath.Join(m, file)) if err != nil { return nil, nil, err } @@ -316,7 +304,7 @@ func GlobPatternList(patterns []string, prefix string) (globedList []string, dep for _, pattern := range patterns { if isWild(pattern) { - matches, deps, err = Glob(filepath.Join(prefix, pattern)) + matches, deps, err = Glob(filepath.Join(prefix, pattern), nil) if err != nil { return nil, nil, err } @@ -358,7 +346,7 @@ func HasGlob(in []string) bool { // should be used instead, as they will automatically set up dependencies // to rerun the primary builder when the list of matching files changes. func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) { - files, dirs, err := GlobWithExcludes(glob, excludes) + files, dirs, err := Glob(glob, excludes) if err != nil { return nil, err } diff --git a/pathtools/glob_test.go b/pathtools/glob_test.go index 803cb6b..6e236fc 100644 --- a/pathtools/glob_test.go +++ b/pathtools/glob_test.go @@ -445,7 +445,7 @@ func TestGlob(t *testing.T) { os.Chdir("testdata") defer os.Chdir("..") for _, testCase := range globTestCases { - matches, dirs, err := GlobWithExcludes(testCase.pattern, testCase.excludes) + matches, dirs, err := Glob(testCase.pattern, testCase.excludes) if err != testCase.err { t.Errorf(" pattern: %q", testCase.pattern) if testCase.excludes != nil { diff --git a/singleton_ctx.go b/singleton_ctx.go index fc4a781..31195af 100644 --- a/singleton_ctx.go +++ b/singleton_ctx.go @@ -16,6 +16,7 @@ package blueprint import ( "fmt" + "github.com/google/blueprint/pathtools" ) type Singleton interface { @@ -68,6 +69,8 @@ type SingletonContext interface { // builder whenever a file matching the pattern as added or removed, without rerunning if a // file that does not match the pattern is added to a searched directory. GlobWithDeps(pattern string, excludes []string) ([]string, error) + + Fs() pathtools.FileSystem } var _ SingletonContext = (*singletonContext)(nil) @@ -239,3 +242,7 @@ func (s *singletonContext) GlobWithDeps(pattern string, excludes []string) ([]string, error) { return s.context.glob(pattern, excludes) } + +func (s *singletonContext) Fs() pathtools.FileSystem { + return s.context.fs +} diff --git a/tests/test_tree/build.ninja.in b/tests/test_tree/build.ninja.in index 5604b40..7728ca3 100644 --- a/tests/test_tree/build.ninja.in +++ b/tests/test_tree/build.ninja.in @@ -58,7 +58,6 @@ rule g.bootstrap.link build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint/pkg/github.com/google/blueprint.a $ : g.bootstrap.compile ${g.bootstrap.srcDir}/blueprint/context.go $ - ${g.bootstrap.srcDir}/blueprint/fs.go $ ${g.bootstrap.srcDir}/blueprint/glob.go $ ${g.bootstrap.srcDir}/blueprint/live_tracker.go $ ${g.bootstrap.srcDir}/blueprint/mangle.go $ @@ -135,7 +134,7 @@ default $ # Variant: # Type: bootstrap_go_package # Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1 -# Defined: blueprint/Blueprints:50:1 +# Defined: blueprint/Blueprints:49:1 build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a $ @@ -151,7 +150,7 @@ default $ # Variant: # Type: bootstrap_go_package # Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1 -# Defined: blueprint/Blueprints:34:1 +# Defined: blueprint/Blueprints:33:1 build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-parser/pkg/github.com/google/blueprint/parser.a $ @@ -170,12 +169,13 @@ default $ # Variant: # Type: bootstrap_go_package # Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1 -# Defined: blueprint/Blueprints:56:1 +# Defined: blueprint/Blueprints:55:1 build $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-pathtools/pkg/github.com/google/blueprint/pathtools.a $ : g.bootstrap.compile $ ${g.bootstrap.srcDir}/blueprint/pathtools/lists.go $ + ${g.bootstrap.srcDir}/blueprint/pathtools/fs.go $ ${g.bootstrap.srcDir}/blueprint/pathtools/glob.go | $ ${g.bootstrap.compileCmd} $ ${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a