tequilaOS/platform_build_blueprint
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add globbing to filesystem mocking

Browse source 
Add globbing to filesystem mocking so that more code can be tested
against the mock.  Also moves the filesystem mock to pathtools,
and renames pathtools.GlobWithExcludes to pathtools.Glob, replacing
the existing pathtools.Glob.

Test: blueprint tests
Change-Id: I722df8121bc870c4a861d7c249245c57dcb607be
This commit is contained in:
Colin Cross 2017-02-01 13:21:35 -08:00
parent 2eeb7ee14b
commit b519a7e1b6
11 changed files with 200 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Blueprints
View file

@ -8,7 +8,6 @@ bootstrap_go_package(
pkgPath = "github.com/google/blueprint",
srcs = [
"context.go",
"fs.go",
"glob.go",
"live_tracker.go",
"mangle.go",
@ -61,6 +60,7 @@ bootstrap_go_package(
],
srcs = [
"pathtools/lists.go",
"pathtools/fs.go",
"pathtools/glob.go",
],
testSrcs = [

12
build.ninja.in
View file

@ -58,9 +58,8 @@ rule g.bootstrap.link
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint/pkg/github.com/google/blueprint.a $
: g.bootstrap.compile ${g.bootstrap.srcDir}/context.go $
${g.bootstrap.srcDir}/fs.go ${g.bootstrap.srcDir}/glob.go $
${g.bootstrap.srcDir}/live_tracker.go ${g.bootstrap.srcDir}/mangle.go $
${g.bootstrap.srcDir}/module_ctx.go $
${g.bootstrap.srcDir}/glob.go ${g.bootstrap.srcDir}/live_tracker.go $
${g.bootstrap.srcDir}/mangle.go ${g.bootstrap.srcDir}/module_ctx.go $
${g.bootstrap.srcDir}/ninja_defs.go $
${g.bootstrap.srcDir}/ninja_strings.go $
${g.bootstrap.srcDir}/ninja_writer.go $
@ -130,7 +129,7 @@ default $
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1
# Defined: Blueprints:50:1
# Defined: Blueprints:49:1
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a $
@ -145,7 +144,7 @@ default $
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1
# Defined: Blueprints:34:1
# Defined: Blueprints:33:1
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint-parser/pkg/github.com/google/blueprint/parser.a $
@ -163,11 +162,12 @@ default $
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1
# Defined: Blueprints:56:1
# Defined: Blueprints:55:1
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint-pathtools/pkg/github.com/google/blueprint/pathtools.a $
: g.bootstrap.compile ${g.bootstrap.srcDir}/pathtools/lists.go $
${g.bootstrap.srcDir}/pathtools/fs.go $
${g.bootstrap.srcDir}/pathtools/glob.go | ${g.bootstrap.compileCmd} $
${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a
incFlags = -I ${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg

9
context.go
View file

@ -31,6 +31,7 @@ import (
"text/template"
"github.com/google/blueprint/parser"
"github.com/google/blueprint/pathtools"
"github.com/google/blueprint/proptools"
)
@ -104,7 +105,7 @@ type Context struct {
globs map[string]GlobPath
globLock sync.Mutex
fs fileSystem
fs pathtools.FileSystem
}
// An Error describes a problem that was encountered that is related to a
@ -270,7 +271,7 @@ func NewContext() *Context {
moduleInfo: make(map[Module]*moduleInfo),
moduleNinjaNames: make(map[string]*moduleGroup),
globs: make(map[string]GlobPath),
fs: fs,
fs: pathtools.OsFs,
}
ctx.RegisterBottomUpMutator("blueprint_deps", blueprintDepsMutator)
@ -768,9 +769,7 @@ loop:
// MockFileSystem causes the Context to replace all reads with accesses to the provided map of
// filenames to contents stored as a byte slice.
func (c *Context) MockFileSystem(files map[string][]byte) {
c.fs = &mockFS{
files: files,
}
c.fs = pathtools.MockFs(files)
}
// parseBlueprintFile parses a single Blueprints file, returning any errors through

73
fs.go
View file

@ -1,73 +0,0 @@
// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blueprint
import (
"bytes"
"io"
"io/ioutil"
"os"
)
// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go"
var fs fileSystem = osFS{}
type fileSystem interface {
Open(name string) (io.ReadCloser, error)
Exists(name string) (bool, bool, error)
}
// osFS implements fileSystem using the local disk.
type osFS struct{}
func (osFS) Open(name string) (io.ReadCloser, error) { return os.Open(name) }
func (osFS) Exists(name string) (bool, bool, error) {
stat, err := os.Stat(name)
if err == nil {
return true, stat.IsDir(), nil
} else if os.IsNotExist(err) {
return false, false, nil
} else {
return false, false, err
}
}
type mockFS struct {
files map[string][]byte
}
func (m mockFS) Open(name string) (io.ReadCloser, error) {
if f, ok := m.files[name]; ok {
return struct {
io.Closer
*bytes.Reader
}{
ioutil.NopCloser(nil),
bytes.NewReader(f),
}, nil
}
return nil, &os.PathError{
Op: "open",
Path: name,
Err: os.ErrNotExist,
}
}
func (m mockFS) Exists(name string) (bool, bool, error) {
_, ok := m.files[name]
return ok, false, nil
}

4
glob.go
View file

@ -18,8 +18,6 @@ import (
"fmt"
"reflect"
"sort"
"github.com/google/blueprint/pathtools"
)
type GlobPath struct {
@ -54,7 +52,7 @@ func (c *Context) glob(pattern string, excludes []string) ([]string, error) {
}
// Get a globbed file list
files, deps, err := pathtools.GlobWithExcludes(pattern, excludes)
files, deps, err := c.fs.Glob(pattern, excludes)
if err != nil {
return nil, err
}

8
module_ctx.go
View file

@ -18,6 +18,8 @@ import (
"fmt"
"path/filepath"
"text/scanner"
"github.com/google/blueprint/pathtools"
)
// A Module handles generating all of the Ninja build actions needed to build a
@ -134,6 +136,8 @@ type BaseModuleContext interface {
// file that does not match the pattern is added to a searched directory.
GlobWithDeps(pattern string, excludes []string) ([]string, error)
Fs() pathtools.FileSystem
moduleInfo() *moduleInfo
error(err error)
}
@ -260,6 +264,10 @@ func (d *baseModuleContext) GlobWithDeps(pattern string,
return d.context.glob(pattern, excludes)
}
func (d *baseModuleContext) Fs() pathtools.FileSystem {
return d.context.fs
}
var _ ModuleContext = (*moduleContext)(nil)
type moduleContext struct {

151
pathtools/fs.go Normal file
View file

@ -0,0 +1,151 @@
// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pathtools
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
)
// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go"
var OsFs FileSystem = osFs{}
func MockFs(files map[string][]byte) FileSystem {
fs := &mockFs{
files: make(map[string][]byte, len(files)),
dirs: make(map[string]bool),
all: []string(nil),
}
for f, b := range files {
fs.files[filepath.Clean(f)] = b
dir := filepath.Dir(f)
for dir != "." && dir != "/" {
fs.dirs[dir] = true
dir = filepath.Dir(dir)
}
}
for f := range fs.files {
fs.all = append(fs.all, f)
}
for d := range fs.dirs {
fs.all = append(fs.all, d)
}
return fs
}
type FileSystem interface {
Open(name string) (io.ReadCloser, error)
Exists(name string) (bool, bool, error)
Glob(pattern string, excludes []string) (matches, dirs []string, err error)
glob(pattern string) (matches []string, err error)
IsDir(name string) (bool, error)
}
// osFs implements FileSystem using the local disk.
type osFs struct{}
func (osFs) Open(name string) (io.ReadCloser, error) { return os.Open(name) }
func (osFs) Exists(name string) (bool, bool, error) {
stat, err := os.Stat(name)
if err == nil {
return true, stat.IsDir(), nil
} else if os.IsNotExist(err) {
return false, false, nil
} else {
return false, false, err
}
}
func (osFs) IsDir(name string) (bool, error) {
info, err := os.Stat(name)
if err != nil {
return false, fmt.Errorf("unexpected error after glob: %s", err)
}
return info.IsDir(), nil
}
func (fs osFs) Glob(pattern string, excludes []string) (matches, dirs []string, err error) {
return startGlob(fs, pattern, excludes)
}
func (osFs) glob(pattern string) ([]string, error) {
return filepath.Glob(pattern)
}
type mockFs struct {
files map[string][]byte
dirs map[string]bool
all []string
}
func (m *mockFs) Open(name string) (io.ReadCloser, error) {
if f, ok := m.files[name]; ok {
return struct {
io.Closer
*bytes.Reader
}{
ioutil.NopCloser(nil),
bytes.NewReader(f),
}, nil
}
return nil, &os.PathError{
Op: "open",
Path: name,
Err: os.ErrNotExist,
}
}
func (m *mockFs) Exists(name string) (bool, bool, error) {
name = filepath.Clean(name)
if _, ok := m.files[name]; ok {
return ok, false, nil
}
if _, ok := m.dirs[name]; ok {
return ok, true, nil
}
return false, false, nil
}
func (m *mockFs) IsDir(name string) (bool, error) {
return m.dirs[filepath.Clean(name)], nil
}
func (m *mockFs) Glob(pattern string, excludes []string) (matches, dirs []string, err error) {
return startGlob(m, pattern, excludes)
}
func (m *mockFs) glob(pattern string) ([]string, error) {
var matches []string
for _, f := range m.all {
match, err := filepath.Match(pattern, f)
if err != nil {
return nil, err
}
if match {
matches = append(matches, f)
}
}
return matches, nil
}

46
pathtools/glob.go
View file

@ -28,35 +28,25 @@ import (
var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **")
var GlobLastRecursiveErr = errors.New("pattern ** as last path element")
// Glob returns the list of files that match the given pattern along with the
// list of directories that were searched to construct the file list.
// The supported glob patterns are equivalent to filepath.Glob, with an
// extension that recursive glob (** matching zero or more complete path
// entries) is supported. Glob also returns a list of directories that were
// searched.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func Glob(pattern string) (matches, dirs []string, err error) {
return GlobWithExcludes(pattern, nil)
}
// GlobWithExcludes returns the list of files that match the given pattern but
// Glob returns the list of files that match the given pattern but
// do not match the given exclude patterns, along with the list of directories
// that were searched to construct the file list. The supported glob and
// exclude patterns are equivalent to filepath.Glob, with an extension that
// recursive glob (** matching zero or more complete path entries) is supported.
// GlobWithExcludes also returns a list of directories that were searched.
// Glob also returns a list of directories that were searched.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func GlobWithExcludes(pattern string, excludes []string) (matches, dirs []string, err error) {
func Glob(pattern string, excludes []string) (matches, dirs []string, err error) {
return startGlob(OsFs, pattern, excludes)
}
func startGlob(fs FileSystem, pattern string, excludes []string) (matches, dirs []string, err error) {
if filepath.Base(pattern) == "**" {
return nil, nil, GlobLastRecursiveErr
} else {
matches, dirs, err = glob(pattern, false)
matches, dirs, err = glob(fs, pattern, false)
}
if err != nil {
@ -73,12 +63,12 @@ func GlobWithExcludes(pattern string, excludes []string) (matches, dirs []string
// glob is a recursive helper function to handle globbing each level of the pattern individually,
// allowing searched directories to be tracked. Also handles the recursive glob pattern, **.
func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error) {
func glob(fs FileSystem, pattern string, hasRecursive bool) (matches, dirs []string, err error) {
if !isWild(pattern) {
// If there are no wilds in the pattern, check whether the file exists or not.
// Uses filepath.Glob instead of manually statting to get consistent results.
pattern = filepath.Clean(pattern)
matches, err = filepath.Glob(pattern)
matches, err = fs.glob(pattern)
if err != nil {
return matches, dirs, err
}
@ -89,7 +79,7 @@ func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error)
var matchDirs []string
for len(matchDirs) == 0 {
pattern, _ = saneSplit(pattern)
matchDirs, err = filepath.Glob(pattern)
matchDirs, err = fs.glob(pattern)
if err != nil {
return matches, dirs, err
}
@ -108,17 +98,15 @@ func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error)
hasRecursive = true
}
dirMatches, dirs, err := glob(dir, hasRecursive)
dirMatches, dirs, err := glob(fs, dir, hasRecursive)
if err != nil {
return nil, nil, err
}
for _, m := range dirMatches {
info, err := os.Stat(m)
if err != nil {
if isDir, err := fs.IsDir(m); err != nil {
return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
}
if info.IsDir() {
} else if isDir {
if file == "**" {
recurseDirs, err := walkAllDirs(m)
if err != nil {
@ -127,7 +115,7 @@ func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error)
matches = append(matches, recurseDirs...)
} else {
dirs = append(dirs, m)
newMatches, err := filepath.Glob(filepath.Join(m, file))
newMatches, err := fs.glob(filepath.Join(m, file))
if err != nil {
return nil, nil, err
}
@ -316,7 +304,7 @@ func GlobPatternList(patterns []string, prefix string) (globedList []string, dep
for _, pattern := range patterns {
if isWild(pattern) {
matches, deps, err = Glob(filepath.Join(prefix, pattern))
matches, deps, err = Glob(filepath.Join(prefix, pattern), nil)
if err != nil {
return nil, nil, err
}
@ -358,7 +346,7 @@ func HasGlob(in []string) bool {
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
files, dirs, err := GlobWithExcludes(glob, excludes)
files, dirs, err := Glob(glob, excludes)
if err != nil {
return nil, err
}

2
pathtools/glob_test.go
View file

@ -445,7 +445,7 @@ func TestGlob(t *testing.T) {
os.Chdir("testdata")
defer os.Chdir("..")
for _, testCase := range globTestCases {
matches, dirs, err := GlobWithExcludes(testCase.pattern, testCase.excludes)
matches, dirs, err := Glob(testCase.pattern, testCase.excludes)
if err != testCase.err {
t.Errorf(" pattern: %q", testCase.pattern)
if testCase.excludes != nil {

7
singleton_ctx.go
View file

@ -16,6 +16,7 @@ package blueprint
import (
"fmt"
"github.com/google/blueprint/pathtools"
)
type Singleton interface {
@ -68,6 +69,8 @@ type SingletonContext interface {
// builder whenever a file matching the pattern as added or removed, without rerunning if a
// file that does not match the pattern is added to a searched directory.
GlobWithDeps(pattern string, excludes []string) ([]string, error)
Fs() pathtools.FileSystem
}
var _ SingletonContext = (*singletonContext)(nil)
@ -239,3 +242,7 @@ func (s *singletonContext) GlobWithDeps(pattern string,
excludes []string) ([]string, error) {
return s.context.glob(pattern, excludes)
}
func (s *singletonContext) Fs() pathtools.FileSystem {
return s.context.fs
}

8
tests/test_tree/build.ninja.in
View file

@ -58,7 +58,6 @@ rule g.bootstrap.link
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint/pkg/github.com/google/blueprint.a $
: g.bootstrap.compile ${g.bootstrap.srcDir}/blueprint/context.go $
${g.bootstrap.srcDir}/blueprint/fs.go $
${g.bootstrap.srcDir}/blueprint/glob.go $
${g.bootstrap.srcDir}/blueprint/live_tracker.go $
${g.bootstrap.srcDir}/blueprint/mangle.go $
@ -135,7 +134,7 @@ default $
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1
# Defined: blueprint/Blueprints:50:1
# Defined: blueprint/Blueprints:49:1
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a $
@ -151,7 +150,7 @@ default $
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1
# Defined: blueprint/Blueprints:34:1
# Defined: blueprint/Blueprints:33:1
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint-parser/pkg/github.com/google/blueprint/parser.a $
@ -170,12 +169,13 @@ default $
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.newGoPackageModuleFactory.func1
# Defined: blueprint/Blueprints:56:1
# Defined: blueprint/Blueprints:55:1
build $
${g.bootstrap.buildDir}/.bootstrap/blueprint-pathtools/pkg/github.com/google/blueprint/pathtools.a $
: g.bootstrap.compile $
${g.bootstrap.srcDir}/blueprint/pathtools/lists.go $
${g.bootstrap.srcDir}/blueprint/pathtools/fs.go $
${g.bootstrap.srcDir}/blueprint/pathtools/glob.go | $
${g.bootstrap.compileCmd} $
${g.bootstrap.buildDir}/.bootstrap/blueprint-deptools/pkg/github.com/google/blueprint/deptools.a