diff --git a/pathtools/fs.go b/pathtools/fs.go index ed1251b..b959289 100644 --- a/pathtools/fs.go +++ b/pathtools/fs.go @@ -89,7 +89,7 @@ type ReaderAtSeekerCloser interface { } type FileSystem interface { - // Open opens a file for reading. Follows symlinks. + // Open opens a file for reading. Follows symlinks. Open(name string) (ReaderAtSeekerCloser, error) // Exists returns whether the file exists and whether it is a directory. Follows symlinks. @@ -124,11 +124,29 @@ type FileSystem interface { // osFs implements FileSystem using the local disk. type osFs struct { - srcDir string + srcDir string + openFilesChan chan bool } func NewOsFs(path string) FileSystem { - return &osFs{srcDir: path} + // Darwin has a default limit of 256 open files, rate limit open files to 200 + limit := 200 + return &osFs{ + srcDir: path, + openFilesChan: make(chan bool, limit), + } +} + +func (fs *osFs) acquire() { + if fs.openFilesChan != nil { + fs.openFilesChan <- true + } +} + +func (fs *osFs) release() { + if fs.openFilesChan != nil { + <-fs.openFilesChan + } } func (fs *osFs) toAbs(path string) string { @@ -163,11 +181,31 @@ func (fs *osFs) removeSrcDirPrefixes(paths []string) []string { return paths } +// OsFile wraps an os.File to also release open file descriptors semaphore on close +type OsFile struct { + *os.File + fs *osFs +} + +// Close closes file and releases the open file descriptor semaphore +func (f *OsFile) Close() error { + err := f.File.Close() + f.fs.release() + return err +} + func (fs *osFs) Open(name string) (ReaderAtSeekerCloser, error) { - return os.Open(fs.toAbs(name)) + fs.acquire() + f, err := os.Open(fs.toAbs(name)) + if err != nil { + return nil, err + } + return &OsFile{f, fs}, nil } func (fs *osFs) Exists(name string) (bool, bool, error) { + fs.acquire() + defer fs.release() stat, err := os.Stat(fs.toAbs(name)) if err == nil { return true, stat.IsDir(), nil @@ -179,6 +217,8 @@ func (fs *osFs) Exists(name string) (bool, bool, error) { } func (fs *osFs) IsDir(name string) (bool, error) { + fs.acquire() + defer fs.release() info, err := os.Stat(fs.toAbs(name)) if err != nil { return false, err @@ -187,6 +227,8 @@ func (fs *osFs) IsDir(name string) (bool, error) { } func (fs *osFs) IsSymlink(name string) (bool, error) { + fs.acquire() + defer fs.release() if info, err := os.Lstat(fs.toAbs(name)); err != nil { return false, err } else { @@ -199,16 +241,22 @@ func (fs *osFs) Glob(pattern string, excludes []string, follow ShouldFollowSymli } func (fs *osFs) glob(pattern string) ([]string, error) { + fs.acquire() + defer fs.release() paths, err := filepath.Glob(fs.toAbs(pattern)) fs.removeSrcDirPrefixes(paths) return paths, err } func (fs *osFs) Lstat(path string) (stats os.FileInfo, err error) { + fs.acquire() + defer fs.release() return os.Lstat(fs.toAbs(path)) } func (fs *osFs) Stat(path string) (stats os.FileInfo, err error) { + fs.acquire() + defer fs.release() return os.Stat(fs.toAbs(path)) } @@ -218,6 +266,8 @@ func (fs *osFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dir } func (fs *osFs) ReadDirNames(name string) ([]string, error) { + fs.acquire() + defer fs.release() dir, err := os.Open(fs.toAbs(name)) if err != nil { return nil, err @@ -234,6 +284,8 @@ func (fs *osFs) ReadDirNames(name string) ([]string, error) { } func (fs *osFs) Readlink(name string) (string, error) { + fs.acquire() + defer fs.release() return os.Readlink(fs.toAbs(name)) }