tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add some tests for singlethreaded usage of the Finder.

Browse source 
Also change the existing tests to use exactly 2 threads.
Also update some tests to also same number of threads as will
be used by the finder.

Bug: 62455338
Test: m -j # which runs unit tests

Change-Id: I2b9d39680ecddd6d28c951df982ac51c077d2777
This commit is contained in:
Jeff Gaston 2017-08-22 14:11:15 -07:00
parent 996716a8e2
commit d311952e93
2 changed files with 102 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
finder/finder.go
View file

@ -18,6 +18,7 @@ import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
@ -30,7 +31,6 @@ import (
"time"
"android/soong/fs"
"errors"
)
// This file provides a Finder struct that can quickly search for files satisfying
@ -159,11 +159,17 @@ type Finder struct {
nodes pathMap
}
var defaultNumThreads = runtime.NumCPU() * 2
// New creates a new Finder for use
func New(cacheParams CacheParams, filesystem fs.FileSystem,
logger Logger, dbPath string) (f *Finder, err error) {
return newImpl(cacheParams, filesystem, logger, dbPath, defaultNumThreads)
}
numThreads := runtime.NumCPU() * 2
// newImpl is like New but accepts more params
func newImpl(cacheParams CacheParams, filesystem fs.FileSystem,
logger Logger, dbPath string, numThreads int) (f *Finder, err error) {
numDbLoadingThreads := numThreads
numSearchingThreads := numThreads

152
finder/finder_test.go
View file

@ -35,14 +35,18 @@ func newFs() *fs.MockFs {
}
func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Finder {
f, err := newFinderAndErr(t, filesystem, cacheParams)
return newFinderWithNumThreads(t, filesystem, cacheParams, 2)
}
func newFinderWithNumThreads(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) *Finder {
f, err := newFinderAndErr(t, filesystem, cacheParams, numThreads)
if err != nil {
fatal(t, err.Error())
}
return f
}
func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) (*Finder, error) {
func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) (*Finder, error) {
cachePath := "/finder/finder-db"
cacheDir := filepath.Dir(cachePath)
filesystem.MkDirs(cacheDir)
@ -51,7 +55,7 @@ func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParam
}
logger := log.New(ioutil.Discard, "", 0)
f, err := New(cacheParams, filesystem, logger, cachePath)
f, err := newImpl(cacheParams, filesystem, logger, cachePath, numThreads)
return f, err
}
@ -64,11 +68,13 @@ func finderWithSameParams(t *testing.T, original *Finder) *Finder {
}
func finderAndErrorWithSameParams(t *testing.T, original *Finder) (*Finder, error) {
f, err := New(
f, err := newImpl(
original.cacheMetadata.Config.CacheParams,
original.filesystem,
original.logger,
original.DbPath)
original.DbPath,
original.numDbLoadingThreads,
)
return f, err
}
@ -234,6 +240,21 @@ func runSimpleTest(t *testing.T, existentPaths []string, expectedMatches []strin
assertSameResponse(t, foundPaths, absoluteMatches)
}
// testAgainstSeveralThreadcounts runs the given test for each threadcount that we care to test
func testAgainstSeveralThreadcounts(t *testing.T, tester func(t *testing.T, numThreads int)) {
// test singlethreaded, multithreaded, and also using the same number of threads as
// will be used on the current system
threadCounts := []int{1, 2, defaultNumThreads}
for _, numThreads := range threadCounts {
testName := fmt.Sprintf("%v threads", numThreads)
// store numThreads in a new variable to prevent numThreads from changing in each loop
localNumThreads := numThreads
t.Run(testName, func(t *testing.T) {
tester(t, localNumThreads)
})
}
}
// end of utils, start of individual tests
func TestSingleFile(t *testing.T) {
@ -285,24 +306,30 @@ func TestEmptyPath(t *testing.T) {
}
func TestFilesystemRoot(t *testing.T) {
filesystem := newFs()
root := "/"
createdPath := "/findme.txt"
create(t, createdPath, filesystem)
finder := newFinder(
t,
filesystem,
CacheParams{
RootDirs: []string{root},
IncludeFiles: []string{"findme.txt", "skipme.txt"},
},
)
defer finder.Shutdown()
testWithNumThreads := func(t *testing.T, numThreads int) {
filesystem := newFs()
root := "/"
createdPath := "/findme.txt"
create(t, createdPath, filesystem)
foundPaths := finder.FindNamedAt(root, "findme.txt")
finder := newFinderWithNumThreads(
t,
filesystem,
CacheParams{
RootDirs: []string{root},
IncludeFiles: []string{"findme.txt", "skipme.txt"},
},
numThreads,
)
defer finder.Shutdown()
assertSameResponse(t, foundPaths, []string{createdPath})
foundPaths := finder.FindNamedAt(root, "findme.txt")
assertSameResponse(t, foundPaths, []string{createdPath})
}
testAgainstSeveralThreadcounts(t, testWithNumThreads)
}
func TestNonexistentDir(t *testing.T) {
@ -316,6 +343,7 @@ func TestNonexistentDir(t *testing.T) {
RootDirs: []string{"/tmp/IDontExist"},
IncludeFiles: []string{"findme.txt", "skipme.txt"},
},
1,
)
if err == nil {
fatal(t, "Did not fail when given a nonexistent root directory")
@ -380,6 +408,8 @@ func TestPruneFiles(t *testing.T) {
"/tmp/include/findme.txt"})
}
// TestRootDir tests that the value of RootDirs is used
// tests of the filesystem root are in TestFilesystemRoot
func TestRootDir(t *testing.T) {
filesystem := newFs()
create(t, "/tmp/a/findme.txt", filesystem)
@ -548,48 +578,54 @@ func TestFindFirst(t *testing.T) {
}
func TestConcurrentFindSameDirectory(t *testing.T) {
filesystem := newFs()
// create a bunch of files and directories
paths := []string{}
for i := 0; i < 10; i++ {
parentDir := fmt.Sprintf("/tmp/%v", i)
for j := 0; j < 10; j++ {
filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
paths = append(paths, filePath)
testWithNumThreads := func(t *testing.T, numThreads int) {
filesystem := newFs()
// create a bunch of files and directories
paths := []string{}
for i := 0; i < 10; i++ {
parentDir := fmt.Sprintf("/tmp/%v", i)
for j := 0; j < 10; j++ {
filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
paths = append(paths, filePath)
}
}
sort.Strings(paths)
for _, path := range paths {
create(t, path, filesystem)
}
// set up a finder
finder := newFinderWithNumThreads(
t,
filesystem,
CacheParams{
RootDirs: []string{"/tmp"},
IncludeFiles: []string{"findme.txt"},
},
numThreads,
)
defer finder.Shutdown()
numTests := 20
results := make(chan []string, numTests)
// make several parallel calls to the finder
for i := 0; i < numTests; i++ {
go func() {
foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
results <- foundPaths
}()
}
// check that each response was correct
for i := 0; i < numTests; i++ {
foundPaths := <-results
assertSameResponse(t, foundPaths, paths)
}
}
sort.Strings(paths)
for _, path := range paths {
create(t, path, filesystem)
}
// set up a finder
finder := newFinder(
t,
filesystem,
CacheParams{
RootDirs: []string{"/tmp"},
IncludeFiles: []string{"findme.txt"},
},
)
defer finder.Shutdown()
numTests := 20
results := make(chan []string, numTests)
// make several parallel calls to the finder
for i := 0; i < numTests; i++ {
go func() {
foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
results <- foundPaths
}()
}
// check that each response was correct
for i := 0; i < numTests; i++ {
foundPaths := <-results
assertSameResponse(t, foundPaths, paths)
}
testAgainstSeveralThreadcounts(t, testWithNumThreads)
}
func TestConcurrentFindDifferentDirectories(t *testing.T) {