tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity
platform_build_soong/finder/fs/readdir_test.go
Colin Cross ae7fd6baf3 Reimplement ioutil.ReadDir with a version that avoids calling lstat 
ioutil.ReadDir returns []os.FileInfo, which contains information on
each entry in the directory that is only available by calling
os.Lstat on the entry.  Finder only the name and type (regular,
directory or symlink) of the files, which on Linux kernels >= 2.6.4
is available in the return values of syscall.Getdents.

Replace ioutil.ReadDir with a call that uses syscall.Getdents
directly and collects the type information from the result.

Testing with:
rm -f /tmp/db && strace -fc finder -names Android.mk,Android.bp,Blueprints,CleanSpec.mk,TEST_MAPPING -exclude-dirs .git,.repo -prune-files .out-dir,.find-ignore -db /tmp/db .

Before:
  7.01   52.688304          63    833398         1 lstat
  1.90   14.246644          68    210523           getdents64
  1.25    9.370471          90    104286         1 openat

After:
  3.48   12.201385         117    104286         1 openat
  3.06   10.729138          51    210523           getdents64
  1.70    5.951892          57    104283         1 lstat

Pros:
Avoids 729115 calls to lstat.

Cons:
Requires copying ~200 lines of finicky buffer parsing code.
Puts all getdents calls (and possibly fallback lstat calls) onto
a non-blocking file descriptor, which will cause it to block a
thread and not just a goroutine.
Only works on Linux and Darwin.

Bug: 70897635
Test: m checkbuild
Change-Id: Iab9f82c38c8675d0b73b4e90540bb9e4d2ee52c1
2017-12-22 13:56:17 -08:00

312 lines
8.9 KiB
Go
Raw Blame History

// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fs
import (
"os"
"reflect"
"runtime"
"testing"
)
func TestParseDirent(t *testing.T) {
testCases := []struct {
name string
in []byte
out []*dirEntryInfo
}{
{
// Test that type DT_DIR is translated to os.ModeDir
name: "dir",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_paths", os.ModeDir, true},
},
},
{
// Test that type DT_REG is translated to a regular file
name: "file",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x08,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_paths", 0, true},
},
},
{
// Test that type DT_LNK is translated to a regular os.ModeSymlink
name: "symlink",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x0a,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_paths", os.ModeSymlink, true},
},
},
{
// Test that type DT_UNKNOWN sets modeExists: false
name: "unknown",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x00,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_paths", 0, false},
},
},
{
// Test a name with no padding after the null terminator
name: "no padding",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x20, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x00,
},
out: []*dirEntryInfo{
{".module_path", os.ModeDir, true},
},
},
{
// Test two sequential entries
name: "two entries",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x74,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_paths", os.ModeDir, true},
{".module_patht", os.ModeDir, true},
},
},
{
// Test two sequential entries with no padding between them
name: "two entries no padding",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x20, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x00,
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_path", os.ModeDir, true},
{".module_paths", os.ModeDir, true},
},
},
{
// Test an empty buffer. This shouldn't happen in practice because
// readdir doesn't call parseDirent if no bytes were returned.
name: "empty",
in: []byte{},
out: nil,
},
{
name: "missing null terminator",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x20, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
},
out: []*dirEntryInfo{
{".module_paths", os.ModeDir, true},
},
},
{
// Test two sequential entries where the first has an incorrect d_reclen.
// Should return with no entries.
name: "two entries first malformed",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x10, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x00,
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: nil,
},
{
// Test two sequential entries where the second has an incorrect d_reclen.
// Should return the first entry.
name: "two entries second malformed",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x28, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x00,
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x10, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
out: []*dirEntryInfo{
{".module_path", os.ModeDir, true},
},
},
{
// Test a reclen that goes past the end of the buffer.
name: "overrun",
in: []byte{
// __ino64_t d_ino;
0xfb, 0x10, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00,
// __off64_t d_off;
0xeb, 0x85, 0x20, 0x91, 0xb9, 0x14, 0x34, 0x03,
// unsigned short int d_reclen;
0x30, 0x00,
// unsigned char d_type;
0x04,
// char d_name[];
0x2e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x00,
},
out: nil,
},
}
if runtime.GOOS != "linux" {
t.Skip("depends on Linux definitions of syscall.Dirent")
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
entries := parseDirent(testCase.in, nil)
if !reflect.DeepEqual(testCase.out, entries) {
t.Fatalf("expected:\n %v\ngot:\n %v\n", testCase.out, entries)
}
})
}
}
Reference in a new issue View git blame Copy permalink