Merge pull request #234 from gyias/bpdoc

bpdoc improvements

Blueprints

@@ -119,6 +119,11 @@ bootstrap_go_package {
     pkgPath: "github.com/google/blueprint/bootstrap/bpdoc",
     srcs: [
         "bootstrap/bpdoc/bpdoc.go",
+        "bootstrap/bpdoc/properties.go",
+        "bootstrap/bpdoc/reader.go",
+    ],
+    testSrcs: [
+        "bootstrap/bpdoc/reader_test.go",
     ],
 }
 

bootstrap/bpdoc/bpdoc.go

@@ -2,92 +2,46 @@ package bpdoc
 
 import (
 	"fmt"
-	"go/ast"
-	"go/doc"
-	"go/parser"
-	"go/token"
 	"html/template"
 	"reflect"
 	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"unicode"
-	"unicode/utf8"
 
 	"github.com/google/blueprint"
 	"github.com/google/blueprint/proptools"
 )
 
-type Context struct {
-	pkgFiles map[string][]string // Map of package name to source files, provided by constructor
+// Package contains the information about a package relevant to generating documentation.
+type Package struct {
+	// Name is the name of the package.
+	Name string
 
-	mutex sync.Mutex
-	pkgs  map[string]*doc.Package    // Map of package name to parsed Go AST, protected by mutex
-	ps    map[string]*PropertyStruct // Map of type name to property struct, protected by mutex
+	// Path is the full package path of the package as used in the primary builder.
+	Path string
+
+	// Text is the contents of the package comment documenting the module types in the package.
+	Text string
+
+	// ModuleTypes is a list of ModuleType objects that contain information about each module type that is
+	// defined by the package.
+	ModuleTypes []*ModuleType
 }
 
-func NewContext(pkgFiles map[string][]string) *Context {
-	return &Context{
-		pkgFiles: pkgFiles,
-		pkgs:     make(map[string]*doc.Package),
-		ps:       make(map[string]*PropertyStruct),
-	}
-}
+// ModuleType contains the information about a module type that is relevant to generating documentation.
+type ModuleType struct {
+	// Name is the string that will appear in Blueprints files when defining a new module of
+	// this type.
+	Name string
 
-// Return the PropertyStruct associated with a property struct type.  The type should be in the
-// format <package path>.<type name>
-func (c *Context) PropertyStruct(pkgPath, name string, defaults reflect.Value) (*PropertyStruct, error) {
-	ps := c.getPropertyStruct(pkgPath, name)
+	// PkgPath is the full package path of the package that contains the module type factory.
+	PkgPath string
 
-	if ps == nil {
-		pkg, err := c.pkg(pkgPath)
-		if err != nil {
-			return nil, err
-		}
+	// Text is the contents of the comment documenting the module type.
+	Text string
 
-		for _, t := range pkg.Types {
-			if t.Name == name {
-				ps, err = newPropertyStruct(t)
-				if err != nil {
-					return nil, err
-				}
-				ps = c.putPropertyStruct(pkgPath, name, ps)
-			}
-		}
-	}
-
-	if ps == nil {
-		return nil, fmt.Errorf("package %q type %q not found", pkgPath, name)
-	}
-
-	ps = ps.Clone()
-	ps.SetDefaults(defaults)
-
-	return ps, nil
-}
-
-func (c *Context) getPropertyStruct(pkgPath, name string) *PropertyStruct {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-
-	name = pkgPath + "." + name
-
-	return c.ps[name]
-}
-
-func (c *Context) putPropertyStruct(pkgPath, name string, ps *PropertyStruct) *PropertyStruct {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-
-	name = pkgPath + "." + name
-
-	if c.ps[name] != nil {
-		return c.ps[name]
-	} else {
-		c.ps[name] = ps
-		return ps
-	}
+	// PropertyStructs is a list of PropertyStruct objects that contain information about each
+	// property struct that is used by the module type, containing all properties that are valid
+	// for the module type.
+	PropertyStructs []*PropertyStruct
 }
 
 type PropertyStruct struct {
@@ -107,338 +61,60 @@ type Property struct {
 	Default    string
 }
 
-func (ps *PropertyStruct) Clone() *PropertyStruct {
-	ret := *ps
-	ret.Properties = append([]Property(nil), ret.Properties...)
-	for i, prop := range ret.Properties {
-		ret.Properties[i] = prop.Clone()
-	}
+func AllPackages(pkgFiles map[string][]string, moduleTypeNameFactories map[string]reflect.Value,
+	moduleTypeNamePropertyStructs map[string][]interface{}) ([]*Package, error) {
+	// Read basic info from the files to construct a Reader instance.
+	r := NewReader(pkgFiles)
 
-	return &ret
-}
-
-func (p *Property) Clone() Property {
-	ret := *p
-	ret.Properties = append([]Property(nil), ret.Properties...)
-	for i, prop := range ret.Properties {
-		ret.Properties[i] = prop.Clone()
-	}
-
-	return ret
-}
-
-func (p *Property) Equal(other Property) bool {
-	return p.Name == other.Name && p.Type == other.Type && p.Tag == other.Tag &&
-		p.Text == other.Text && p.Default == other.Default &&
-		stringArrayEqual(p.OtherNames, other.OtherNames) &&
-		htmlArrayEqual(p.OtherTexts, other.OtherTexts) &&
-		p.SameSubProperties(other)
-}
-
-func (ps *PropertyStruct) SetDefaults(defaults reflect.Value) {
-	setDefaults(ps.Properties, defaults)
-}
-
-func setDefaults(properties []Property, defaults reflect.Value) {
-	for i := range properties {
-		prop := &properties[i]
-		fieldName := proptools.FieldNameForProperty(prop.Name)
-		f := defaults.FieldByName(fieldName)
-		if (f == reflect.Value{}) {
-			panic(fmt.Errorf("property %q does not exist in %q", fieldName, defaults.Type()))
+	pkgMap := map[string]*Package{}
+	var pkgs []*Package
+	// Scan through per-module-type property structs map.
+	for mtName, propertyStructs := range moduleTypeNamePropertyStructs {
+		// Construct ModuleType with the given info.
+		mtInfo, err := assembleModuleTypeInfo(r, mtName, moduleTypeNameFactories[mtName], propertyStructs)
+		if err != nil {
+			return nil, err
 		}
+		// Some pruning work
+		removeEmptyPropertyStructs(mtInfo)
+		collapseDuplicatePropertyStructs(mtInfo)
+		collapseNestedPropertyStructs(mtInfo)
+		combineDuplicateProperties(mtInfo)
 
-		if reflect.DeepEqual(f.Interface(), reflect.Zero(f.Type()).Interface()) {
-			continue
-		}
-
-		if f.Kind() == reflect.Interface {
-			f = f.Elem()
-		}
-
-		if f.Kind() == reflect.Ptr {
-			if f.IsNil() {
-				continue
+		// Add the ModuleInfo to the corresponding Package map/slice entries.
+		pkg := pkgMap[mtInfo.PkgPath]
+		if pkg == nil {
+			var err error
+			pkg, err = r.Package(mtInfo.PkgPath)
+			if err != nil {
+				return nil, err
 			}
-			f = f.Elem()
-		}
-
-		if f.Kind() == reflect.Struct {
-			setDefaults(prop.Properties, f)
-		} else {
-			prop.Default = fmt.Sprintf("%v", f.Interface())
+			pkgMap[mtInfo.PkgPath] = pkg
+			pkgs = append(pkgs, pkg)
 		}
+		pkg.ModuleTypes = append(pkg.ModuleTypes, mtInfo)
 	}
+
+	// Sort ModuleTypes within each package.
+	for _, pkg := range pkgs {
+		sort.Slice(pkg.ModuleTypes, func(i, j int) bool { return pkg.ModuleTypes[i].Name < pkg.ModuleTypes[j].Name })
+	}
+	// Sort packages.
+	sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].Path < pkgs[j].Path })
+
+	return pkgs, nil
 }
 
-func stringArrayEqual(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
+func assembleModuleTypeInfo(r *Reader, name string, factory reflect.Value,
+	propertyStructs []interface{}) (*ModuleType, error) {
 
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-
-	return true
-}
-
-func htmlArrayEqual(a, b []template.HTML) bool {
-	if len(a) != len(b) {
-		return false
-	}
-
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-
-	return true
-}
-
-func (p *Property) SameSubProperties(other Property) bool {
-	if len(p.Properties) != len(other.Properties) {
-		return false
-	}
-
-	for i := range p.Properties {
-		if !p.Properties[i].Equal(other.Properties[i]) {
-			return false
-		}
-	}
-
-	return true
-}
-
-func (ps *PropertyStruct) GetByName(name string) *Property {
-	return getByName(name, "", &ps.Properties)
-}
-
-func getByName(name string, prefix string, props *[]Property) *Property {
-	for i := range *props {
-		if prefix+(*props)[i].Name == name {
-			return &(*props)[i]
-		} else if strings.HasPrefix(name, prefix+(*props)[i].Name+".") {
-			return getByName(name, prefix+(*props)[i].Name+".", &(*props)[i].Properties)
-		}
-	}
-	return nil
-}
-
-func (p *Property) Nest(nested *PropertyStruct) {
-	//p.Name += "(" + nested.Name + ")"
-	//p.Text += "(" + nested.Text + ")"
-	p.Properties = append(p.Properties, nested.Properties...)
-}
-
-func newPropertyStruct(t *doc.Type) (*PropertyStruct, error) {
-	typeSpec := t.Decl.Specs[0].(*ast.TypeSpec)
-	ps := PropertyStruct{
-		Name: t.Name,
-		Text: t.Doc,
-	}
-
-	structType, ok := typeSpec.Type.(*ast.StructType)
-	if !ok {
-		return nil, fmt.Errorf("type of %q is not a struct", t.Name)
-	}
-
-	var err error
-	ps.Properties, err = structProperties(structType)
+	mt, err := r.ModuleType(name, factory)
 	if err != nil {
 		return nil, err
 	}
 
-	return &ps, nil
-}
-
-func structProperties(structType *ast.StructType) (props []Property, err error) {
-	for _, f := range structType.Fields.List {
-		names := f.Names
-		if names == nil {
-			// Anonymous fields have no name, use the type as the name
-			// TODO: hide the name and make the properties show up in the embedding struct
-			if t, ok := f.Type.(*ast.Ident); ok {
-				names = append(names, t)
-			}
-		}
-		for _, n := range names {
-			var name, typ, tag, text string
-			var innerProps []Property
-			if n != nil {
-				name = proptools.PropertyNameForField(n.Name)
-			}
-			if f.Doc != nil {
-				text = f.Doc.Text()
-			}
-			if f.Tag != nil {
-				tag, err = strconv.Unquote(f.Tag.Value)
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			t := f.Type
-			if star, ok := t.(*ast.StarExpr); ok {
-				t = star.X
-			}
-			switch a := t.(type) {
-			case *ast.ArrayType:
-				typ = "list of strings"
-			case *ast.InterfaceType:
-				typ = "interface"
-			case *ast.Ident:
-				typ = a.Name
-			case *ast.StructType:
-				innerProps, err = structProperties(a)
-				if err != nil {
-					return nil, err
-				}
-			default:
-				typ = fmt.Sprintf("%T", f.Type)
-			}
-
-			var html template.HTML
-
-			lines := strings.Split(text, "\n")
-			preformatted := false
-			for _, line := range lines {
-				r, _ := utf8.DecodeRuneInString(line)
-				indent := unicode.IsSpace(r)
-				if indent && !preformatted {
-					html += "<pre>\n"
-				} else if !indent && preformatted {
-					html += "</pre>\n"
-				}
-				preformatted = indent
-				html += template.HTML(template.HTMLEscapeString(line)) + "\n"
-			}
-			if preformatted {
-				html += "</pre>\n"
-			}
-
-			props = append(props, Property{
-				Name:       name,
-				Type:       typ,
-				Tag:        reflect.StructTag(tag),
-				Text:       html,
-				Properties: innerProps,
-			})
-		}
-	}
-
-	return props, nil
-}
-
-func (ps *PropertyStruct) ExcludeByTag(key, value string) {
-	filterPropsByTag(&ps.Properties, key, value, true)
-}
-
-func (ps *PropertyStruct) IncludeByTag(key, value string) {
-	filterPropsByTag(&ps.Properties, key, value, false)
-}
-
-func filterPropsByTag(props *[]Property, key, value string, exclude bool) {
-	// Create a slice that shares the storage of props but has 0 length.  Appending up to
-	// len(props) times to this slice will overwrite the original slice contents
-	filtered := (*props)[:0]
-	for _, x := range *props {
-		tag := x.Tag.Get(key)
-		for _, entry := range strings.Split(tag, ",") {
-			if (entry == value) == !exclude {
-				filtered = append(filtered, x)
-			}
-		}
-	}
-
-	*props = filtered
-}
-
-// Package AST generation and storage
-func (c *Context) pkg(pkgPath string) (*doc.Package, error) {
-	pkg := c.getPackage(pkgPath)
-	if pkg == nil {
-		if files, ok := c.pkgFiles[pkgPath]; ok {
-			var err error
-			pkgAST, err := NewPackageAST(files)
-			if err != nil {
-				return nil, err
-			}
-			pkg = doc.New(pkgAST, pkgPath, doc.AllDecls)
-			pkg = c.putPackage(pkgPath, pkg)
-		} else {
-			return nil, fmt.Errorf("unknown package %q", pkgPath)
-		}
-	}
-	return pkg, nil
-}
-
-func (c *Context) getPackage(pkgPath string) *doc.Package {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-
-	return c.pkgs[pkgPath]
-}
-
-func (c *Context) putPackage(pkgPath string, pkg *doc.Package) *doc.Package {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-
-	if c.pkgs[pkgPath] != nil {
-		return c.pkgs[pkgPath]
-	} else {
-		c.pkgs[pkgPath] = pkg
-		return pkg
-	}
-}
-
-func NewPackageAST(files []string) (*ast.Package, error) {
-	asts := make(map[string]*ast.File)
-
-	fset := token.NewFileSet()
-	for _, file := range files {
-		ast, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
-		if err != nil {
-			return nil, err
-		}
-		asts[file] = ast
-	}
-
-	pkg, _ := ast.NewPackage(fset, asts, nil, nil)
-	return pkg, nil
-}
-
-func ModuleTypes(pkgFiles map[string][]string, moduleTypePropertyStructs map[string][]interface{}) ([]*ModuleType, error) {
-	c := NewContext(pkgFiles)
-
-	var moduleTypeList []*ModuleType
-	for moduleType, propertyStructs := range moduleTypePropertyStructs {
-		mt, err := getModuleType(c, moduleType, propertyStructs)
-		if err != nil {
-			return nil, err
-		}
-		removeEmptyPropertyStructs(mt)
-		collapseDuplicatePropertyStructs(mt)
-		collapseNestedPropertyStructs(mt)
-		combineDuplicateProperties(mt)
-		moduleTypeList = append(moduleTypeList, mt)
-	}
-
-	sort.Sort(moduleTypeByName(moduleTypeList))
-
-	return moduleTypeList, nil
-}
-
-func getModuleType(c *Context, moduleTypeName string,
-	propertyStructs []interface{}) (*ModuleType, error) {
-	mt := &ModuleType{
-		Name: moduleTypeName,
-		//Text: c.ModuleTypeDocs(moduleType),
-	}
-
+	// Reader.ModuleType only fills basic information such as name and package path. Collect more info
+	// from property struct data.
 	for _, s := range propertyStructs {
 		v := reflect.ValueOf(s).Elem()
 		t := v.Type()
@@ -447,7 +123,7 @@ func getModuleType(c *Context, moduleTypeName string,
 		if t.PkgPath() == "" {
 			continue
 		}
-		ps, err := c.PropertyStruct(t.PkgPath(), t.Name(), v)
+		ps, err := r.PropertyStruct(t.PkgPath(), t.Name(), v)
 		if err != nil {
 			return nil, err
 		}
@@ -460,7 +136,7 @@ func getModuleType(c *Context, moduleTypeName string,
 			if nestedType.PkgPath() == "" {
 				continue
 			}
-			nested, err := c.PropertyStruct(nestedType.PkgPath(), nestedType.Name(), nestedValue)
+			nested, err := r.PropertyStruct(nestedType.PkgPath(), nestedType.Name(), nestedValue)
 			if err != nil {
 				return nil, err
 			}
@@ -528,7 +204,6 @@ func nestedPropertyStructs(s reflect.Value) map[string]reflect.Value {
 					field.Name, fieldValue.Kind()))
 			}
 		}
-
 	}
 
 	walk(s, "")
@@ -633,27 +308,5 @@ propertyLoop:
 		}
 		n = append(n, child)
 	}
-
 	*p = n
 }
-
-type moduleTypeByName []*ModuleType
-
-func (l moduleTypeByName) Len() int           { return len(l) }
-func (l moduleTypeByName) Less(i, j int) bool { return l[i].Name < l[j].Name }
-func (l moduleTypeByName) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
-
-// ModuleType contains the info about a module type that is relevant to generating documentation.
-type ModuleType struct {
-	// Name is the string that will appear in Blueprints files when defining a new module of
-	// this type.
-	Name string
-
-	// Text is the contents of the comment documenting the module type
-	Text string
-
-	// PropertyStructs is a list of PropertyStruct objects that contain information about each
-	// property struct that is used by the module type, containing all properties that are valid
-	// for the module type.
-	PropertyStructs []*PropertyStruct
-}

bootstrap/bpdoc/properties.go

@@ -0,0 +1,281 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bpdoc
+
+import (
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"html/template"
+	"reflect"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/google/blueprint/proptools"
+)
+
+//
+// Utility functions for PropertyStruct and Property
+//
+
+func (ps *PropertyStruct) Clone() *PropertyStruct {
+	ret := *ps
+	ret.Properties = append([]Property(nil), ret.Properties...)
+	for i, prop := range ret.Properties {
+		ret.Properties[i] = prop.Clone()
+	}
+
+	return &ret
+}
+
+func (p *Property) Clone() Property {
+	ret := *p
+	ret.Properties = append([]Property(nil), ret.Properties...)
+	for i, prop := range ret.Properties {
+		ret.Properties[i] = prop.Clone()
+	}
+
+	return ret
+}
+
+func (p *Property) Equal(other Property) bool {
+	return p.Name == other.Name && p.Type == other.Type && p.Tag == other.Tag &&
+		p.Text == other.Text && p.Default == other.Default &&
+		stringArrayEqual(p.OtherNames, other.OtherNames) &&
+		htmlArrayEqual(p.OtherTexts, other.OtherTexts) &&
+		p.SameSubProperties(other)
+}
+
+func (ps *PropertyStruct) SetDefaults(defaults reflect.Value) {
+	setDefaults(ps.Properties, defaults)
+}
+
+func setDefaults(properties []Property, defaults reflect.Value) {
+	for i := range properties {
+		prop := &properties[i]
+		fieldName := proptools.FieldNameForProperty(prop.Name)
+		f := defaults.FieldByName(fieldName)
+		if (f == reflect.Value{}) {
+			panic(fmt.Errorf("property %q does not exist in %q", fieldName, defaults.Type()))
+		}
+
+		if reflect.DeepEqual(f.Interface(), reflect.Zero(f.Type()).Interface()) {
+			continue
+		}
+
+		if f.Kind() == reflect.Interface {
+			f = f.Elem()
+		}
+
+		if f.Kind() == reflect.Ptr {
+			if f.IsNil() {
+				continue
+			}
+			f = f.Elem()
+		}
+
+		if f.Kind() == reflect.Struct {
+			setDefaults(prop.Properties, f)
+		} else {
+			prop.Default = fmt.Sprintf("%v", f.Interface())
+		}
+	}
+}
+
+func stringArrayEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func htmlArrayEqual(a, b []template.HTML) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (p *Property) SameSubProperties(other Property) bool {
+	if len(p.Properties) != len(other.Properties) {
+		return false
+	}
+
+	for i := range p.Properties {
+		if !p.Properties[i].Equal(other.Properties[i]) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ps *PropertyStruct) GetByName(name string) *Property {
+	return getByName(name, "", &ps.Properties)
+}
+
+func getByName(name string, prefix string, props *[]Property) *Property {
+	for i := range *props {
+		if prefix+(*props)[i].Name == name {
+			return &(*props)[i]
+		} else if strings.HasPrefix(name, prefix+(*props)[i].Name+".") {
+			return getByName(name, prefix+(*props)[i].Name+".", &(*props)[i].Properties)
+		}
+	}
+	return nil
+}
+
+func (p *Property) Nest(nested *PropertyStruct) {
+	p.Properties = append(p.Properties, nested.Properties...)
+}
+
+func newPropertyStruct(t *doc.Type) (*PropertyStruct, error) {
+	typeSpec := t.Decl.Specs[0].(*ast.TypeSpec)
+	ps := PropertyStruct{
+		Name: t.Name,
+		Text: t.Doc,
+	}
+
+	structType, ok := typeSpec.Type.(*ast.StructType)
+	if !ok {
+		return nil, fmt.Errorf("type of %q is not a struct", t.Name)
+	}
+
+	var err error
+	ps.Properties, err = structProperties(structType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ps, nil
+}
+
+func structProperties(structType *ast.StructType) (props []Property, err error) {
+	for _, f := range structType.Fields.List {
+		names := f.Names
+		if names == nil {
+			// Anonymous fields have no name, use the type as the name
+			// TODO: hide the name and make the properties show up in the embedding struct
+			if t, ok := f.Type.(*ast.Ident); ok {
+				names = append(names, t)
+			}
+		}
+		for _, n := range names {
+			var name, typ, tag, text string
+			var innerProps []Property
+			if n != nil {
+				name = proptools.PropertyNameForField(n.Name)
+			}
+			if f.Doc != nil {
+				text = f.Doc.Text()
+			}
+			if f.Tag != nil {
+				tag, err = strconv.Unquote(f.Tag.Value)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			t := f.Type
+			if star, ok := t.(*ast.StarExpr); ok {
+				t = star.X
+			}
+			switch a := t.(type) {
+			case *ast.ArrayType:
+				typ = "list of strings"
+			case *ast.InterfaceType:
+				typ = "interface"
+			case *ast.Ident:
+				typ = a.Name
+			case *ast.StructType:
+				innerProps, err = structProperties(a)
+				if err != nil {
+					return nil, err
+				}
+			default:
+				typ = fmt.Sprintf("%T", f.Type)
+			}
+
+			var html template.HTML
+
+			lines := strings.Split(text, "\n")
+			preformatted := false
+			for _, line := range lines {
+				r, _ := utf8.DecodeRuneInString(line)
+				indent := unicode.IsSpace(r)
+				if indent && !preformatted {
+					html += "<pre>\n"
+				} else if !indent && preformatted {
+					html += "</pre>\n"
+				}
+				preformatted = indent
+				html += template.HTML(template.HTMLEscapeString(line)) + "\n"
+			}
+			if preformatted {
+				html += "</pre>\n"
+			}
+
+			props = append(props, Property{
+				Name:       name,
+				Type:       typ,
+				Tag:        reflect.StructTag(tag),
+				Text:       html,
+				Properties: innerProps,
+			})
+		}
+	}
+
+	return props, nil
+}
+
+func (ps *PropertyStruct) ExcludeByTag(key, value string) {
+	filterPropsByTag(&ps.Properties, key, value, true)
+}
+
+func (ps *PropertyStruct) IncludeByTag(key, value string) {
+	filterPropsByTag(&ps.Properties, key, value, false)
+}
+
+func filterPropsByTag(props *[]Property, key, value string, exclude bool) {
+	// Create a slice that shares the storage of props but has 0 length.  Appending up to
+	// len(props) times to this slice will overwrite the original slice contents
+	filtered := (*props)[:0]
+	for _, x := range *props {
+		tag := x.Tag.Get(key)
+		for _, entry := range strings.Split(tag, ",") {
+			if (entry == value) == !exclude {
+				filtered = append(filtered, x)
+			}
+		}
+	}
+
+	*props = filtered
+}

bootstrap/bpdoc/reader.go

@@ -0,0 +1,227 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bpdoc
+
+import (
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// Handles parsing and low-level processing of Blueprint module source files. Note that most getter
+// functions associated with Reader only fill basic information that can be simply extracted from
+// AST parsing results. More sophisticated processing is performed in bpdoc.go
+type Reader struct {
+	pkgFiles map[string][]string // Map of package name to source files, provided by constructor
+
+	mutex  sync.Mutex
+	goPkgs map[string]*doc.Package    // Map of package name to parsed Go AST, protected by mutex
+	ps     map[string]*PropertyStruct // Map of module type name to property struct, protected by mutex
+}
+
+func NewReader(pkgFiles map[string][]string) *Reader {
+	return &Reader{
+		pkgFiles: pkgFiles,
+		goPkgs:   make(map[string]*doc.Package),
+		ps:       make(map[string]*PropertyStruct),
+	}
+}
+
+func (r *Reader) Package(path string) (*Package, error) {
+	goPkg, err := r.goPkg(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Package{
+		Name: goPkg.Name,
+		Path: path,
+		Text: goPkg.Doc,
+	}, nil
+}
+
+func (r *Reader) ModuleType(name string, factory reflect.Value) (*ModuleType, error) {
+	f := runtime.FuncForPC(factory.Pointer())
+
+	pkgPath, err := funcNameToPkgPath(f.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	factoryName := strings.TrimPrefix(f.Name(), pkgPath+".")
+
+	text, err := r.getModuleTypeDoc(pkgPath, factoryName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ModuleType{
+		Name:    name,
+		PkgPath: pkgPath,
+		Text:    text,
+	}, nil
+}
+
+// Return the PropertyStruct associated with a property struct type.  The type should be in the
+// format <package path>.<type name>
+func (r *Reader) PropertyStruct(pkgPath, name string, defaults reflect.Value) (*PropertyStruct, error) {
+	ps := r.getPropertyStruct(pkgPath, name)
+
+	if ps == nil {
+		pkg, err := r.goPkg(pkgPath)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, t := range pkg.Types {
+			if t.Name == name {
+				ps, err = newPropertyStruct(t)
+				if err != nil {
+					return nil, err
+				}
+				ps = r.putPropertyStruct(pkgPath, name, ps)
+			}
+		}
+	}
+
+	if ps == nil {
+		return nil, fmt.Errorf("package %q type %q not found", pkgPath, name)
+	}
+
+	ps = ps.Clone()
+	ps.SetDefaults(defaults)
+
+	return ps, nil
+}
+
+func (r *Reader) getModuleTypeDoc(pkgPath, factoryFuncName string) (string, error) {
+	goPkg, err := r.goPkg(pkgPath)
+	if err != nil {
+		return "", err
+	}
+
+	for _, fn := range goPkg.Funcs {
+		if fn.Name == factoryFuncName {
+			return fn.Doc, nil
+		}
+	}
+
+	// The doc package may associate the method with the type it returns, so iterate through those too
+	for _, typ := range goPkg.Types {
+		for _, fn := range typ.Funcs {
+			if fn.Name == factoryFuncName {
+				return fn.Doc, nil
+			}
+		}
+	}
+
+	return "", nil
+}
+
+func (r *Reader) getPropertyStruct(pkgPath, name string) *PropertyStruct {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	name = pkgPath + "." + name
+
+	return r.ps[name]
+}
+
+func (r *Reader) putPropertyStruct(pkgPath, name string, ps *PropertyStruct) *PropertyStruct {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	name = pkgPath + "." + name
+
+	if r.ps[name] != nil {
+		return r.ps[name]
+	} else {
+		r.ps[name] = ps
+		return ps
+	}
+}
+
+// Package AST generation and storage
+func (r *Reader) goPkg(pkgPath string) (*doc.Package, error) {
+	pkg := r.getGoPkg(pkgPath)
+	if pkg == nil {
+		if files, ok := r.pkgFiles[pkgPath]; ok {
+			var err error
+			pkgAST, err := packageAST(files)
+			if err != nil {
+				return nil, err
+			}
+			pkg = doc.New(pkgAST, pkgPath, doc.AllDecls)
+			pkg = r.putGoPkg(pkgPath, pkg)
+		} else {
+			return nil, fmt.Errorf("unknown package %q", pkgPath)
+		}
+	}
+	return pkg, nil
+}
+
+func (r *Reader) getGoPkg(pkgPath string) *doc.Package {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	return r.goPkgs[pkgPath]
+}
+
+func (r *Reader) putGoPkg(pkgPath string, pkg *doc.Package) *doc.Package {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	if r.goPkgs[pkgPath] != nil {
+		return r.goPkgs[pkgPath]
+	} else {
+		r.goPkgs[pkgPath] = pkg
+		return pkg
+	}
+}
+
+// A regex to find a package path within a function name. It finds the shortest string that is
+// followed by '.' and doesn't have any '/'s left.
+var pkgPathRe = regexp.MustCompile("^(.*?)\\.[^/]+$")
+
+func funcNameToPkgPath(f string) (string, error) {
+	s := pkgPathRe.FindStringSubmatch(f)
+	if len(s) < 2 {
+		return "", fmt.Errorf("failed to extract package path from %q", f)
+	}
+	return s[1], nil
+}
+
+func packageAST(files []string) (*ast.Package, error) {
+	asts := make(map[string]*ast.File)
+
+	fset := token.NewFileSet()
+	for _, file := range files {
+		ast, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
+		if err != nil {
+			return nil, err
+		}
+		asts[file] = ast
+	}
+
+	pkg, _ := ast.NewPackage(fset, asts, nil, nil)
+	return pkg, nil
+}

bootstrap/bpdoc/reader_test.go

@@ -0,0 +1,136 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// bpdoc docs.
+package bpdoc
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+
+	"github.com/google/blueprint"
+)
+
+// foo docs.
+func fooFactory() (blueprint.Module, []interface{}) {
+	return nil, []interface{}{&props{}}
+}
+
+// props docs.
+type props struct {
+	// A docs.
+	A string
+}
+
+var pkgPath string
+var pkgFiles map[string][]string
+
+func init() {
+	pc, filename, _, _ := runtime.Caller(0)
+	fn := runtime.FuncForPC(pc)
+
+	var err error
+	pkgPath, err = funcNameToPkgPath(fn.Name())
+	if err != nil {
+		panic(err)
+	}
+
+	pkgFiles = map[string][]string{
+		pkgPath: {filename},
+	}
+}
+
+func TestModuleTypeDocs(t *testing.T) {
+	r := NewReader(pkgFiles)
+	mt, err := r.ModuleType("foo_module", reflect.ValueOf(fooFactory))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if mt.Text != "foo docs.\n" {
+		t.Errorf("unexpected docs %q", mt.Text)
+	}
+
+	if mt.PkgPath != pkgPath {
+		t.Errorf("expected pkgpath %q, got %q", pkgPath, mt.PkgPath)
+	}
+}
+
+func TestPropertyStruct(t *testing.T) {
+	r := NewReader(pkgFiles)
+	ps, err := r.PropertyStruct(pkgPath, "props", reflect.ValueOf(props{A: "B"}))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if ps.Text != "props docs.\n" {
+		t.Errorf("unexpected docs %q", ps.Text)
+	}
+	if len(ps.Properties) != 1 {
+		t.Fatalf("want 1 property, got %d", len(ps.Properties))
+	}
+
+	if ps.Properties[0].Name != "a" || ps.Properties[0].Text != "A docs.\n\n" || ps.Properties[0].Default != "B" {
+		t.Errorf("unexpected property docs %q %q %q",
+			ps.Properties[0].Name, ps.Properties[0].Text, ps.Properties[0].Default)
+	}
+}
+
+func TestPackage(t *testing.T) {
+	r := NewReader(pkgFiles)
+	pkg, err := r.Package(pkgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pkg.Text != "bpdoc docs.\n" {
+		t.Errorf("unexpected docs %q", pkg.Text)
+	}
+}
+
+func TestFuncToPkgPath(t *testing.T) {
+	tests := []struct {
+		f    string
+		want string
+	}{
+		{
+			f:    "github.com/google/blueprint/bootstrap.Main",
+			want: "github.com/google/blueprint/bootstrap",
+		},
+		{
+			f:    "android/soong/android.GenruleFactory",
+			want: "android/soong/android",
+		},
+		{
+			f:    "android/soong/android.ModuleFactoryAdapter.func1",
+			want: "android/soong/android",
+		},
+		{
+			f:    "main.Main",
+			want: "main",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.f, func(t *testing.T) {
+			got, err := funcNameToPkgPath(tt.f)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tt.want {
+				t.Errorf("funcNameToPkgPath(%v) = %v, want %v", tt.f, got, tt.want)
+			}
+		})
+	}
+}

bootstrap/writedocs.go

@@ -6,6 +6,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"path/filepath"
+	"reflect"
 
 	"github.com/google/blueprint"
 	"github.com/google/blueprint/bootstrap/bpdoc"
@@ -14,7 +15,7 @@ import (
 
 // ModuleTypeDocs returns a list of bpdoc.ModuleType objects that contain information relevant
 // to generating documentation for module types supported by the primary builder.
-func ModuleTypeDocs(ctx *blueprint.Context) ([]*bpdoc.ModuleType, error) {
+func ModuleTypeDocs(ctx *blueprint.Context, factories map[string]reflect.Value) ([]*bpdoc.Package, error) {
 	// Find the module that's marked as the "primary builder", which means it's
 	// creating the binary that we'll use to generate the non-bootstrap
 	// build.ninja file.
@@ -60,11 +61,22 @@ func ModuleTypeDocs(ctx *blueprint.Context) ([]*bpdoc.ModuleType, error) {
 		}
 	})
 
-	return bpdoc.ModuleTypes(pkgFiles, ctx.ModuleTypePropertyStructs())
+	mergedFactories := make(map[string]reflect.Value)
+	for moduleType, factory := range factories {
+		mergedFactories[moduleType] = factory
+	}
+
+	for moduleType, factory := range ctx.ModuleTypeFactories() {
+		if _, exists := mergedFactories[moduleType]; !exists {
+			mergedFactories[moduleType] = reflect.ValueOf(factory)
+		}
+	}
+
+	return bpdoc.AllPackages(pkgFiles, mergedFactories, ctx.ModuleTypePropertyStructs())
 }
 
 func writeDocs(ctx *blueprint.Context, filename string) error {
-	moduleTypeList, err := ModuleTypeDocs(ctx)
+	moduleTypeList, err := ModuleTypeDocs(ctx, nil)
 	if err != nil {
 		return err
 	}
@@ -108,25 +120,28 @@ const (
 <h1>Build Docs</h1>
 <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
   {{range .}}
-    {{ $collapseIndex := unique }}
-    <div class="panel panel-default">
-      <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}">
-        <h2 class="panel-title">
-          <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}">
-             {{.Name}}
-          </a>
-        </h2>
+    <p>{{.Text}}</p>
+    {{range .ModuleTypes}}
+      {{ $collapseIndex := unique }}
+      <div class="panel panel-default">
+        <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}">
+          <h2 class="panel-title">
+            <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}">
+               {{.Name}}
+            </a>
+          </h2>
+        </div>
       </div>
-    </div>
-    <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}">
-      <div class="panel-body">
-        <p>{{.Text}}</p>
-        {{range .PropertyStructs}}
+      <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}">
+        <div class="panel-body">
           <p>{{.Text}}</p>
-          {{template "properties" .Properties}}
-        {{end}}
+          {{range .PropertyStructs}}
+            <p>{{.Text}}</p>
+            {{template "properties" .Properties}}
+          {{end}}
+        </div>
       </div>
-    </div>
+    {{end}}
   {{end}}
 </div>
 </body>

context.go

@@ -2829,6 +2829,14 @@ func (c *Context) ModuleTypePropertyStructs() map[string][]interface{} {
 	return ret
 }
 
+func (c *Context) ModuleTypeFactories() map[string]ModuleFactory {
+	ret := make(map[string]ModuleFactory)
+	for k, v := range c.moduleFactories {
+		ret[k] = v
+	}
+	return ret
+}
+
 func (c *Context) ModuleName(logicModule Module) string {
 	module := c.moduleInfo[logicModule]
 	return module.Name()