Merge changes Iba57c949,Ief43ff51,Ib1809a4d,I2ab64f36 am: b08a091502 am: 3f57de54b2 am: a088bff723 am: 6c4cdb0bb2

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1480605

Change-Id: Ie0876cdaf142dbd75c094f9e680849efb4105b28

android/Android.bp

@@ -32,6 +32,7 @@ bootstrap_go_package {
         "mutator.go",
         "namespace.go",
         "neverallow.go",
+        "ninja_deps.go",
         "notices.go",
         "onceper.go",
         "override_module.go",
@@ -74,6 +75,7 @@ bootstrap_go_package {
         "mutator_test.go",
         "namespace_test.go",
         "neverallow_test.go",
+        "ninja_deps_test.go",
         "onceper_test.go",
         "package_test.go",
         "path_properties_test.go",

android/config.go

@@ -126,6 +126,8 @@ type config struct {
 	// in tests when a path doesn't exist.
 	testAllowNonExistentPaths bool
 
+	ninjaFileDepsSet sync.Map
+
 	OncePer
 }
 

android/makevars.go

@@ -139,15 +139,24 @@ type SingletonMakeVarsProvider interface {
 	MakeVars(ctx MakeVarsContext)
 }
 
-// registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to the list of
-// MakeVarsProviders to run.
-func registerSingletonMakeVarsProvider(singleton SingletonMakeVarsProvider) {
-	singletonMakeVarsProviders = append(singletonMakeVarsProviders,
-		makeVarsProvider{pctx, SingletonmakeVarsProviderAdapter(singleton)})
+var singletonMakeVarsProvidersKey = NewOnceKey("singletonMakeVarsProvidersKey")
+
+// registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to
+// the list of MakeVarsProviders to run.
+func registerSingletonMakeVarsProvider(config Config, singleton SingletonMakeVarsProvider) {
+	// Singletons are registered on the Context and may be different between different Contexts,
+	// for example when running multiple tests.  Store the SingletonMakeVarsProviders in the
+	// Config so they are attached to the Context.
+	singletonMakeVarsProviders := config.Once(singletonMakeVarsProvidersKey, func() interface{} {
+		return &[]makeVarsProvider{}
+	}).(*[]makeVarsProvider)
+
+	*singletonMakeVarsProviders = append(*singletonMakeVarsProviders,
+		makeVarsProvider{pctx, singletonMakeVarsProviderAdapter(singleton)})
 }
 
-// SingletonmakeVarsProviderAdapter converts a SingletonMakeVarsProvider to a MakeVarsProvider.
-func SingletonmakeVarsProviderAdapter(singleton SingletonMakeVarsProvider) MakeVarsProvider {
+// singletonMakeVarsProviderAdapter converts a SingletonMakeVarsProvider to a MakeVarsProvider.
+func singletonMakeVarsProviderAdapter(singleton SingletonMakeVarsProvider) MakeVarsProvider {
 	return func(ctx MakeVarsContext) { singleton.MakeVars(ctx) }
 }
 
@@ -175,9 +184,6 @@ type makeVarsProvider struct {
 // Collection of makevars providers that are registered in init() methods.
 var makeVarsInitProviders []makeVarsProvider
 
-// Collection of singleton makevars providers that are not registered as part of init() methods.
-var singletonMakeVarsProviders []makeVarsProvider
-
 type makeVarsContext struct {
 	SingletonContext
 	config  Config
@@ -224,7 +230,11 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) {
 	var vars []makeVarsVariable
 	var dists []dist
 	var phonies []phony
-	for _, provider := range append(makeVarsInitProviders) {
+
+	providers := append([]makeVarsProvider(nil), makeVarsInitProviders...)
+	providers = append(providers, *ctx.Config().Get(singletonMakeVarsProvidersKey).(*[]makeVarsProvider)...)
+
+	for _, provider := range providers {
 		mctx := &makeVarsContext{
 			SingletonContext: ctx,
 			pctx:             provider.pctx,
@@ -237,25 +247,6 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) {
 		dists = append(dists, mctx.dists...)
 	}
 
-	for _, provider := range append(singletonMakeVarsProviders) {
-		mctx := &makeVarsContext{
-			SingletonContext: ctx,
-			pctx:             provider.pctx,
-		}
-
-		provider.call(mctx)
-
-		vars = append(vars, mctx.vars...)
-		phonies = append(phonies, mctx.phonies...)
-		dists = append(dists, mctx.dists...)
-	}
-
-	// Clear singleton makevars providers after use. Since these are in-memory
-	// singletons, this ensures state is reset if the build tree is processed
-	// multiple times.
-	// TODO(cparsons): Clean up makeVarsProviders to be part of the context.
-	singletonMakeVarsProviders = nil
-
 	ctx.VisitAllModules(func(m Module) {
 		if provider, ok := m.(ModuleMakeVarsProvider); ok && m.Enabled() {
 			mctx := &makeVarsContext{

android/ninja_deps.go

@@ -0,0 +1,43 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import "sort"
+
+func (c *config) addNinjaFileDeps(deps ...string) {
+	for _, dep := range deps {
+		c.ninjaFileDepsSet.Store(dep, true)
+	}
+}
+
+func (c *config) ninjaFileDeps() []string {
+	var deps []string
+	c.ninjaFileDepsSet.Range(func(key, value interface{}) bool {
+		deps = append(deps, key.(string))
+		return true
+	})
+	sort.Strings(deps)
+	return deps
+}
+
+func ninjaDepsSingletonFactory() Singleton {
+	return &ninjaDepsSingleton{}
+}
+
+type ninjaDepsSingleton struct{}
+
+func (ninjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) {
+	ctx.AddNinjaFileDeps(ctx.Config().ninjaFileDeps()...)
+}

android/ninja_deps_test.go

@@ -0,0 +1,75 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+	"testing"
+)
+
+func init() {
+	// This variable uses ExistentPathForSource on a PackageVarContext, which is a PathContext
+	// that is not a PathGlobContext.  That requires the deps to be stored in the Config.
+	pctx.VariableFunc("test_ninja_deps_variable", func(ctx PackageVarContext) string {
+		// Using ExistentPathForSource to look for a file that does not exist in a directory that
+		// does exist (test_ninja_deps) from a PackageVarContext adds a dependency from build.ninja
+		// to the directory.
+		if ExistentPathForSource(ctx, "test_ninja_deps/does_not_exist").Valid() {
+			return "true"
+		} else {
+			return "false"
+		}
+	})
+}
+
+func testNinjaDepsSingletonFactory() Singleton {
+	return testNinjaDepsSingleton{}
+}
+
+type testNinjaDepsSingleton struct{}
+
+func (testNinjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) {
+	// Reference the test_ninja_deps_variable in a build statement so Blueprint is forced to
+	// evaluate it.
+	ctx.Build(pctx, BuildParams{
+		Rule:   Cp,
+		Input:  PathForTesting("foo"),
+		Output: PathForOutput(ctx, "test_ninja_deps_out"),
+		Args: map[string]string{
+			"cpFlags": "${test_ninja_deps_variable}",
+		},
+	})
+}
+
+func TestNinjaDeps(t *testing.T) {
+	fs := map[string][]byte{
+		"test_ninja_deps/exists": nil,
+	}
+	config := TestConfig(buildDir, nil, "", fs)
+
+	ctx := NewTestContext(config)
+	ctx.RegisterSingletonType("test_ninja_deps_singleton", testNinjaDepsSingletonFactory)
+	ctx.RegisterSingletonType("ninja_deps_singleton", ninjaDepsSingletonFactory)
+	ctx.Register()
+
+	_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+	FailIfErrored(t, errs)
+	ninjaDeps, errs := ctx.PrepareBuildActions(config)
+	FailIfErrored(t, errs)
+
+	// Verify that the ninja file has a dependency on the test_ninja_deps directory.
+	if g, w := ninjaDeps, "test_ninja_deps"; !InList(w, g) {
+		t.Errorf("expected %q in %q", w, g)
+	}
+}

android/package_ctx.go

@@ -56,7 +56,7 @@ func (e *configErrorWrapper) Errorf(format string, args ...interface{}) {
 	e.errors = append(e.errors, fmt.Errorf(format, args...))
 }
 func (e *configErrorWrapper) AddNinjaFileDeps(deps ...string) {
-	e.pctx.AddNinjaFileDeps(deps...)
+	e.config.addNinjaFileDeps(deps...)
 }
 
 type PackageVarContext interface {

android/register.go

@@ -29,7 +29,7 @@ var moduleTypes []moduleType
 
 type singleton struct {
 	name    string
-	factory blueprint.SingletonFactory
+	factory SingletonFactory
 }
 
 var singletons []singleton
@@ -57,11 +57,11 @@ type SingletonFactory func() Singleton
 
 // SingletonFactoryAdaptor wraps a SingletonFactory into a blueprint.SingletonFactory by converting
 // a Singleton into a blueprint.Singleton
-func SingletonFactoryAdaptor(factory SingletonFactory) blueprint.SingletonFactory {
+func SingletonFactoryAdaptor(ctx *Context, factory SingletonFactory) blueprint.SingletonFactory {
 	return func() blueprint.Singleton {
 		singleton := factory()
 		if makevars, ok := singleton.(SingletonMakeVarsProvider); ok {
-			registerSingletonMakeVarsProvider(makevars)
+			registerSingletonMakeVarsProvider(ctx.config, makevars)
 		}
 		return &singletonAdaptor{Singleton: singleton}
 	}
@@ -72,11 +72,11 @@ func RegisterModuleType(name string, factory ModuleFactory) {
 }
 
 func RegisterSingletonType(name string, factory SingletonFactory) {
-	singletons = append(singletons, singleton{name, SingletonFactoryAdaptor(factory)})
+	singletons = append(singletons, singleton{name, factory})
 }
 
 func RegisterPreSingletonType(name string, factory SingletonFactory) {
-	preSingletons = append(preSingletons, singleton{name, SingletonFactoryAdaptor(factory)})
+	preSingletons = append(preSingletons, singleton{name, factory})
 }
 
 type Context struct {
@@ -92,7 +92,7 @@ func NewContext(config Config) *Context {
 
 func (ctx *Context) Register() {
 	for _, t := range preSingletons {
-		ctx.RegisterPreSingletonType(t.name, t.factory)
+		ctx.RegisterPreSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory))
 	}
 
 	for _, t := range moduleTypes {
@@ -100,21 +100,23 @@ func (ctx *Context) Register() {
 	}
 
 	for _, t := range singletons {
-		ctx.RegisterSingletonType(t.name, t.factory)
+		ctx.RegisterSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory))
 	}
 
 	registerMutators(ctx.Context, preArch, preDeps, postDeps, finalDeps)
 
-	ctx.RegisterSingletonType("bazeldeps", SingletonFactoryAdaptor(BazelSingleton))
+	ctx.RegisterSingletonType("bazeldeps", SingletonFactoryAdaptor(ctx, BazelSingleton))
 
 	// Register phony just before makevars so it can write out its phony rules as Make rules
-	ctx.RegisterSingletonType("phony", SingletonFactoryAdaptor(phonySingletonFactory))
+	ctx.RegisterSingletonType("phony", SingletonFactoryAdaptor(ctx, phonySingletonFactory))
 
 	// Register makevars after other singletons so they can export values through makevars
-	ctx.RegisterSingletonType("makevars", SingletonFactoryAdaptor(makeVarsSingletonFunc))
+	ctx.RegisterSingletonType("makevars", SingletonFactoryAdaptor(ctx, makeVarsSingletonFunc))
 
-	// Register env last so that it can track all used environment variables
-	ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(EnvSingleton))
+	// Register env and ninjadeps last so that they can track all used environment variables and
+	// Ninja file dependencies stored in the config.
+	ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(ctx, EnvSingleton))
+	ctx.RegisterSingletonType("ninjadeps", SingletonFactoryAdaptor(ctx, ninjaDepsSingletonFactory))
 }
 
 func ModuleTypeFactories() map[string]ModuleFactory {

android/testing.go

@@ -104,7 +104,7 @@ func (ctx *TestContext) RegisterModuleType(name string, factory ModuleFactory) {
 }
 
 func (ctx *TestContext) RegisterSingletonType(name string, factory SingletonFactory) {
-	ctx.Context.RegisterSingletonType(name, SingletonFactoryAdaptor(factory))
+	ctx.Context.RegisterSingletonType(name, SingletonFactoryAdaptor(ctx.Context, factory))
 }
 
 func (ctx *TestContext) ModuleForTests(name, variant string) TestingModule {

cc/cc.go

@@ -88,7 +88,7 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) {
 		ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel()
 	})
 
-	android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory)
+	ctx.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory)
 }
 
 type Deps struct {
@@ -369,7 +369,7 @@ type ModuleContextIntf interface {
 	useSdk() bool
 	sdkVersion() string
 	useVndk() bool
-	isNdk() bool
+	isNdk(config android.Config) bool
 	isLlndk(config android.Config) bool
 	isLlndkPublic(config android.Config) bool
 	isVndkPrivate(config android.Config) bool
@@ -939,8 +939,8 @@ func (c *Module) isCoverageVariant() bool {
 	return c.coverage.Properties.IsCoverageVariant
 }
 
-func (c *Module) IsNdk() bool {
-	return inList(c.BaseModuleName(), ndkKnownLibs)
+func (c *Module) IsNdk(config android.Config) bool {
+	return inList(c.BaseModuleName(), *getNDKKnownLibs(config))
 }
 
 func (c *Module) isLlndk(config android.Config) bool {
@@ -1145,8 +1145,8 @@ func (ctx *moduleContextImpl) useVndk() bool {
 	return ctx.mod.UseVndk()
 }
 
-func (ctx *moduleContextImpl) isNdk() bool {
-	return ctx.mod.IsNdk()
+func (ctx *moduleContextImpl) isNdk(config android.Config) bool {
+	return ctx.mod.IsNdk(config)
 }
 
 func (ctx *moduleContextImpl) isLlndk(config android.Config) bool {
@@ -1766,7 +1766,7 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) {
 			for _, entry := range list {
 				// strip #version suffix out
 				name, _ := StubsLibNameAndVersion(entry)
-				if ctx.useSdk() && inList(name, ndkKnownLibs) {
+				if ctx.useSdk() && inList(name, *getNDKKnownLibs(ctx.Config())) {
 					variantLibs = append(variantLibs, name+ndkLibrarySuffix)
 				} else if ctx.useVndk() {
 					nonvariantLibs = append(nonvariantLibs, rewriteVendorLibs(entry))

cc/library.go

@@ -567,7 +567,7 @@ func (library *libraryDecorator) classifySourceAbiDump(ctx ModuleContext) string
 		return ""
 	}
 	// Return NDK if the library is both NDK and LLNDK.
-	if ctx.isNdk() {
+	if ctx.isNdk(ctx.Config()) {
 		return "NDK"
 	}
 	if ctx.isLlndkPublic(ctx.Config()) {
@@ -1099,7 +1099,7 @@ func (library *libraryDecorator) coverageOutputFilePath() android.OptionalPath {
 
 func getRefAbiDumpFile(ctx ModuleContext, vndkVersion, fileName string) android.Path {
 	// The logic must be consistent with classifySourceAbiDump.
-	isNdk := ctx.isNdk()
+	isNdk := ctx.isNdk(ctx.Config())
 	isLlndkOrVndk := ctx.isLlndkPublic(ctx.Config()) || (ctx.useVndk() && ctx.isVndk())
 
 	refAbiDumpTextFile := android.PathForVndkRefAbiDump(ctx, vndkVersion, fileName, isNdk, isLlndkOrVndk, false)
@@ -1153,7 +1153,7 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objec
 			library.sAbiDiff = SourceAbiDiff(ctx, library.sAbiOutputFile.Path(),
 				refAbiDumpFile, fileName, exportedHeaderFlags,
 				Bool(library.Properties.Header_abi_checker.Check_all_apis),
-				ctx.isLlndk(ctx.Config()), ctx.isNdk(), ctx.isVndkExt())
+				ctx.isLlndk(ctx.Config()), ctx.isNdk(ctx.Config()), ctx.isVndkExt())
 		}
 	}
 }

cc/makevars.go

@@ -171,6 +171,7 @@ func makeVarsProvider(ctx android.MakeVarsContext) {
 	ctx.StrictRaw("SRC_HEADERS", strings.Join(includes, " "))
 	ctx.StrictRaw("SRC_SYSTEM_HEADERS", strings.Join(systemIncludes, " "))
 
+	ndkKnownLibs := *getNDKKnownLibs(ctx.Config())
 	sort.Strings(ndkKnownLibs)
 	ctx.Strict("NDK_KNOWN_LIBS", strings.Join(ndkKnownLibs, " "))
 

cc/ndk_library.go

@@ -45,8 +45,7 @@ var (
 
 	ndkLibrarySuffix = ".ndk"
 
-	// Added as a variation dependency via depsMutator.
-	ndkKnownLibs = []string{}
+	ndkKnownLibsKey = android.NewOnceKey("ndkKnownLibsKey")
 	// protects ndkKnownLibs writes during parallel BeginMutator.
 	ndkKnownLibsLock sync.Mutex
 )
@@ -158,6 +157,12 @@ func (this *stubDecorator) initializeProperties(ctx BaseModuleContext) bool {
 	return true
 }
 
+func getNDKKnownLibs(config android.Config) *[]string {
+	return config.Once(ndkKnownLibsKey, func() interface{} {
+		return &[]string{}
+	}).(*[]string)
+}
+
 func (c *stubDecorator) compilerInit(ctx BaseModuleContext) {
 	c.baseCompiler.compilerInit(ctx)
 
@@ -168,12 +173,13 @@ func (c *stubDecorator) compilerInit(ctx BaseModuleContext) {
 
 	ndkKnownLibsLock.Lock()
 	defer ndkKnownLibsLock.Unlock()
-	for _, lib := range ndkKnownLibs {
+	ndkKnownLibs := getNDKKnownLibs(ctx.Config())
+	for _, lib := range *ndkKnownLibs {
 		if lib == name {
 			return
 		}
 	}
-	ndkKnownLibs = append(ndkKnownLibs, name)
+	*ndkKnownLibs = append(*ndkKnownLibs, name)
 }
 
 func addStubLibraryCompilerFlags(flags Flags) Flags {

java/app.go

@@ -890,7 +890,7 @@ func collectAppDeps(ctx android.ModuleContext, app appDepsInterface,
 
 		if IsJniDepTag(tag) || cc.IsSharedDepTag(tag) {
 			if dep, ok := module.(*cc.Module); ok {
-				if dep.IsNdk() || dep.IsStubs() {
+				if dep.IsNdk(ctx.Config()) || dep.IsStubs() {
 					return false
 				}
 

java/java_test.go

@@ -92,8 +92,8 @@ func testContext(config android.Config) *android.TestContext {
 
 	ctx.PreDepsMutators(python.RegisterPythonPreDepsMutators)
 	ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators)
-	ctx.RegisterPreSingletonType("overlay", android.SingletonFactoryAdaptor(OverlaySingletonFactory))
-	ctx.RegisterPreSingletonType("sdk_versions", android.SingletonFactoryAdaptor(sdkPreSingletonFactory))
+	ctx.RegisterPreSingletonType("overlay", android.SingletonFactoryAdaptor(ctx.Context, OverlaySingletonFactory))
+	ctx.RegisterPreSingletonType("sdk_versions", android.SingletonFactoryAdaptor(ctx.Context, sdkPreSingletonFactory))
 
 	android.RegisterPrebuiltMutators(ctx)