Merge "Load starlark files from soong"

android/Android.bp

@@ -17,6 +17,7 @@ bootstrap_go_package {
         "soong-remoteexec",
         "soong-response",
         "soong-shared",
+        "soong-starlark",
         "soong-starlark-format",
         "soong-ui-metrics_proto",
         "soong-android-allowlists",

android/ninja_deps.go

@@ -14,7 +14,10 @@
 
 package android
 
-import "sort"
+import (
+	"android/soong/starlark_import"
+	"sort"
+)
 
 func (c *config) addNinjaFileDeps(deps ...string) {
 	for _, dep := range deps {
@@ -40,4 +43,11 @@ type ninjaDepsSingleton struct{}
 
 func (ninjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) {
 	ctx.AddNinjaFileDeps(ctx.Config().ninjaFileDeps()...)
+
+	deps, err := starlark_import.GetNinjaDeps()
+	if err != nil {
+		ctx.Errorf("Error running starlark code: %s", err)
+	} else {
+		ctx.AddNinjaFileDeps(deps...)
+	}
 }

bp2build/bp2build.go

@@ -15,6 +15,7 @@
 package bp2build
 
 import (
+	"android/soong/starlark_import"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -93,6 +94,12 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics {
 		os.Exit(1)
 	}
 	writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles)
+	starlarkDeps, err := starlark_import.GetNinjaDeps()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+	ctx.AddNinjaFileDeps(starlarkDeps...)
 	return &res.metrics
 }
 

cmd/soong_build/queryview.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"android/soong/starlark_import"
 	"io/fs"
 	"io/ioutil"
 	"os"
@@ -47,6 +48,14 @@ func createBazelWorkspace(ctx *bp2build.CodegenContext, outDir string, generateF
 		}
 	}
 
+	// Add starlark deps here, so that they apply to both queryview and apibp2build which
+	// both run this function.
+	starlarkDeps, err2 := starlark_import.GetNinjaDeps()
+	if err2 != nil {
+		return err2
+	}
+	ctx.AddNinjaFileDeps(starlarkDeps...)
+
 	return nil
 }
 

go.mod

@@ -6,4 +6,5 @@ require (
 	github.com/google/blueprint v0.0.0
 	google.golang.org/protobuf v0.0.0
 	prebuilts/bazel/common/proto/analysis_v2 v0.0.0
+	go.starlark.net v0.0.0
 )

go.work

@@ -4,6 +4,7 @@ use (
 	.
 	../../external/go-cmp
 	../../external/golang-protobuf
+	../../external/starlark-go
 	../../prebuilts/bazel/common/proto/analysis_v2
 	../../prebuilts/bazel/common/proto/build
 	../blueprint
@@ -16,4 +17,5 @@ replace (
 	google.golang.org/protobuf v0.0.0 => ../../external/golang-protobuf
 	prebuilts/bazel/common/proto/analysis_v2 v0.0.0 => ../../prebuilts/bazel/common/proto/analysis_v2
 	prebuilts/bazel/common/proto/build v0.0.0 => ../../prebuilts/bazel/common/proto/build
+	go.starlark.net v0.0.0 => ../../external/starlark-go
 )

starlark_import/Android.bp

@@ -0,0 +1,36 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+bootstrap_go_package {
+    name: "soong-starlark",
+    pkgPath: "android/soong/starlark_import",
+    srcs: [
+        "starlark_import.go",
+        "unmarshal.go",
+    ],
+    testSrcs: [
+        "starlark_import_test.go",
+        "unmarshal_test.go",
+    ],
+    deps: [
+        "go-starlark-starlark",
+        "go-starlark-starlarkstruct",
+        "go-starlark-starlarkjson",
+        "go-starlark-starlarktest",
+    ],
+}

starlark_import/README.md

@@ -0,0 +1,14 @@
+# starlark_import package
+
+This allows soong to read constant information from starlark files. At package initialization
+time, soong will read `build/bazel/constants_exported_to_soong.bzl`, and then make the
+variables from that file available via `starlark_import.GetStarlarkValue()`. So to import
+a new variable, it must be added to `constants_exported_to_soong.bzl` and then it can
+be accessed by name.
+
+Only constant information can be read, since this is not a full bazel execution but a
+standalone starlark interpreter. This means you can't use bazel contructs like `rule`,
+`provider`, `select`, `glob`, etc.
+
+All starlark files that were loaded must be added as ninja deps that cause soong to rerun.
+The loaded files can be retrieved via `starlark_import.GetNinjaDeps()`.

starlark_import/starlark_import.go

@@ -0,0 +1,306 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package starlark_import
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarkjson"
+	"go.starlark.net/starlarkstruct"
+)
+
+func init() {
+	go func() {
+		startTime := time.Now()
+		v, d, err := runStarlarkFile("//build/bazel/constants_exported_to_soong.bzl")
+		endTime := time.Now()
+		//fmt.Fprintf(os.Stderr, "starlark run time: %s\n", endTime.Sub(startTime).String())
+		globalResult.Set(starlarkResult{
+			values:    v,
+			ninjaDeps: d,
+			err:       err,
+			startTime: startTime,
+			endTime:   endTime,
+		})
+	}()
+}
+
+type starlarkResult struct {
+	values    starlark.StringDict
+	ninjaDeps []string
+	err       error
+	startTime time.Time
+	endTime   time.Time
+}
+
+// setOnce wraps a value and exposes Set() and Get() accessors for it.
+// The Get() calls will block until a Set() has been called.
+// A second call to Set() will panic.
+// setOnce must be created using newSetOnce()
+type setOnce[T any] struct {
+	value T
+	lock  sync.Mutex
+	wg    sync.WaitGroup
+	isSet bool
+}
+
+func (o *setOnce[T]) Set(value T) {
+	o.lock.Lock()
+	defer o.lock.Unlock()
+	if o.isSet {
+		panic("Value already set")
+	}
+
+	o.value = value
+	o.isSet = true
+	o.wg.Done()
+}
+
+func (o *setOnce[T]) Get() T {
+	if !o.isSet {
+		o.wg.Wait()
+	}
+	return o.value
+}
+
+func newSetOnce[T any]() *setOnce[T] {
+	result := &setOnce[T]{}
+	result.wg.Add(1)
+	return result
+}
+
+var globalResult = newSetOnce[starlarkResult]()
+
+func GetStarlarkValue[T any](key string) (T, error) {
+	result := globalResult.Get()
+	if result.err != nil {
+		var zero T
+		return zero, result.err
+	}
+	if !result.values.Has(key) {
+		var zero T
+		return zero, fmt.Errorf("a starlark variable by that name wasn't found, did you update //build/bazel/constants_exported_to_soong.bzl?")
+	}
+	return Unmarshal[T](result.values[key])
+}
+
+func GetNinjaDeps() ([]string, error) {
+	result := globalResult.Get()
+	if result.err != nil {
+		return nil, result.err
+	}
+	return result.ninjaDeps, nil
+}
+
+func getTopDir() (string, error) {
+	// It's hard to communicate the top dir to this package in any other way than reading the
+	// arguments directly, because we need to know this at package initialization time. Many
+	// soong constants that we'd like to read from starlark are initialized during package
+	// initialization.
+	for i, arg := range os.Args {
+		if arg == "--top" {
+			if i < len(os.Args)-1 && os.Args[i+1] != "" {
+				return os.Args[i+1], nil
+			}
+		}
+	}
+
+	// When running tests, --top is not passed. Instead, search for the top dir manually
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	for cwd != "/" {
+		if _, err := os.Stat(filepath.Join(cwd, "build/soong/soong_ui.bash")); err == nil {
+			return cwd, nil
+		}
+		cwd = filepath.Dir(cwd)
+	}
+	return "", fmt.Errorf("could not find top dir")
+}
+
+const callerDirKey = "callerDir"
+
+type modentry struct {
+	globals starlark.StringDict
+	err     error
+}
+
+func unsupportedMethod(t *starlark.Thread, fn *starlark.Builtin, _ starlark.Tuple, _ []starlark.Tuple) (starlark.Value, error) {
+	return nil, fmt.Errorf("%sthis file is read by soong, and must therefore be pure starlark and include only constant information. %q is not allowed", t.CallStack().String(), fn.Name())
+}
+
+var builtins = starlark.StringDict{
+	"aspect":     starlark.NewBuiltin("aspect", unsupportedMethod),
+	"glob":       starlark.NewBuiltin("glob", unsupportedMethod),
+	"json":       starlarkjson.Module,
+	"provider":   starlark.NewBuiltin("provider", unsupportedMethod),
+	"rule":       starlark.NewBuiltin("rule", unsupportedMethod),
+	"struct":     starlark.NewBuiltin("struct", starlarkstruct.Make),
+	"select":     starlark.NewBuiltin("select", unsupportedMethod),
+	"transition": starlark.NewBuiltin("transition", unsupportedMethod),
+}
+
+// Takes a module name (the first argument to the load() function) and returns the path
+// it's trying to load, stripping out leading //, and handling leading :s.
+func cleanModuleName(moduleName string, callerDir string) (string, error) {
+	if strings.Count(moduleName, ":") > 1 {
+		return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName)
+	}
+
+	// We don't have full support for external repositories, but at least support skylib's dicts.
+	if moduleName == "@bazel_skylib//lib:dicts.bzl" {
+		return "external/bazel-skylib/lib/dicts.bzl", nil
+	}
+
+	localLoad := false
+	if strings.HasPrefix(moduleName, "@//") {
+		moduleName = moduleName[3:]
+	} else if strings.HasPrefix(moduleName, "//") {
+		moduleName = moduleName[2:]
+	} else if strings.HasPrefix(moduleName, ":") {
+		moduleName = moduleName[1:]
+		localLoad = true
+	} else {
+		return "", fmt.Errorf("load path must start with // or :")
+	}
+
+	if ix := strings.LastIndex(moduleName, ":"); ix >= 0 {
+		moduleName = moduleName[:ix] + string(os.PathSeparator) + moduleName[ix+1:]
+	}
+
+	if filepath.Clean(moduleName) != moduleName {
+		return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName))
+	}
+	if strings.HasPrefix(moduleName, "../") {
+		return "", fmt.Errorf("load path must not start with ../: %s", moduleName)
+	}
+	if strings.HasPrefix(moduleName, "/") {
+		return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)
+	}
+
+	if localLoad {
+		return filepath.Join(callerDir, moduleName), nil
+	}
+
+	return moduleName, nil
+}
+
+// loader implements load statement. The format of the loaded module URI is
+//
+//	[//path]:base
+//
+// The file path is $ROOT/path/base if path is present, <caller_dir>/base otherwise.
+func loader(thread *starlark.Thread, module string, topDir string, moduleCache map[string]*modentry, moduleCacheLock *sync.Mutex, filesystem map[string]string) (starlark.StringDict, error) {
+	modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string))
+	if err != nil {
+		return nil, err
+	}
+	moduleCacheLock.Lock()
+	e, ok := moduleCache[modulePath]
+	if e == nil {
+		if ok {
+			moduleCacheLock.Unlock()
+			return nil, fmt.Errorf("cycle in load graph")
+		}
+
+		// Add a placeholder to indicate "load in progress".
+		moduleCache[modulePath] = nil
+		moduleCacheLock.Unlock()
+
+		childThread := &starlark.Thread{Name: "exec " + module, Load: thread.Load}
+
+		// Cheating for the sake of testing:
+		// propagate starlarktest's Reporter key, otherwise testing
+		// the load function may cause panic in starlarktest code.
+		const testReporterKey = "Reporter"
+		if v := thread.Local(testReporterKey); v != nil {
+			childThread.SetLocal(testReporterKey, v)
+		}
+
+		childThread.SetLocal(callerDirKey, filepath.Dir(modulePath))
+
+		if filesystem != nil {
+			globals, err := starlark.ExecFile(childThread, filepath.Join(topDir, modulePath), filesystem[modulePath], builtins)
+			e = &modentry{globals, err}
+		} else {
+			globals, err := starlark.ExecFile(childThread, filepath.Join(topDir, modulePath), nil, builtins)
+			e = &modentry{globals, err}
+		}
+
+		// Update the cache.
+		moduleCacheLock.Lock()
+		moduleCache[modulePath] = e
+	}
+	moduleCacheLock.Unlock()
+	return e.globals, e.err
+}
+
+// Run runs the given starlark file and returns its global variables and a list of all starlark
+// files that were loaded. The top dir for starlark's // is found via getTopDir().
+func runStarlarkFile(filename string) (starlark.StringDict, []string, error) {
+	topDir, err := getTopDir()
+	if err != nil {
+		return nil, nil, err
+	}
+	return runStarlarkFileWithFilesystem(filename, topDir, nil)
+}
+
+func runStarlarkFileWithFilesystem(filename string, topDir string, filesystem map[string]string) (starlark.StringDict, []string, error) {
+	if !strings.HasPrefix(filename, "//") && !strings.HasPrefix(filename, ":") {
+		filename = "//" + filename
+	}
+	filename, err := cleanModuleName(filename, "")
+	if err != nil {
+		return nil, nil, err
+	}
+	moduleCache := make(map[string]*modentry)
+	moduleCache[filename] = nil
+	moduleCacheLock := &sync.Mutex{}
+	mainThread := &starlark.Thread{
+		Name: "main",
+		Print: func(_ *starlark.Thread, msg string) {
+			// Ignore prints
+		},
+		Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
+			return loader(thread, module, topDir, moduleCache, moduleCacheLock, filesystem)
+		},
+	}
+	mainThread.SetLocal(callerDirKey, filepath.Dir(filename))
+
+	var result starlark.StringDict
+	if filesystem != nil {
+		result, err = starlark.ExecFile(mainThread, filepath.Join(topDir, filename), filesystem[filename], builtins)
+	} else {
+		result, err = starlark.ExecFile(mainThread, filepath.Join(topDir, filename), nil, builtins)
+	}
+	return result, sortedStringKeys(moduleCache), err
+}
+
+func sortedStringKeys(m map[string]*modentry) []string {
+	s := make([]string, 0, len(m))
+	for k := range m {
+		s = append(s, k)
+	}
+	sort.Strings(s)
+	return s
+}

starlark_import/starlark_import_test.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package starlark_import
+
+import (
+	"strings"
+	"testing"
+
+	"go.starlark.net/starlark"
+)
+
+func TestBasic(t *testing.T) {
+	globals, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
+		"a.bzl": `
+my_string = "hello, world!"
+`})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if globals["my_string"].(starlark.String) != "hello, world!" {
+		t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String())
+	}
+}
+
+func TestLoad(t *testing.T) {
+	globals, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
+		"a.bzl": `
+load("//b.bzl", _b_string = "my_string")
+my_string = "hello, " + _b_string
+`,
+		"b.bzl": `
+my_string = "world!"
+`})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if globals["my_string"].(starlark.String) != "hello, world!" {
+		t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String())
+	}
+}
+
+func TestLoadRelative(t *testing.T) {
+	globals, ninjaDeps, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
+		"a.bzl": `
+load(":b.bzl", _b_string = "my_string")
+load("//foo/c.bzl", _c_string = "my_string")
+my_string = "hello, " + _b_string
+c_string = _c_string
+`,
+		"b.bzl": `
+my_string = "world!"
+`,
+		"foo/c.bzl": `
+load(":d.bzl", _d_string = "my_string")
+my_string = "hello, " + _d_string
+`,
+		"foo/d.bzl": `
+my_string = "world!"
+`})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if globals["my_string"].(starlark.String) != "hello, world!" {
+		t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String())
+	}
+
+	expectedNinjaDeps := []string{
+		"a.bzl",
+		"b.bzl",
+		"foo/c.bzl",
+		"foo/d.bzl",
+	}
+	if !slicesEqual(ninjaDeps, expectedNinjaDeps) {
+		t.Errorf("Expected %v ninja deps, got %v", expectedNinjaDeps, ninjaDeps)
+	}
+}
+
+func TestLoadCycle(t *testing.T) {
+	_, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{
+		"a.bzl": `
+load(":b.bzl", _b_string = "my_string")
+my_string = "hello, " + _b_string
+`,
+		"b.bzl": `
+load(":a.bzl", _a_string = "my_string")
+my_string = "hello, " + _a_string
+`})
+	if err == nil || !strings.Contains(err.Error(), "cycle in load graph") {
+		t.Errorf("Expected cycle in load graph, got: %v", err)
+		return
+	}
+}
+
+func slicesEqual[T comparable](a []T, b []T) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}

starlark_import/unmarshal.go

@@ -0,0 +1,288 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package starlark_import
+
+import (
+	"fmt"
+	"math"
+	"reflect"
+	"unsafe"
+
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarkstruct"
+)
+
+func Unmarshal[T any](value starlark.Value) (T, error) {
+	var zero T
+	x, err := UnmarshalReflect(value, reflect.TypeOf(zero))
+	return x.Interface().(T), err
+}
+
+func UnmarshalReflect(value starlark.Value, ty reflect.Type) (reflect.Value, error) {
+	zero := reflect.Zero(ty)
+	var result reflect.Value
+	if ty.Kind() == reflect.Interface {
+		var err error
+		ty, err = typeOfStarlarkValue(value)
+		if err != nil {
+			return zero, err
+		}
+	}
+	if ty.Kind() == reflect.Map {
+		result = reflect.MakeMap(ty)
+	} else {
+		result = reflect.Indirect(reflect.New(ty))
+	}
+
+	switch v := value.(type) {
+	case starlark.String:
+		if result.Type().Kind() != reflect.String {
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+		result.SetString(v.GoString())
+	case starlark.Int:
+		signedValue, signedOk := v.Int64()
+		unsignedValue, unsignedOk := v.Uint64()
+		switch result.Type().Kind() {
+		case reflect.Int64:
+			if !signedOk {
+				return zero, fmt.Errorf("starlark int didn't fit in go int64")
+			}
+			result.SetInt(signedValue)
+		case reflect.Int32:
+			if !signedOk || signedValue > math.MaxInt32 || signedValue < math.MinInt32 {
+				return zero, fmt.Errorf("starlark int didn't fit in go int32")
+			}
+			result.SetInt(signedValue)
+		case reflect.Int16:
+			if !signedOk || signedValue > math.MaxInt16 || signedValue < math.MinInt16 {
+				return zero, fmt.Errorf("starlark int didn't fit in go int16")
+			}
+			result.SetInt(signedValue)
+		case reflect.Int8:
+			if !signedOk || signedValue > math.MaxInt8 || signedValue < math.MinInt8 {
+				return zero, fmt.Errorf("starlark int didn't fit in go int8")
+			}
+			result.SetInt(signedValue)
+		case reflect.Int:
+			if !signedOk || signedValue > math.MaxInt || signedValue < math.MinInt {
+				return zero, fmt.Errorf("starlark int didn't fit in go int")
+			}
+			result.SetInt(signedValue)
+		case reflect.Uint64:
+			if !unsignedOk {
+				return zero, fmt.Errorf("starlark int didn't fit in go uint64")
+			}
+			result.SetUint(unsignedValue)
+		case reflect.Uint32:
+			if !unsignedOk || unsignedValue > math.MaxUint32 {
+				return zero, fmt.Errorf("starlark int didn't fit in go uint32")
+			}
+			result.SetUint(unsignedValue)
+		case reflect.Uint16:
+			if !unsignedOk || unsignedValue > math.MaxUint16 {
+				return zero, fmt.Errorf("starlark int didn't fit in go uint16")
+			}
+			result.SetUint(unsignedValue)
+		case reflect.Uint8:
+			if !unsignedOk || unsignedValue > math.MaxUint8 {
+				return zero, fmt.Errorf("starlark int didn't fit in go uint8")
+			}
+			result.SetUint(unsignedValue)
+		case reflect.Uint:
+			if !unsignedOk || unsignedValue > math.MaxUint {
+				return zero, fmt.Errorf("starlark int didn't fit in go uint")
+			}
+			result.SetUint(unsignedValue)
+		default:
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+	case starlark.Float:
+		f := float64(v)
+		switch result.Type().Kind() {
+		case reflect.Float64:
+			result.SetFloat(f)
+		case reflect.Float32:
+			if f > math.MaxFloat32 || f < -math.MaxFloat32 {
+				return zero, fmt.Errorf("starlark float didn't fit in go float32")
+			}
+			result.SetFloat(f)
+		default:
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+	case starlark.Bool:
+		if result.Type().Kind() != reflect.Bool {
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+		result.SetBool(bool(v))
+	case starlark.Tuple:
+		if result.Type().Kind() != reflect.Slice {
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+		elemType := result.Type().Elem()
+		// TODO: Add this grow call when we're on go 1.20
+		//result.Grow(v.Len())
+		for i := 0; i < v.Len(); i++ {
+			elem, err := UnmarshalReflect(v.Index(i), elemType)
+			if err != nil {
+				return zero, err
+			}
+			result = reflect.Append(result, elem)
+		}
+	case *starlark.List:
+		if result.Type().Kind() != reflect.Slice {
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+		elemType := result.Type().Elem()
+		// TODO: Add this grow call when we're on go 1.20
+		//result.Grow(v.Len())
+		for i := 0; i < v.Len(); i++ {
+			elem, err := UnmarshalReflect(v.Index(i), elemType)
+			if err != nil {
+				return zero, err
+			}
+			result = reflect.Append(result, elem)
+		}
+	case *starlark.Dict:
+		if result.Type().Kind() != reflect.Map {
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+		keyType := result.Type().Key()
+		valueType := result.Type().Elem()
+		for _, pair := range v.Items() {
+			key := pair.Index(0)
+			value := pair.Index(1)
+
+			unmarshalledKey, err := UnmarshalReflect(key, keyType)
+			if err != nil {
+				return zero, err
+			}
+			unmarshalledValue, err := UnmarshalReflect(value, valueType)
+			if err != nil {
+				return zero, err
+			}
+
+			result.SetMapIndex(unmarshalledKey, unmarshalledValue)
+		}
+	case *starlarkstruct.Struct:
+		if result.Type().Kind() != reflect.Struct {
+			return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String())
+		}
+		if result.NumField() != len(v.AttrNames()) {
+			return zero, fmt.Errorf("starlark struct and go struct have different number of fields (%d and %d)", len(v.AttrNames()), result.NumField())
+		}
+		for _, attrName := range v.AttrNames() {
+			attr, err := v.Attr(attrName)
+			if err != nil {
+				return zero, err
+			}
+
+			// TODO(b/279787235): this should probably support tags to rename the field
+			resultField := result.FieldByName(attrName)
+			if resultField == (reflect.Value{}) {
+				return zero, fmt.Errorf("starlark struct had field %s, but requested struct type did not", attrName)
+			}
+			// This hack allows us to change unexported fields
+			resultField = reflect.NewAt(resultField.Type(), unsafe.Pointer(resultField.UnsafeAddr())).Elem()
+			x, err := UnmarshalReflect(attr, resultField.Type())
+			if err != nil {
+				return zero, err
+			}
+			resultField.Set(x)
+		}
+	default:
+		return zero, fmt.Errorf("unimplemented starlark type: %s", value.Type())
+	}
+
+	return result, nil
+}
+
+func typeOfStarlarkValue(value starlark.Value) (reflect.Type, error) {
+	var err error
+	switch v := value.(type) {
+	case starlark.String:
+		return reflect.TypeOf(""), nil
+	case *starlark.List:
+		innerType := reflect.TypeOf("")
+		if v.Len() > 0 {
+			innerType, err = typeOfStarlarkValue(v.Index(0))
+			if err != nil {
+				return nil, err
+			}
+		}
+		for i := 1; i < v.Len(); i++ {
+			innerTypeI, err := typeOfStarlarkValue(v.Index(i))
+			if err != nil {
+				return nil, err
+			}
+			if innerType != innerTypeI {
+				return nil, fmt.Errorf("List must contain elements of entirely the same type, found %v and %v", innerType, innerTypeI)
+			}
+		}
+		return reflect.SliceOf(innerType), nil
+	case *starlark.Dict:
+		keyType := reflect.TypeOf("")
+		valueType := reflect.TypeOf("")
+		keys := v.Keys()
+		if v.Len() > 0 {
+			firstKey := keys[0]
+			keyType, err = typeOfStarlarkValue(firstKey)
+			if err != nil {
+				return nil, err
+			}
+			firstValue, found, err := v.Get(firstKey)
+			if !found {
+				err = fmt.Errorf("value not found")
+			}
+			if err != nil {
+				return nil, err
+			}
+			valueType, err = typeOfStarlarkValue(firstValue)
+			if err != nil {
+				return nil, err
+			}
+		}
+		for _, key := range keys {
+			keyTypeI, err := typeOfStarlarkValue(key)
+			if err != nil {
+				return nil, err
+			}
+			if keyType != keyTypeI {
+				return nil, fmt.Errorf("dict must contain elements of entirely the same type, found %v and %v", keyType, keyTypeI)
+			}
+			value, found, err := v.Get(key)
+			if !found {
+				err = fmt.Errorf("value not found")
+			}
+			if err != nil {
+				return nil, err
+			}
+			valueTypeI, err := typeOfStarlarkValue(value)
+			if valueType.Kind() != reflect.Interface && valueTypeI != valueType {
+				// If we see conflicting value types, change the result value type to an empty interface
+				valueType = reflect.TypeOf([]interface{}{}).Elem()
+			}
+		}
+		return reflect.MapOf(keyType, valueType), nil
+	case starlark.Int:
+		return reflect.TypeOf(0), nil
+	case starlark.Float:
+		return reflect.TypeOf(0.0), nil
+	case starlark.Bool:
+		return reflect.TypeOf(true), nil
+	default:
+		return nil, fmt.Errorf("unimplemented starlark type: %s", value.Type())
+	}
+}

starlark_import/unmarshal_test.go

@@ -0,0 +1,133 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package starlark_import
+
+import (
+	"reflect"
+	"testing"
+
+	"go.starlark.net/starlark"
+)
+
+func createStarlarkValue(t *testing.T, code string) starlark.Value {
+	t.Helper()
+	result, err := starlark.ExecFile(&starlark.Thread{}, "main.bzl", "x = "+code, builtins)
+	if err != nil {
+		panic(err)
+	}
+	return result["x"]
+}
+
+func TestUnmarshallConcreteType(t *testing.T) {
+	x, err := Unmarshal[string](createStarlarkValue(t, `"foo"`))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if x != "foo" {
+		t.Errorf(`Expected "foo", got %q`, x)
+	}
+}
+
+func TestUnmarshallConcreteTypeWithInterfaces(t *testing.T) {
+	x, err := Unmarshal[map[string]map[string]interface{}](createStarlarkValue(t,
+		`{"foo": {"foo2": "foo3"}, "bar": {"bar2": ["bar3"]}}`))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	expected := map[string]map[string]interface{}{
+		"foo": {"foo2": "foo3"},
+		"bar": {"bar2": []string{"bar3"}},
+	}
+	if !reflect.DeepEqual(x, expected) {
+		t.Errorf(`Expected %v, got %v`, expected, x)
+	}
+}
+
+func TestUnmarshall(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected interface{}
+	}{
+		{
+			input:    `"foo"`,
+			expected: "foo",
+		},
+		{
+			input:    `5`,
+			expected: 5,
+		},
+		{
+			input:    `["foo", "bar"]`,
+			expected: []string{"foo", "bar"},
+		},
+		{
+			input:    `("foo", "bar")`,
+			expected: []string{"foo", "bar"},
+		},
+		{
+			input:    `("foo",5)`,
+			expected: []interface{}{"foo", 5},
+		},
+		{
+			input:    `{"foo": 5, "bar": 10}`,
+			expected: map[string]int{"foo": 5, "bar": 10},
+		},
+		{
+			input:    `{"foo": ["qux"], "bar": []}`,
+			expected: map[string][]string{"foo": {"qux"}, "bar": nil},
+		},
+		{
+			input: `struct(Foo="foo", Bar=5)`,
+			expected: struct {
+				Foo string
+				Bar int
+			}{Foo: "foo", Bar: 5},
+		},
+		{
+			// Unexported fields version of the above
+			input: `struct(foo="foo", bar=5)`,
+			expected: struct {
+				foo string
+				bar int
+			}{foo: "foo", bar: 5},
+		},
+		{
+			input: `{"foo": "foo2", "bar": ["bar2"], "baz": 5, "qux": {"qux2": "qux3"}, "quux": {"quux2": "quux3", "quux4": 5}}`,
+			expected: map[string]interface{}{
+				"foo": "foo2",
+				"bar": []string{"bar2"},
+				"baz": 5,
+				"qux": map[string]string{"qux2": "qux3"},
+				"quux": map[string]interface{}{
+					"quux2": "quux3",
+					"quux4": 5,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		x, err := UnmarshalReflect(createStarlarkValue(t, tc.input), reflect.TypeOf(tc.expected))
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if !reflect.DeepEqual(x.Interface(), tc.expected) {
+			t.Errorf(`Expected %#v, got %#v`, tc.expected, x.Interface())
+		}
+	}
+}

tests/bp2build_bazel_test.sh

@@ -53,6 +53,20 @@ EOF
   if [[ "$buildfile_mtime1" != "$buildfile_mtime2" ]]; then
     fail "BUILD.bazel was updated even though contents are same"
   fi
+
+  # Force bp2build to rerun by updating the timestamp of the constants_exported_to_soong.bzl file.
+  touch build/bazel/constants_exported_to_soong.bzl
+
+  run_soong bp2build
+  local -r buildfile_mtime3=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)
+  local -r marker_mtime3=$(stat -c "%y" out/soong/bp2build_workspace_marker)
+
+  if [[ "$marker_mtime2" == "$marker_mtime3" ]]; then
+    fail "Expected bp2build marker file to change"
+  fi
+  if [[ "$buildfile_mtime2" != "$buildfile_mtime3" ]]; then
+    fail "BUILD.bazel was updated even though contents are same"
+  fi
 }
 
 # Tests that blueprint files that are deleted are not present when the

tests/persistent_bazel_test.sh

@@ -73,8 +73,8 @@ function test_bazel_failure {
 
   USE_PERSISTENT_BAZEL=1 run_soong nothing 1>out/failurelog.txt 2>&1 && fail "Expected build failure" || true
 
-  if ! grep -sq "'build/bazel/rules' is not a package" out/failurelog.txt ; then
-    fail "Expected error to contain 'build/bazel/rules' is not a package, instead got:\n$(cat out/failurelog.txt)"
+  if ! grep -sq "cannot load //build/bazel/rules/common/api_constants.bzl" out/failurelog.txt ; then
+    fail "Expected error to contain 'cannot load //build/bazel/rules/common/api_constants.bzl', instead got:\n$(cat out/failurelog.txt)"
   fi
 
   kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true