Merge "Allow importing starlark code in makefiles"

This commit is contained in:
Cole Faust 2023-05-11 17:04:46 +00:00 committed by Gerrit Code Review
commit 99d89154f1
9 changed files with 310 additions and 74 deletions

23
core/all_versions.bzl Normal file
View file

@ -0,0 +1,23 @@
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
_all_versions = ["OPR1", "OPD1", "OPD2", "OPM1", "OPM2", "PPR1", "PPD1", "PPD2", "PPM1", "PPM2", "QPR1"] + [
version + subversion
for version in ["Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
for subversion in ["P1A", "P1B", "P2A", "P2B", "D1A", "D1B", "D2A", "D2B", "Q1A", "Q1B", "Q2A", "Q2B", "Q3A", "Q3B"]
]
variables_to_export_to_make = {
"ALL_VERSIONS": _all_versions,
}

View file

@ -256,7 +256,7 @@ else
endif
$(shell build/soong/scripts/update_out $(OUT_DIR)/rbc/rbc_board_config_results.mk \
$(OUT_DIR)/rbcrun $(OUT_DIR)/rbc/boardlauncher.rbc)
$(OUT_DIR)/rbcrun --mode=rbc $(OUT_DIR)/rbc/boardlauncher.rbc)
ifneq ($(.SHELLSTATUS),0)
$(error board configuration runner failed: $(.SHELLSTATUS))
endif

View file

@ -24,14 +24,25 @@ endef
#$(warning $(call find_and_earlier,A B C,C))
#$(warning $(call find_and_earlier,A B C,D))
define version-list
$(1)P1A $(1)P1B $(1)P2A $(1)P2B $(1)D1A $(1)D1B $(1)D2A $(1)D2B $(1)Q1A $(1)Q1B $(1)Q2A $(1)Q2B $(1)Q3A $(1)Q3B
# Runs the starlark file given in $(1), and sets all the variables in its top-level
# variables_to_export_to_make variable as make variables.
#
# In order to avoid running starlark every time the stamp file is checked, we use
# $(KATI_shell_no_rerun). Then, to make sure that we actually do rerun kati when
# modifying the starlark files, we add the starlark files to the kati stamp file with
# $(KATI_extra_file_deps).
define run-starlark
$(eval _starlark_results := $(OUT_DIR)/starlark_results/$(subst /,_,$(1)).mk)
$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results))
$(if $(filter-out 0,$(.SHELLSTATUS)),$(error Starlark failed to run))
$(eval include $(_starlark_results))
$(KATI_extra_file_deps $(LOADED_STARLARK_FILES))
$(eval LOADED_STARLARK_FILES :=)
$(eval _starlark_results :=)
endef
PREV_VERSIONS := OPR1 OPD1 OPD2 OPM1 OPM2 PPR1 PPD1 PPD2 PPM1 PPM2 QPR1
ALL_VERSIONS := Q R S T U V W X Y Z
ALL_VERSIONS := $(PREV_VERSIONS) $(foreach v,$(ALL_VERSIONS),$(call version-list,$(v)))
PREV_VERSIONS :=
# defines ALL_VERSIONS
$(call run-starlark,build/make/core/all_versions.bzl)
# Filters ALL_VERSIONS down to the range [$1, $2], and errors if $1 > $2 or $3 is
# not in [$1, $2]

View file

@ -247,7 +247,7 @@ else
endif
$(shell build/soong/scripts/update_out $(OUT_DIR)/rbc/rbc_product_config_results.mk \
$(OUT_DIR)/rbcrun $(OUT_DIR)/rbc/launcher.rbc)
$(OUT_DIR)/rbcrun --mode=rbc $(OUT_DIR)/rbc/launcher.rbc)
ifneq ($(.SHELLSTATUS),0)
$(error product configuration runner failed: $(.SHELLSTATUS))
endif

View file

@ -34,6 +34,7 @@ bootstrap_go_package {
pkgPath: "rbcrun",
deps: [
"go-starlark-starlark",
"go-starlark-starlarkjson",
"go-starlark-starlarkstruct",
"go-starlark-starlarktest",
],

View file

@ -24,13 +24,19 @@ import (
"strings"
"go.starlark.net/starlark"
"go.starlark.net/starlarkjson"
"go.starlark.net/starlarkstruct"
)
const callerDirKey = "callerDir"
type ExecutionMode int
const (
ExecutionModeRbc ExecutionMode = iota
ExecutionModeMake ExecutionMode = iota
)
var LoadPathRoot = "."
var shellPath string
const callerDirKey = "callerDir"
const shellKey = "shell"
const executionModeKey = "executionMode"
type modentry struct {
globals starlark.StringDict
@ -39,20 +45,66 @@ type modentry struct {
var moduleCache = make(map[string]*modentry)
var builtins starlark.StringDict
var rbcBuiltins starlark.StringDict = starlark.StringDict{
"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
// To convert find-copy-subdir and product-copy-files-by pattern
"rblf_find_files": starlark.NewBuiltin("rblf_find_files", find),
// To convert makefile's $(shell cmd)
"rblf_shell": starlark.NewBuiltin("rblf_shell", shell),
// Output to stderr
"rblf_log": starlark.NewBuiltin("rblf_log", log),
// To convert makefile's $(wildcard foo*)
"rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard),
}
func moduleName2AbsPath(moduleName string, callerDir string) (string, error) {
path := moduleName
if ix := strings.LastIndex(path, ":"); ix >= 0 {
path = path[0:ix] + string(os.PathSeparator) + path[ix+1:]
var makeBuiltins starlark.StringDict = starlark.StringDict{
"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
"json": starlarkjson.Module,
}
// Takes a module name (the first argument to the load() function) and returns the path
// it's trying to load, stripping out leading //, and handling leading :s.
func cleanModuleName(moduleName string, callerDir string) (string, error) {
if strings.Count(moduleName, ":") > 1 {
return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName)
}
if strings.HasPrefix(path, "//") {
return filepath.Abs(filepath.Join(LoadPathRoot, path[2:]))
// We don't have full support for external repositories, but at least support skylib's dicts.
if moduleName == "@bazel_skylib//lib:dicts.bzl" {
return "external/bazel-skylib/lib/dicts.bzl", nil
}
localLoad := false
if strings.HasPrefix(moduleName, "@//") {
moduleName = moduleName[3:]
} else if strings.HasPrefix(moduleName, "//") {
moduleName = moduleName[2:]
} else if strings.HasPrefix(moduleName, ":") {
return filepath.Abs(filepath.Join(callerDir, path[1:]))
moduleName = moduleName[1:]
localLoad = true
} else {
return filepath.Abs(path)
return "", fmt.Errorf("load path must start with // or :")
}
if ix := strings.LastIndex(moduleName, ":"); ix >= 0 {
moduleName = moduleName[:ix] + string(os.PathSeparator) + moduleName[ix+1:]
}
if filepath.Clean(moduleName) != moduleName {
return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName))
}
if strings.HasPrefix(moduleName, "../") {
return "", fmt.Errorf("load path must not start with ../: %s", moduleName)
}
if strings.HasPrefix(moduleName, "/") {
return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)
}
if localLoad {
return filepath.Join(callerDir, moduleName), nil
}
return moduleName, nil
}
// loader implements load statement. The format of the loaded module URI is
@ -61,14 +113,18 @@ func moduleName2AbsPath(moduleName string, callerDir string) (string, error) {
// The presence of `|symbol` indicates that the loader should return a single 'symbol'
// bound to None if file is missing.
func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) {
pipePos := strings.LastIndex(module, "|")
mustLoad := pipePos < 0
mode := thread.Local(executionModeKey).(ExecutionMode)
var defaultSymbol string
if !mustLoad {
defaultSymbol = module[pipePos+1:]
module = module[:pipePos]
mustLoad := true
if mode == ExecutionModeRbc {
pipePos := strings.LastIndex(module, "|")
mustLoad = pipePos < 0
if !mustLoad {
defaultSymbol = module[pipePos+1:]
module = module[:pipePos]
}
}
modulePath, err := moduleName2AbsPath(module, thread.Local(callerDirKey).(string))
modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string))
if err != nil {
return nil, err
}
@ -100,8 +156,17 @@ func loader(thread *starlark.Thread, module string) (starlark.StringDict, error)
}
childThread.SetLocal(callerDirKey, filepath.Dir(modulePath))
globals, err := starlark.ExecFile(childThread, modulePath, nil, builtins)
e = &modentry{globals, err}
childThread.SetLocal(shellKey, thread.Local(shellKey))
childThread.SetLocal(executionModeKey, mode)
if mode == ExecutionModeRbc {
globals, err := starlark.ExecFile(childThread, modulePath, nil, rbcBuiltins)
e = &modentry{globals, err}
} else if mode == ExecutionModeMake {
globals, err := starlark.ExecFile(childThread, modulePath, nil, makeBuiltins)
e = &modentry{globals, err}
} else {
return nil, fmt.Errorf("unknown executionMode %d", mode)
}
} else {
e = &modentry{starlark.StringDict{defaultSymbol: starlark.None}, nil}
}
@ -189,12 +254,13 @@ func find(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple,
// its output the same way as Make's $(shell ) function. The end-of-lines
// ("\n" or "\r\n") are replaced with " " in the result, and the trailing
// end-of-line is removed.
func shell(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple,
func shell(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple,
kwargs []starlark.Tuple) (starlark.Value, error) {
var command string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &command); err != nil {
return starlark.None, err
}
shellPath := thread.Local(shellKey).(string)
if shellPath == "" {
return starlark.None,
fmt.Errorf("cannot run shell, /bin/sh is missing (running on Windows?)")
@ -245,45 +311,68 @@ func log(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwa
return starlark.None, nil
}
func setup() {
// Create the symbols that aid makefile conversion. See README.md
builtins = starlark.StringDict{
"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
// To convert find-copy-subdir and product-copy-files-by pattern
"rblf_find_files": starlark.NewBuiltin("rblf_find_files", find),
// To convert makefile's $(shell cmd)
"rblf_shell": starlark.NewBuiltin("rblf_shell", shell),
// Output to stderr
"rblf_log": starlark.NewBuiltin("rblf_log", log),
// To convert makefile's $(wildcard foo*)
"rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard),
}
// NOTE(asmundak): OS-specific. Behave similar to Linux `system` call,
// which always uses /bin/sh to run the command
shellPath = "/bin/sh"
if _, err := os.Stat(shellPath); err != nil {
shellPath = ""
}
}
// Parses, resolves, and executes a Starlark file.
// filename and src parameters are as for starlark.ExecFile:
// * filename is the name of the file to execute,
// and the name that appears in error messages;
// * src is an optional source of bytes to use instead of filename
// (it can be a string, or a byte array, or an io.Reader instance)
func Run(filename string, src interface{}) error {
setup()
// Returns the top-level starlark variables, the list of starlark files loaded, and an error
func Run(filename string, src interface{}, mode ExecutionMode) (starlark.StringDict, []string, error) {
// NOTE(asmundak): OS-specific. Behave similar to Linux `system` call,
// which always uses /bin/sh to run the command
shellPath := "/bin/sh"
if _, err := os.Stat(shellPath); err != nil {
shellPath = ""
}
mainThread := &starlark.Thread{
Name: "main",
Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) },
Print: func(_ *starlark.Thread, msg string) {
if mode == ExecutionModeRbc {
// In rbc mode, rblf_log is used to print to stderr
fmt.Println(msg)
} else if mode == ExecutionModeMake {
fmt.Fprintln(os.Stderr, msg)
}
},
Load: loader,
}
absPath, err := filepath.Abs(filename)
if err == nil {
mainThread.SetLocal(callerDirKey, filepath.Dir(absPath))
_, err = starlark.ExecFile(mainThread, absPath, src, builtins)
filename, err := filepath.Abs(filename)
if err != nil {
return nil, nil, err
}
return err
if wd, err := os.Getwd(); err == nil {
filename, err = filepath.Rel(wd, filename)
if err != nil {
return nil, nil, err
}
if strings.HasPrefix(filename, "../") {
return nil, nil, fmt.Errorf("path could not be made relative to workspace root: %s", filename)
}
} else {
return nil, nil, err
}
// Add top-level file to cache for cycle detection purposes
moduleCache[filename] = nil
var results starlark.StringDict
mainThread.SetLocal(callerDirKey, filepath.Dir(filename))
mainThread.SetLocal(shellKey, shellPath)
mainThread.SetLocal(executionModeKey, mode)
if mode == ExecutionModeRbc {
results, err = starlark.ExecFile(mainThread, filename, src, rbcBuiltins)
} else if mode == ExecutionModeMake {
results, err = starlark.ExecFile(mainThread, filename, src, makeBuiltins)
} else {
return results, nil, fmt.Errorf("unknown executionMode %d", mode)
}
loadedStarlarkFiles := make([]string, 0, len(moduleCache))
for file := range moduleCache {
loadedStarlarkFiles = append(loadedStarlarkFiles, file)
}
sort.Strings(loadedStarlarkFiles)
return results, loadedStarlarkFiles, err
}

View file

@ -54,7 +54,6 @@ func starlarktestSetup() {
// Common setup for the tests: create thread, change to the test directory
func testSetup(t *testing.T) *starlark.Thread {
setup()
thread := &starlark.Thread{
Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
if module == "assert.star" {
@ -78,7 +77,6 @@ func exerciseStarlarkTestFile(t *testing.T, starFile string) {
// In order to use "assert.star" from go/starlark.net/starlarktest in the tests, provide:
// * load function that handles "assert.star"
// * starlarktest.DataFile function that finds its location
setup()
if err := os.Chdir(dataDir()); err != nil {
t.Fatal(err)
}
@ -92,7 +90,9 @@ func exerciseStarlarkTestFile(t *testing.T, starFile string) {
starlarktest.SetReporter(thread, t)
_, thisSrcFile, _, _ := runtime.Caller(0)
filename := filepath.Join(filepath.Dir(thisSrcFile), starFile)
if _, err := starlark.ExecFile(thread, filename, nil, builtins); err != nil {
thread.SetLocal(executionModeKey, ExecutionModeRbc)
thread.SetLocal(shellKey, "/bin/sh")
if _, err := starlark.ExecFile(thread, filename, nil, rbcBuiltins); err != nil {
if err, ok := err.(*starlark.EvalError); ok {
t.Fatal(err.Backtrace())
}
@ -103,7 +103,7 @@ func exerciseStarlarkTestFile(t *testing.T, starFile string) {
func TestFileOps(t *testing.T) {
// TODO(asmundak): convert this to use exerciseStarlarkTestFile
thread := testSetup(t)
if _, err := starlark.ExecFile(thread, "file_ops.star", nil, builtins); err != nil {
if _, err := starlark.ExecFile(thread, "file_ops.star", nil, rbcBuiltins); err != nil {
if err, ok := err.(*starlark.EvalError); ok {
t.Fatal(err.Backtrace())
}
@ -122,9 +122,12 @@ func TestLoad(t *testing.T) {
}
}
dir := dataDir()
if err := os.Chdir(filepath.Dir(dir)); err != nil {
t.Fatal(err)
}
thread.SetLocal(callerDirKey, dir)
LoadPathRoot = filepath.Dir(dir)
if _, err := starlark.ExecFile(thread, "load.star", nil, builtins); err != nil {
thread.SetLocal(executionModeKey, ExecutionModeRbc)
if _, err := starlark.ExecFile(thread, "testdata/load.star", nil, rbcBuiltins); err != nil {
if err, ok := err.(*starlark.EvalError); ok {
t.Fatal(err.Backtrace())
}

View file

@ -17,18 +17,22 @@ package main
import (
"flag"
"fmt"
"go.starlark.net/starlark"
"os"
"rbcrun"
"regexp"
"strings"
"go.starlark.net/starlark"
)
var (
modeFlag = flag.String("mode", "", "the general behavior of rbcrun. Can be \"rbc\" or \"make\". Required.")
rootdir = flag.String("d", ".", "the value of // for load paths")
perfFile = flag.String("perf", "", "save performance data")
identifierRe = regexp.MustCompile("[a-zA-Z_][a-zA-Z0-9_]*")
)
func main() {
flag.Parse()
func getEntrypointStarlarkFile() string {
filename := ""
for _, arg := range flag.Args() {
@ -42,8 +46,108 @@ func main() {
flag.Usage()
os.Exit(1)
}
if stat, err := os.Stat(*rootdir); os.IsNotExist(err) || !stat.IsDir() {
quit("%s is not a directory\n", *rootdir)
return filename
}
func getMode() rbcrun.ExecutionMode {
switch *modeFlag {
case "rbc":
return rbcrun.ExecutionModeRbc
case "make":
return rbcrun.ExecutionModeMake
case "":
quit("-mode flag is required.")
default:
quit("Unknown -mode value %q, expected 1 of \"rbc\", \"make\"", *modeFlag)
}
return rbcrun.ExecutionModeMake
}
var makeStringReplacer = strings.NewReplacer("#", "\\#", "$", "$$")
func cleanStringForMake(s string) (string, error) {
if strings.ContainsAny(s, "\\\n") {
// \\ in make is literally \\, not a single \, so we can't allow them.
// \<newline> in make will produce a space, not a newline.
return "", fmt.Errorf("starlark strings exported to make cannot contain backslashes or newlines")
}
return makeStringReplacer.Replace(s), nil
}
func getValueInMakeFormat(value starlark.Value, allowLists bool) (string, error) {
switch v := value.(type) {
case starlark.String:
if cleanedValue, err := cleanStringForMake(v.GoString()); err == nil {
return cleanedValue, nil
} else {
return "", err
}
case starlark.Int:
return v.String(), nil
case *starlark.List:
if !allowLists {
return "", fmt.Errorf("nested lists are not allowed to be exported from starlark to make, flatten the list in starlark first")
}
result := ""
for i := 0; i < v.Len(); i++ {
value, err := getValueInMakeFormat(v.Index(i), false)
if err != nil {
return "", err
}
if i > 0 {
result += " "
}
result += value
}
return result, nil
default:
return "", fmt.Errorf("only starlark strings, ints, and lists of strings/ints can be exported to make. Please convert all other types in starlark first. Found type: %s", value.Type())
}
}
func printVarsInMakeFormat(globals starlark.StringDict) error {
// We could just directly export top level variables by name instead of going through
// a variables_to_export_to_make dictionary, but that wouldn't allow for exporting a
// runtime-defined number of variables to make. This can be important because dictionaries
// in make are often represented by a unique variable for every key in the dictionary.
variablesValue, ok := globals["variables_to_export_to_make"]
if !ok {
return fmt.Errorf("expected top-level starlark file to have a \"variables_to_export_to_make\" variable")
}
variables, ok := variablesValue.(*starlark.Dict)
if !ok {
return fmt.Errorf("expected variables_to_export_to_make to be a dict, got %s", variablesValue.Type())
}
for _, varTuple := range variables.Items() {
varNameStarlark, ok := varTuple.Index(0).(starlark.String)
if !ok {
return fmt.Errorf("all keys in variables_to_export_to_make must be strings, but got %q", varTuple.Index(0).Type())
}
varName := varNameStarlark.GoString()
if !identifierRe.MatchString(varName) {
return fmt.Errorf("all variables at the top level starlark file must be valid c identifiers, but got %q", varName)
}
if varName == "LOADED_STARLARK_FILES" {
return fmt.Errorf("the name LOADED_STARLARK_FILES is reserved for use by the starlark interpreter")
}
valueMake, err := getValueInMakeFormat(varTuple.Index(1), true)
if err != nil {
return err
}
// The :=$= is special Kati syntax that means "set and make readonly"
fmt.Printf("%s :=$= %s\n", varName, valueMake)
}
return nil
}
func main() {
flag.Parse()
filename := getEntrypointStarlarkFile()
mode := getMode()
if os.Chdir(*rootdir) != nil {
quit("could not chdir to %s\n", *rootdir)
}
if *perfFile != "" {
pprof, err := os.Create(*perfFile)
@ -55,8 +159,7 @@ func main() {
quit("%s\n", err)
}
}
rbcrun.LoadPathRoot = *rootdir
err := rbcrun.Run(filename, nil)
variables, loadedStarlarkFiles, err := rbcrun.Run(filename, nil, mode)
rc := 0
if *perfFile != "" {
if err2 := starlark.StopProfile(); err2 != nil {
@ -71,6 +174,12 @@ func main() {
quit("%s\n", err)
}
}
if mode == rbcrun.ExecutionModeMake {
if err := printVarsInMakeFormat(variables); err != nil {
quit("%s\n", err)
}
fmt.Printf("LOADED_STARLARK_FILES := %s\n", strings.Join(loadedStarlarkFiles, " "))
}
os.Exit(rc)
}

View file

@ -2,6 +2,6 @@
load("assert.star", "assert")
# Make sure that builtins are defined for the loaded module, too
assert.true(rblf_wildcard("module1.star"))
assert.true(not rblf_wildcard("no_such file"))
assert.true(rblf_wildcard("testdata/module1.star"))
assert.true(not rblf_wildcard("testdata/no_such file"))
test = "module1"