Pass input variables to product config

Passing variables via a makefile instead of
rblf_cli / rblf_env allows us to give them correct
types while converting the makefile to starlark,
as opposed to the variables always being strings
when given via rblf_cli / rblf_env.

This also allows us to remove some hand-converted
starlark code.

Bug: 201700692
Test: ./out/soong/rbcrun ./build/make/tests/run.rbc
Change-Id: I58c4f20b29171c14e5ae759beb26a849426f6961
This commit is contained in:
Cole Faust 2021-12-01 13:49:12 -08:00
parent 5a0cef1c00
commit f1f49bb910
7 changed files with 55 additions and 269 deletions

View file

@ -221,16 +221,6 @@ else
.KATI_READONLY := TARGET_DEVICE_DIR
endif
# Dumps all variables that match [A-Z][A-Z0-9_]* to the file at $(1)
# It is used to print only the variables that are likely to be relevant to the
# board configuration.
define dump-public-variables
$(file >$(OUT_DIR)/dump-public-variables-temp.txt,$(subst $(space),$(newline),$(.VARIABLES)))\
$(file >$(1),\
$(foreach v, $(shell grep -he "^[A-Z][A-Z0-9_]*$$" $(OUT_DIR)/dump-public-variables-temp.txt | grep -vhE "^(SOONG_.*|LOCAL_PATH|TOPDIR|PRODUCT_COPY_OUT_.*)$$"),\
$(v) := $(strip $($(v)))$(newline)))
endef
# TODO(colefaust) change this if to RBC_PRODUCT_CONFIG when
# the board configuration is known to work on everything
# the product config works on.
@ -238,8 +228,7 @@ ifndef RBC_BOARD_CONFIG
include $(board_config_mk)
else
$(shell mkdir -p $(OUT_DIR)/rbc)
$(call dump-public-variables, $(OUT_DIR)/rbc/make_vars_pre_board_config.mk)
$(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_board_config.mk)
$(shell $(OUT_DIR)/soong/mk2rbc \
--mode=write -r --outdir $(OUT_DIR)/rbc \

View file

@ -310,6 +310,16 @@ ifeq (true,$(EMMA_INSTRUMENT_FRAMEWORK))
endif
#################################################################
# Dumps all variables that match [A-Z][A-Z0-9_]* (with a few exceptions)
# to the file at $(1). It is used to print only the variables that are
# likely to be relevant to the product or board configuration.
define dump-variables-rbc
$(file >$(OUT_DIR)/dump-variables-rbc-temp.txt,$(subst $(space),$(newline),$(.VARIABLES)))\
$(file >$(1),\
$(foreach v, $(shell grep -he "^[A-Z][A-Z0-9_]*$$" $(OUT_DIR)/dump-variables-rbc-temp.txt | grep -vhE "^(SOONG_.*|LOCAL_PATH|TOPDIR|PRODUCT_COPY_OUT_.*)$$"),\
$(v) := $(strip $($(v)))$(newline)))
endef
# Read the product specs so we can get TARGET_DEVICE and other
# variables that we need in order to locate the output files.
include $(BUILD_SYSTEM)/product_config.mk

View file

@ -1,224 +0,0 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(":build_id.rbc|init", _build_id_init = "init")
def _all_versions():
"""Returns all known versions."""
versions = ["OPR1", "OPD1", "OPD2", "OPM1", "OPM2", "PPR1", "PPD1", "PPD2", "PPM1", "PPM2", "QPR1"]
for v in ("Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"):
for e in ("P1A", "P1B", "P2A", "P2B", "D1A", "D1B", "D2A", "D2B", "Q1A", "Q1B", "Q2A", "Q2B", "Q3A", "Q3B"):
versions.append(v + e)
return versions
def _allowed_versions(all_versions, min_version, max_version, default_version):
"""Checks that version range and default versions is valid, returns all versions in range."""
for v in (min_version, max_version, default_version):
if v not in all_versions:
fail("% is invalid" % v)
min_i = all_versions.index(min_version)
max_i = all_versions.index(max_version)
def_i = all_versions.index(default_version)
if min_i > max_i:
fail("%s should come before %s in the version list" % (min_version, max_version))
if def_i < min_i or def_i > max_i:
fail("%s should come between % and %s" % (default_version, min_version, max_version))
return all_versions[min_i:max_i + 1]
# This function is a manual conversion of the version_defaults.mk
def _versions_default(g, all_versions, v):
"""Handle various build version information.
Guarantees that the following are defined:
PLATFORM_VERSION
PLATFORM_SDK_VERSION
PLATFORM_VERSION_CODENAME
DEFAULT_APP_TARGET_SDK
BUILD_ID
BUILD_NUMBER
PLATFORM_SECURITY_PATCH
PLATFORM_VNDK_VERSION
PLATFORM_SYSTEMSDK_VERSIONS
"""
# If build_id.rbc exists, it may override some of the defaults.
# Note that build.prop target also wants INTERNAL_BUILD_ID_MAKEFILE to be set if the file exists.
if _build_id_init != None:
_build_id_init(g)
g["INTERNAL_BUILD_ID_MAKEFILE"] = "build/make/core/build_id"
allowed_versions = _allowed_versions(all_versions, v.min_platform_version, v.max_platform_version, v.default_platform_version)
g.setdefault("TARGET_PLATFORM_VERSION", v.default_platform_version)
if g["TARGET_PLATFORM_VERSION"] not in allowed_versions:
fail("% is not valid, must be one of %s" % (g["TARGET_PLATFORM_VERSION"], allowed_versions))
g["DEFAULT_PLATFORM_VERSION"] = v.default_platform_version
g["PLATFORM_VERSION_LAST_STABLE"] = v.platform_version_last_stable
target_platform_version = g["TARGET_PLATFORM_VERSION"]
if v.codenames[target_platform_version]:
g.setdefault("PLATFORM_VERSION_CODENAME", v.codenames[target_platform_version])
else:
g.setdefault("PLATFORM_VERSION_CODENAME", target_platform_version)
# TODO(asmundak): set PLATFORM_VERSION_ALL_CODENAMES
g.setdefault("PLATFORM_SDK_VERSION", v.platform_sdk_version)
version_codename = g["PLATFORM_VERSION_CODENAME"]
if version_codename == "REL":
g.setdefault("PLATFORM_VERSION", g["PLATFORM_VERSION_LAST_STABLE"])
g["PLATFORM_PREVIEW_SDK_VERSION"] = 0
g.setdefault("DEFAULT_APP_TARGET_SDK", g["PLATFORM_SDK_VERSION"])
g.setdefault("PLATFORM_VNDK_VERSION", g["PLATFORM_SDK_VERSION"])
else:
g.setdefault("PLATFORM_VERSION", version_codename)
g.setdefault("PLATFORM_PREVIEW_SDK_VERSION", 1)
g.setdefault("DEFAULT_APP_TARGET_SDK", version_codename)
g.setdefault("PLATFORM_VNDK_VERSION", version_codename)
g.setdefault("PLATFORM_SYSTEMSDK_MIN_VERSION", 28)
versions = [str(i) for i in range(g["PLATFORM_SYSTEMSDK_MIN_VERSION"], g["PLATFORM_SDK_VERSION"] + 1)]
versions.append(version_codename)
g["PLATFORM_SYSTEMSDK_VERSIONS"] = sorted(versions)
# Used to indicate the security patch that has been applied to the device.
# It must signify that the build includes all security patches issued up through the designated Android Public Security Bulletin.
# It must be of the form "YYYY-MM-DD" on production devices.
# It must match one of the Android Security Patch Level strings of the Public Security Bulletins.
# If there is no $PLATFORM_SECURITY_PATCH set, keep it empty.
g.setdefault("PLATFORM_SECURITY_PATCH", v.platform_security_patch)
dt = 'TZ="GMT" %s' % g["PLATFORM_SECURITY_PATCH"]
g.setdefault("PLATFORM_SECURITY_PATCH_TIMESTAMP", rblf_shell("date -d '%s' +%%s" % dt))
# Used to indicate the base os applied to the device. Can be an arbitrary string, but must be a single word.
# If there is no $PLATFORM_BASE_OS set, keep it empty.
g.setdefault("PLATFORM_BASE_OS", "")
# Used to signify special builds. E.g., branches and/or releases, like "M5-RC7". Can be an arbitrary string, but
# must be a single word and a valid file name. If there is no BUILD_ID set, make it obvious.
g.setdefault("BUILD_ID", "UNKNOWN")
# BUILD_NUMBER should be set to the source control value that represents the current state of the source code.
# E.g., a perforce changelist number or a git hash. Can be an arbitrary string (to allow for source control that
# uses something other than numbers), but must be a single word and a valid file name.
#
# If no BUILD_NUMBER is set, create a useful "I am an engineering build from this date/time" value. Make it start
# with a non-digit so that anyone trying to parse it as an integer will probably get "0".
g.setdefault("BUILD_NUMBER", "eng.%s.%s" % (g["USER"], "TIMESTAMP"))
# Used to set minimum supported target sdk version. Apps targeting SDK version lower than the set value will result
# in a warning being shown when any activity from the app is started.
g.setdefault("PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION", 23)
# This is the sdk extension version of this tree.
g["PLATFORM_SDK_EXTENSION_VERSION"] = v.platform_sdk_extension_version
# This is the sdk extension version that PLATFORM_SDK_VERSION ships with.
g["PLATFORM_BASE_SDK_EXTENSION_VERSION"] = v.platform_base_sdk_extension_version
def init(g, v):
"""Initializes globals.
The code is the Starlark counterpart of the contents of the
envsetup.mk file.
Args:
g: globals dictionary
v: version info struct
"""
all_versions = _all_versions()
_versions_default(g, all_versions, v)
for v in all_versions:
g["IS_AT_LEAST" + v] = True
if v == g["TARGET_PLATFORM_VERSION"]:
break
# ---------------------------------------------------------------
# If you update the build system such that the environment setup or buildspec.mk need to be updated,
# increment this number, and people who haven't re-run those will have to do so before they can build.
# Make sure to also update the corresponding value in buildspec.mk.default and envsetup.sh.
g["CORRECT_BUILD_ENV_SEQUENCE_NUMBER"] = 13
g.setdefault("TARGET_PRODUCT", "aosp_arm")
g.setdefault("TARGET_BUILD_VARIANT", "eng")
g.setdefault("TARGET_BUILD_APPS", [])
g["TARGET_BUILD_UNBUNDLED"] = (g["TARGET_BUILD_APPS"] != []) or (getattr(g, "TARGET_BUILD_UNBUNDLED_IMAGE", "") != "")
# ---------------------------------------------------------------
# Set up configuration for host machine. We don't do cross-compiles except for arm, so the HOST
# is whatever we are running on.
host = rblf_shell("uname -sm")
if host.find("Linux") >= 0:
g["HOST_OS"] = "linux"
elif host.find("Darwin") >= 0:
g["HOST_OS"] = "darwin"
else:
fail("Cannot run on %s OS" % host)
# TODO(asmundak): set g.HOST_OS_EXTRA
g["BUILD_OS"] = g["HOST_OS"]
# TODO(asmundak): check cross-OS build
if host.find("x86_64") >= 0:
g["HOST_ARCH"] = "x86_64"
g["HOST_2ND_ARCH"] = "x86"
g["HOST_IS_64_BIT"] = True
elif host.find("i686") >= 0 or host.find("x86") >= 0:
fail("Building on a 32-bit x86 host is not supported: %s" % host)
elif g["HOST_OS"] == "darwin":
g["HOST_2ND_ARCH"] = ""
g["HOST_2ND_ARCH_VAR_PREFIX"] = "2ND_"
g["HOST_2ND_ARCH_MODULE_SUFFIX"] = "_32"
g["HOST_CROSS_2ND_ARCH_VAR_PREFIX"] = "2ND_"
g["HOST_CROSS_2ND_ARCH_MODULE_SUFFIX"] = "_64"
g["TARGET_2ND_ARCH_VAR_PREFIX"] = "2ND_"
# TODO(asmundak): envsetup.mk lines 216-226:
# convert combo-related stuff from combo/select.mk
# on windows, the tools have .exe at the end, and we depend on the
# host config stuff being done first
g["BUILD_ARCH"] = g["HOST_ARCH"]
g["BUILD_2ND_ARCH"] = g["HOST_2ND_ARCH"]
# the host build defaults to release, and it must be release or debug
g.setdefault("HOST_BUILD_TYPE", "release")
if g["HOST_BUILD_TYPE"] not in ["release", "debug"]:
fail("HOST_BUILD_TYPE must be either release or debug, not '%s'" % g["HOST_BUILD_TYPE"])
g.update([
("TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"),
("TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"),
("TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"),
("TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"),
("TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"),
("TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"),
("TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"),
])
# TODO(asmundak): there is more stuff in envsetup.mk lines 249-292, but
# it does not seem to affect product configuration. Revisit this.
g["ART_APEX_JARS"] = [
"com.android.art:core-oj",
"com.android.art:core-libart",
"com.android.art:okhttp",
"com.android.art:bouncycastle",
"com.android.art:apache-xml",
]
if g.get("TARGET_BUILD_TYPE", "") != "debug":
g["TARGET_BUILD_TYPE"] = "release"

View file

@ -206,12 +206,18 @@ endif
ifndef RBC_PRODUCT_CONFIG
$(call import-products, $(current_product_makefile))
else
$(shell build/soong/scripts/update_out $(OUT_DIR)/rbctemp.mk \
build/soong/scripts/rbc-run $(current_product_makefile))
$(shell mkdir -p $(OUT_DIR)/rbc)
$(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_product_config.mk)
$(shell build/soong/scripts/update_out \
$(OUT_DIR)/rbc/rbc_product_config_results.mk \
build/soong/scripts/rbc-run \
$(current_product_makefile) \
$(OUT_DIR)/rbc/make_vars_pre_product_config.mk)
ifneq ($(.SHELLSTATUS),0)
$(error product configuration converter failed: $(.SHELLSTATUS))
endif
include $(OUT_DIR)/rbctemp.mk
include $(OUT_DIR)/rbc/rbc_product_config_results.mk
PRODUCTS += $(current_product_makefile)
endif
endif # Import all or just the current product makefile

View file

@ -12,26 +12,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//build/make/core:envsetup.rbc", _envsetup_init = "init")
"""Runtime functions."""
_soong_config_namespaces_key = "$SOONG_CONFIG_NAMESPACES"
_dist_for_goals_key = "$dist_for_goals"
def _init_globals(version_info):
"""Returns dict created from the runtime environment."""
globals = dict()
def _init_globals(input_variables_init):
"""Initializes dictionaries of global variables.
# Environment variables
for k in dir(rblf_env):
globals[k] = getattr(rblf_env, k)
This function runs the given input_variables_init function,
passing it a globals dictionary and a handle as if it
were a regular product. It then returns 2 copies of
the globals dictionary, so that one can be kept around
to diff changes made to the other later.
"""
globals_base = {"PRODUCT_SOONG_NAMESPACES": []}
input_variables_init(globals_base, __h_new())
# Variables set as var=value command line arguments
for k in dir(rblf_cli):
globals[k] = getattr(rblf_cli, k)
globals.setdefault("PRODUCT_SOONG_NAMESPACES", [])
_envsetup_init(globals, version_info)
# Rerun input_variables_init to produce a copy
# of globals_base, because starlark doesn't support
# deep copying objects.
globals = {"PRODUCT_SOONG_NAMESPACES": []}
input_variables_init(globals, __h_new())
# Variables that should be defined.
mandatory_vars = [
@ -39,15 +40,14 @@ def _init_globals(version_info):
"PLATFORM_VERSION",
"PRODUCT_SOONG_NAMESPACES",
# TODO(asmundak): do we need TARGET_ARCH? AOSP does not reference it
"TARGET_BUILD_TYPE",
"TARGET_BUILD_VARIANT",
"TARGET_PRODUCT",
]
for bv in mandatory_vars:
if not bv in globals:
fail(bv, " is not defined")
return globals
return (globals, globals_base)
def __print_attr(attr, value):
# Allow using empty strings to clear variables, but not None values
@ -111,7 +111,7 @@ def __printvars_rearrange_list(value_list):
seen = {item: 0 for item in value_list}
return sorted(seen.keys()) if _options.rearrange == "sort" else seen.keys()
def _product_configuration(top_pcm_name, top_pcm, version_info):
def _product_configuration(top_pcm_name, top_pcm, input_variables_init):
"""Creates configuration."""
# Product configuration is created by traversing product's inheritance
@ -125,8 +125,7 @@ def _product_configuration(top_pcm_name, top_pcm, version_info):
# PCM means "Product Configuration Module", i.e., a Starlark file
# whose body consists of a single init function.
globals_base = _init_globals(version_info)
globals = dict(**globals_base)
globals, globals_base = _init_globals(input_variables_init)
config_postfix = [] # Configs in postfix order
@ -725,7 +724,6 @@ rblf = struct(
filter = _filter,
filter_out = _filter_out,
find_and_copy = _find_and_copy,
init_globals = _init_globals,
inherit = _inherit,
indirect = _indirect,
mk2rbc_error = _mk2rbc_error,

View file

@ -1,4 +1,3 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -13,9 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# This file has been manually converted from build_id.mk
def init(g):
# This file was generated by running `m RBC_PRODUCT_CONFIG=1 nothing`
# and then copying it from out/rbc/out/rbc/make_vars_pre_product_config.rbc.
# It was manually trimmed down afterwards to just the variables we need.
# BUILD_ID is usually used to specify the branch name (like "MAIN") or a branch name and a release candidate
# (like "CRB01"). It must be a single word, and is capitalized by convention.
g["BUILD_ID"]="AOSP.MASTER"
load("//build/make/core:product_config.rbc", "rblf")
def init(g, handle):
cfg = rblf.cfg(handle)
g["PLATFORM_VERSION_CODENAME"] = "Tiramisu"
g["PLATFORM_VERSION"] = "Tiramisu"
g["TARGET_BUILD_VARIANT"] = "userdebug"
g["TARGET_BUILD_TYPE"] = "release"
g["TARGET_PRODUCT"] = "aosp_arm64"
g["PLATFORM_SDK_VERSION"] = "31"

View file

@ -21,7 +21,7 @@
# * all runtime functions (wildcard, regex, etc.) work
load("//build/make/core:product_config.rbc", "rblf")
load(":version_defaults.rbc", "version_defaults")
load(":input_variables.rbc", input_variables_init = "init")
load(":product.rbc", "init")
load(":board.rbc", board_init = "init")
load(":board_input_vars.rbc", board_input_vars_init = "init")
@ -59,7 +59,7 @@ assert_eq("foo.c no_folder", rblf.notdir("src/foo.c no_folder"))
assert_eq("", rblf.notdir("/"))
assert_eq("", rblf.notdir(""))
(globals, config, globals_base) = rblf.product_configuration("test/device", init, version_defaults)
(globals, config, globals_base) = rblf.product_configuration("test/device", init, input_variables_init)
assert_eq(
{
"PRODUCT_COPY_FILES": [
@ -99,8 +99,8 @@ assert_eq(
{k:v for k, v in sorted(ns.items()) }
)
assert_eq("S", globals["PLATFORM_VERSION"])
assert_eq(30, globals["PLATFORM_SDK_VERSION"])
assert_eq("Tiramisu", globals["PLATFORM_VERSION"])
assert_eq("31", globals["PLATFORM_SDK_VERSION"])
assert_eq("xyz", rblf.soong_config_get(globals, "NS2", "v3"))
assert_eq(None, rblf.soong_config_get(globals, "NS2", "nonexistant_var"))