Pass input variables to product config

Passing variables via a makefile instead of
rblf_cli / rblf_env allows us to give them correct
types while converting the makefile to starlark,
as opposed to the variables always being strings
when given via rblf_cli / rblf_env.

This also allows us to remove some hand-converted
starlark code.

Bug: 201700692
Test: ./out/soong/rbcrun ./build/make/tests/run.rbc
Change-Id: I58c4f20b29171c14e5ae759beb26a849426f6961

core/board_config.mk

@@ -221,16 +221,6 @@ else
   .KATI_READONLY := TARGET_DEVICE_DIR
 endif
 
-# Dumps all variables that match [A-Z][A-Z0-9_]* to the file at $(1)
-# It is used to print only the variables that are likely to be relevant to the
-# board configuration.
-define dump-public-variables
-$(file >$(OUT_DIR)/dump-public-variables-temp.txt,$(subst $(space),$(newline),$(.VARIABLES)))\
-$(file >$(1),\
-$(foreach v, $(shell grep -he "^[A-Z][A-Z0-9_]*$$" $(OUT_DIR)/dump-public-variables-temp.txt | grep -vhE "^(SOONG_.*|LOCAL_PATH|TOPDIR|PRODUCT_COPY_OUT_.*)$$"),\
-$(v) := $(strip $($(v)))$(newline)))
-endef
-
 # TODO(colefaust) change this if to RBC_PRODUCT_CONFIG when
 # the board configuration is known to work on everything
 # the product config works on.
@@ -238,8 +228,7 @@ ifndef RBC_BOARD_CONFIG
 include $(board_config_mk)
 else
   $(shell mkdir -p $(OUT_DIR)/rbc)
-
+  $(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_board_config.mk)
-  $(call dump-public-variables, $(OUT_DIR)/rbc/make_vars_pre_board_config.mk)
 
   $(shell $(OUT_DIR)/soong/mk2rbc \
     --mode=write -r --outdir $(OUT_DIR)/rbc \

core/envsetup.mk

@@ -310,6 +310,16 @@ ifeq (true,$(EMMA_INSTRUMENT_FRAMEWORK))
 endif
 #################################################################
 
+# Dumps all variables that match [A-Z][A-Z0-9_]* (with a few exceptions)
+# to the file at $(1). It is used to print only the variables that are
+# likely to be relevant to the product or board configuration.
+define dump-variables-rbc
+$(file >$(OUT_DIR)/dump-variables-rbc-temp.txt,$(subst $(space),$(newline),$(.VARIABLES)))\
+$(file >$(1),\
+$(foreach v, $(shell grep -he "^[A-Z][A-Z0-9_]*$$" $(OUT_DIR)/dump-variables-rbc-temp.txt | grep -vhE "^(SOONG_.*|LOCAL_PATH|TOPDIR|PRODUCT_COPY_OUT_.*)$$"),\
+$(v) := $(strip $($(v)))$(newline)))
+endef
+
 # Read the product specs so we can get TARGET_DEVICE and other
 # variables that we need in order to locate the output files.
 include $(BUILD_SYSTEM)/product_config.mk

core/envsetup.rbc

@@ -1,224 +0,0 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-load(":build_id.rbc|init", _build_id_init = "init")
-
-def _all_versions():
-    """Returns all known versions."""
-    versions = ["OPR1", "OPD1", "OPD2", "OPM1", "OPM2", "PPR1", "PPD1", "PPD2", "PPM1", "PPM2", "QPR1"]
-    for v in ("Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"):
-        for e in ("P1A", "P1B", "P2A", "P2B", "D1A", "D1B", "D2A", "D2B", "Q1A", "Q1B", "Q2A", "Q2B", "Q3A", "Q3B"):
-            versions.append(v + e)
-    return versions
-
-def _allowed_versions(all_versions, min_version, max_version, default_version):
-    """Checks that version range and default versions is valid, returns all versions in range."""
-    for v in (min_version, max_version, default_version):
-        if v not in all_versions:
-            fail("% is invalid" % v)
-
-    min_i = all_versions.index(min_version)
-    max_i = all_versions.index(max_version)
-    def_i = all_versions.index(default_version)
-    if min_i > max_i:
-        fail("%s should come before %s in the version list" % (min_version, max_version))
-    if def_i < min_i or def_i > max_i:
-        fail("%s should come between % and %s" % (default_version, min_version, max_version))
-    return all_versions[min_i:max_i + 1]
-
-# This function is a manual conversion of the version_defaults.mk
-def _versions_default(g, all_versions, v):
-    """Handle various build version information.
-
-    Guarantees that the following are defined:
-     PLATFORM_VERSION
-     PLATFORM_SDK_VERSION
-     PLATFORM_VERSION_CODENAME
-     DEFAULT_APP_TARGET_SDK
-     BUILD_ID
-     BUILD_NUMBER
-     PLATFORM_SECURITY_PATCH
-     PLATFORM_VNDK_VERSION
-     PLATFORM_SYSTEMSDK_VERSIONS
-    """
-
-    # If build_id.rbc exists, it may override some of the defaults.
-    # Note that build.prop target also wants INTERNAL_BUILD_ID_MAKEFILE to be set if the file exists.
-    if _build_id_init != None:
-        _build_id_init(g)
-        g["INTERNAL_BUILD_ID_MAKEFILE"] = "build/make/core/build_id"
-
-    allowed_versions = _allowed_versions(all_versions, v.min_platform_version, v.max_platform_version, v.default_platform_version)
-    g.setdefault("TARGET_PLATFORM_VERSION", v.default_platform_version)
-    if g["TARGET_PLATFORM_VERSION"] not in allowed_versions:
-        fail("% is not valid, must be one of %s" % (g["TARGET_PLATFORM_VERSION"], allowed_versions))
-
-    g["DEFAULT_PLATFORM_VERSION"] = v.default_platform_version
-    g["PLATFORM_VERSION_LAST_STABLE"] = v.platform_version_last_stable
-    target_platform_version = g["TARGET_PLATFORM_VERSION"]
-    if v.codenames[target_platform_version]:
-        g.setdefault("PLATFORM_VERSION_CODENAME", v.codenames[target_platform_version])
-    else:
-        g.setdefault("PLATFORM_VERSION_CODENAME", target_platform_version)
-    # TODO(asmundak): set PLATFORM_VERSION_ALL_CODENAMES
-
-    g.setdefault("PLATFORM_SDK_VERSION", v.platform_sdk_version)
-    version_codename = g["PLATFORM_VERSION_CODENAME"]
-    if version_codename == "REL":
-        g.setdefault("PLATFORM_VERSION", g["PLATFORM_VERSION_LAST_STABLE"])
-        g["PLATFORM_PREVIEW_SDK_VERSION"] = 0
-        g.setdefault("DEFAULT_APP_TARGET_SDK", g["PLATFORM_SDK_VERSION"])
-        g.setdefault("PLATFORM_VNDK_VERSION", g["PLATFORM_SDK_VERSION"])
-    else:
-        g.setdefault("PLATFORM_VERSION", version_codename)
-        g.setdefault("PLATFORM_PREVIEW_SDK_VERSION", 1)
-        g.setdefault("DEFAULT_APP_TARGET_SDK", version_codename)
-        g.setdefault("PLATFORM_VNDK_VERSION", version_codename)
-
-    g.setdefault("PLATFORM_SYSTEMSDK_MIN_VERSION", 28)
-    versions = [str(i) for i in range(g["PLATFORM_SYSTEMSDK_MIN_VERSION"], g["PLATFORM_SDK_VERSION"] + 1)]
-    versions.append(version_codename)
-    g["PLATFORM_SYSTEMSDK_VERSIONS"] = sorted(versions)
-
-    #  Used to indicate the security patch that has been applied to the device.
-    #  It must signify that the build includes all security patches issued up through the designated Android Public Security Bulletin.
-    #  It must be of the form "YYYY-MM-DD" on production devices.
-    #  It must match one of the Android Security Patch Level strings of the Public Security Bulletins.
-    #  If there is no $PLATFORM_SECURITY_PATCH set, keep it empty.
-
-    g.setdefault("PLATFORM_SECURITY_PATCH", v.platform_security_patch)
-    dt = 'TZ="GMT" %s' % g["PLATFORM_SECURITY_PATCH"]
-    g.setdefault("PLATFORM_SECURITY_PATCH_TIMESTAMP", rblf_shell("date -d '%s' +%%s" % dt))
-
-    # Used to indicate the base os applied to the device. Can be an arbitrary string, but must be a single word.
-    # If there is no $PLATFORM_BASE_OS set, keep it empty.
-    g.setdefault("PLATFORM_BASE_OS", "")
-
-    # Used to signify special builds.  E.g., branches and/or releases, like "M5-RC7".  Can be an arbitrary string, but
-    # must be a single word and a valid file name. If there is no BUILD_ID set, make it obvious.
-    g.setdefault("BUILD_ID", "UNKNOWN")
-
-    # BUILD_NUMBER should be set to the source control value that represents the current state of the source code.
-    # E.g., a perforce changelist number or a git hash.  Can be an arbitrary string (to allow for source control that
-    # uses something other than numbers), but must be a single word and a valid file name.
-    #
-    # If no BUILD_NUMBER is set, create a useful "I am an engineering build from this date/time" value.  Make it start
-    # with a non-digit so that anyone trying to parse it as an integer will probably get "0".
-    g.setdefault("BUILD_NUMBER", "eng.%s.%s" % (g["USER"], "TIMESTAMP"))
-
-    # Used to set minimum supported target sdk version. Apps targeting SDK version lower than the set value will result
-    # in a warning being shown when any activity from the app is started.
-    g.setdefault("PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION", 23)
-
-    # This is the sdk extension version of this tree.
-    g["PLATFORM_SDK_EXTENSION_VERSION"] = v.platform_sdk_extension_version
-    # This is the sdk extension version that PLATFORM_SDK_VERSION ships with.
-    g["PLATFORM_BASE_SDK_EXTENSION_VERSION"] = v.platform_base_sdk_extension_version
-
-
-def init(g, v):
-    """Initializes globals.
-
-    The code is the Starlark counterpart of the contents of the
-    envsetup.mk file.
-    Args:
-        g: globals dictionary
-        v: version info struct
-    """
-    all_versions = _all_versions()
-    _versions_default(g, all_versions, v)
-    for v in all_versions:
-        g["IS_AT_LEAST" + v] = True
-        if v == g["TARGET_PLATFORM_VERSION"]:
-            break
-
-    # ---------------------------------------------------------------
-    # If you update the build system such that the environment setup or buildspec.mk need to be updated,
-    # increment this number, and people who haven't re-run those will have to do so before they can build.
-    # Make sure to also update the corresponding value in buildspec.mk.default and envsetup.sh.
-    g["CORRECT_BUILD_ENV_SEQUENCE_NUMBER"] = 13
-
-    g.setdefault("TARGET_PRODUCT", "aosp_arm")
-    g.setdefault("TARGET_BUILD_VARIANT", "eng")
-
-    g.setdefault("TARGET_BUILD_APPS", [])
-    g["TARGET_BUILD_UNBUNDLED"] = (g["TARGET_BUILD_APPS"] != []) or (getattr(g, "TARGET_BUILD_UNBUNDLED_IMAGE", "") != "")
-
-    # ---------------------------------------------------------------
-    # Set up configuration for host machine.  We don't do cross-compiles except for arm, so the HOST
-    # is whatever we are running on.
-    host = rblf_shell("uname -sm")
-    if host.find("Linux") >= 0:
-        g["HOST_OS"] = "linux"
-    elif host.find("Darwin") >= 0:
-        g["HOST_OS"] = "darwin"
-    else:
-        fail("Cannot run on %s OS" % host)
-
-    # TODO(asmundak): set g.HOST_OS_EXTRA
-
-    g["BUILD_OS"] = g["HOST_OS"]
-
-    # TODO(asmundak): check cross-OS build
-
-    if host.find("x86_64") >= 0:
-        g["HOST_ARCH"] = "x86_64"
-        g["HOST_2ND_ARCH"] = "x86"
-        g["HOST_IS_64_BIT"] = True
-    elif host.find("i686") >= 0 or host.find("x86") >= 0:
-        fail("Building on a 32-bit x86 host is not supported: %s" % host)
-    elif g["HOST_OS"] == "darwin":
-        g["HOST_2ND_ARCH"] = ""
-
-    g["HOST_2ND_ARCH_VAR_PREFIX"] = "2ND_"
-    g["HOST_2ND_ARCH_MODULE_SUFFIX"] = "_32"
-    g["HOST_CROSS_2ND_ARCH_VAR_PREFIX"] = "2ND_"
-    g["HOST_CROSS_2ND_ARCH_MODULE_SUFFIX"] = "_64"
-    g["TARGET_2ND_ARCH_VAR_PREFIX"] = "2ND_"
-
-    # TODO(asmundak): envsetup.mk lines 216-226:
-    # convert combo-related stuff from combo/select.mk
-
-    # on windows, the tools have .exe at the end, and we depend on the
-    # host config stuff being done first
-    g["BUILD_ARCH"] = g["HOST_ARCH"]
-    g["BUILD_2ND_ARCH"] = g["HOST_2ND_ARCH"]
-
-    # the host build defaults to release, and it must be release or debug
-    g.setdefault("HOST_BUILD_TYPE", "release")
-    if g["HOST_BUILD_TYPE"] not in ["release", "debug"]:
-        fail("HOST_BUILD_TYPE must be either release or debug, not '%s'" % g["HOST_BUILD_TYPE"])
-
-    g.update([
-	    ("TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"),
-    	("TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"),
-	    ("TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"),
-	    ("TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"),
-	    ("TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"),
-	    ("TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"),
-	    ("TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"),
-        ])
-
-    # TODO(asmundak): there is more stuff in envsetup.mk lines 249-292, but
-    # it does not seem to affect product configuration. Revisit this.
-    g["ART_APEX_JARS"] = [
-        "com.android.art:core-oj",
-        "com.android.art:core-libart",
-        "com.android.art:okhttp",
-        "com.android.art:bouncycastle",
-        "com.android.art:apache-xml",
-    ]
-
-    if g.get("TARGET_BUILD_TYPE", "") != "debug":
-        g["TARGET_BUILD_TYPE"] = "release"

core/product_config.mk

@@ -206,12 +206,18 @@ endif
 ifndef RBC_PRODUCT_CONFIG
 $(call import-products, $(current_product_makefile))
 else
-  $(shell build/soong/scripts/update_out $(OUT_DIR)/rbctemp.mk \
+  $(shell mkdir -p $(OUT_DIR)/rbc)
-      build/soong/scripts/rbc-run $(current_product_makefile))
+  $(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_product_config.mk)
+
+  $(shell build/soong/scripts/update_out \
+    $(OUT_DIR)/rbc/rbc_product_config_results.mk \
+    build/soong/scripts/rbc-run \
+    $(current_product_makefile) \
+    $(OUT_DIR)/rbc/make_vars_pre_product_config.mk)
   ifneq ($(.SHELLSTATUS),0)
     $(error product configuration converter failed: $(.SHELLSTATUS))
   endif
-  include $(OUT_DIR)/rbctemp.mk
+  include $(OUT_DIR)/rbc/rbc_product_config_results.mk
   PRODUCTS += $(current_product_makefile)
 endif
 endif  # Import all or just the current product makefile

core/product_config.rbc

@@ -12,26 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//build/make/core:envsetup.rbc", _envsetup_init = "init")
-
 """Runtime functions."""
 
 _soong_config_namespaces_key = "$SOONG_CONFIG_NAMESPACES"
 _dist_for_goals_key = "$dist_for_goals"
-def _init_globals(version_info):
+def _init_globals(input_variables_init):
-    """Returns dict created from the runtime environment."""
+    """Initializes dictionaries of global variables.
-    globals = dict()
 
-    # Environment variables
+    This function runs the given input_variables_init function,
-    for k in dir(rblf_env):
+    passing it a globals dictionary and a handle as if it
-        globals[k] = getattr(rblf_env, k)
+    were a regular product. It then returns 2 copies of
+    the globals dictionary, so that one can be kept around
+    to diff changes made to the other later.
+    """
+    globals_base = {"PRODUCT_SOONG_NAMESPACES": []}
+    input_variables_init(globals_base, __h_new())
 
-    # Variables set as var=value command line arguments
+    # Rerun input_variables_init to produce a copy
-    for k in dir(rblf_cli):
+    # of globals_base, because starlark doesn't support
-        globals[k] = getattr(rblf_cli, k)
+    # deep copying objects.
-
+    globals = {"PRODUCT_SOONG_NAMESPACES": []}
-    globals.setdefault("PRODUCT_SOONG_NAMESPACES", [])
+    input_variables_init(globals, __h_new())
-    _envsetup_init(globals, version_info)
 
     # Variables that should be defined.
     mandatory_vars = [
@@ -39,15 +40,14 @@ def _init_globals(version_info):
         "PLATFORM_VERSION",
         "PRODUCT_SOONG_NAMESPACES",
         # TODO(asmundak): do we need TARGET_ARCH? AOSP does not reference it
-        "TARGET_BUILD_TYPE",
         "TARGET_BUILD_VARIANT",
         "TARGET_PRODUCT",
     ]
     for bv in mandatory_vars:
         if not bv in globals:
             fail(bv, " is not defined")
-    return globals
 
+    return (globals, globals_base)
 
 def __print_attr(attr, value):
     # Allow using empty strings to clear variables, but not None values
@@ -111,7 +111,7 @@ def __printvars_rearrange_list(value_list):
     seen = {item: 0 for item in value_list}
     return sorted(seen.keys()) if _options.rearrange == "sort" else seen.keys()
 
-def _product_configuration(top_pcm_name, top_pcm, version_info):
+def _product_configuration(top_pcm_name, top_pcm, input_variables_init):
     """Creates configuration."""
 
     # Product configuration is created by traversing product's inheritance
@@ -125,8 +125,7 @@ def _product_configuration(top_pcm_name, top_pcm, version_info):
     # PCM means "Product Configuration Module", i.e., a Starlark file
     # whose body consists of a single init function.
 
-    globals_base = _init_globals(version_info)
+    globals, globals_base = _init_globals(input_variables_init)
-    globals = dict(**globals_base)
 
     config_postfix = []  # Configs in postfix order
 
@@ -725,7 +724,6 @@ rblf = struct(
     filter = _filter,
     filter_out = _filter_out,
     find_and_copy = _find_and_copy,
-    init_globals = _init_globals,
     inherit = _inherit,
     indirect = _indirect,
     mk2rbc_error = _mk2rbc_error,

tests/input_variables.rbc

@@ -1,4 +1,3 @@
-
 # Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This file has been manually converted from build_id.mk
+# This file was generated by running `m RBC_PRODUCT_CONFIG=1 nothing`
-def init(g):
+# and then copying it from out/rbc/out/rbc/make_vars_pre_product_config.rbc.
+# It was manually trimmed down afterwards to just the variables we need.
 
-    # BUILD_ID is usually used to specify the branch name (like "MAIN") or a branch name and a release candidate
+load("//build/make/core:product_config.rbc", "rblf")
-    # (like "CRB01").  It must be a single word, and is capitalized by convention.
+
-    g["BUILD_ID"]="AOSP.MASTER"
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  g["PLATFORM_VERSION_CODENAME"] = "Tiramisu"
+  g["PLATFORM_VERSION"] = "Tiramisu"
+  g["TARGET_BUILD_VARIANT"] = "userdebug"
+  g["TARGET_BUILD_TYPE"] = "release"
+  g["TARGET_PRODUCT"] = "aosp_arm64"
+  g["PLATFORM_SDK_VERSION"] = "31"

tests/run.rbc

@@ -21,7 +21,7 @@
 #  * all runtime functions (wildcard, regex, etc.) work
 
 load("//build/make/core:product_config.rbc", "rblf")
-load(":version_defaults.rbc", "version_defaults")
+load(":input_variables.rbc", input_variables_init = "init")
 load(":product.rbc", "init")
 load(":board.rbc", board_init = "init")
 load(":board_input_vars.rbc", board_input_vars_init = "init")
@@ -59,7 +59,7 @@ assert_eq("foo.c no_folder", rblf.notdir("src/foo.c no_folder"))
 assert_eq("", rblf.notdir("/"))
 assert_eq("", rblf.notdir(""))
 
-(globals, config, globals_base) = rblf.product_configuration("test/device", init, version_defaults)
+(globals, config, globals_base) = rblf.product_configuration("test/device", init, input_variables_init)
 assert_eq(
     {
       "PRODUCT_COPY_FILES": [
@@ -99,8 +99,8 @@ assert_eq(
     {k:v for k, v in sorted(ns.items()) }
 )
 
-assert_eq("S", globals["PLATFORM_VERSION"])
+assert_eq("Tiramisu", globals["PLATFORM_VERSION"])
-assert_eq(30, globals["PLATFORM_SDK_VERSION"])
+assert_eq("31", globals["PLATFORM_SDK_VERSION"])
 
 assert_eq("xyz", rblf.soong_config_get(globals, "NS2", "v3"))
 assert_eq(None, rblf.soong_config_get(globals, "NS2", "nonexistant_var"))