Merge "Emit a system image target for the current product" into main

This commit is contained in:
Cole Faust 2023-09-25 17:08:34 +00:00 committed by Gerrit Code Review
commit 71aa535721
4 changed files with 377 additions and 55 deletions

View file

@ -1779,18 +1779,4 @@ var (
"art_": DEFAULT_PRIORITIZED_WEIGHT,
"ndk_library": DEFAULT_PRIORITIZED_WEIGHT,
}
BazelSandwichTargets = []struct {
Label string
Host bool
}{
{
Label: "//build/bazel/examples/partitions:system_image",
Host: false,
},
{
Label: "//build/bazel/examples/partitions:run_test",
Host: false,
},
}
)

View file

@ -33,10 +33,10 @@ import (
"android/soong/shared"
"android/soong/starlark_import"
"android/soong/bazel"
"github.com/google/blueprint"
"github.com/google/blueprint/metrics"
"android/soong/bazel"
)
var (
@ -1048,39 +1048,46 @@ var (
allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd}
)
// This can't be part of bp2build_product_config.go because it would create a circular go package dependency
func getLabelsForBazelSandwichPartitions(variables *ProductVariables) []string {
targetProduct := "unknown"
if variables.DeviceProduct != nil {
targetProduct = *variables.DeviceProduct
}
currentProductFolder := fmt.Sprintf("build/bazel/products/%s", targetProduct)
if len(variables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory) > 0 {
currentProductFolder = fmt.Sprintf("%s%s", variables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory, targetProduct)
}
var ret []string
if variables.PartitionVarsForBazelMigrationOnlyDoNotUse.PartitionQualifiedVariables["system"].BuildingImage {
ret = append(ret, "@//"+currentProductFolder+":system_image")
ret = append(ret, "@//"+currentProductFolder+":run_system_image_test")
}
return ret
}
func GetBazelSandwichCqueryRequests(config Config) ([]cqueryKey, error) {
result := make([]cqueryKey, 0, len(allowlists.BazelSandwichTargets))
partitionLabels := getLabelsForBazelSandwichPartitions(&config.productVariables)
result := make([]cqueryKey, 0, len(partitionLabels))
labelRegex := regexp.MustCompile("^@?//([a-zA-Z0-9/_-]+):[a-zA-Z0-9_-]+$")
// Note that bazel "targets" are different from soong "targets", the bazel targets are
// synonymous with soong modules, and soong targets are a configuration a module is built in.
for _, target := range allowlists.BazelSandwichTargets {
match := labelRegex.FindStringSubmatch(target.Label)
for _, target := range partitionLabels {
match := labelRegex.FindStringSubmatch(target)
if match == nil {
return nil, fmt.Errorf("invalid label, must match `^@?//([a-zA-Z0-9/_-]+):[a-zA-Z0-9_-]+$`: %s", target.Label)
}
if _, err := os.Stat(absolutePath(match[1])); err != nil {
if os.IsNotExist(err) {
// Ignore bazel sandwich targets that don't exist.
continue
} else {
return nil, err
}
return nil, fmt.Errorf("invalid label, must match `^@?//([a-zA-Z0-9/_-]+):[a-zA-Z0-9_-]+$`: %s", target)
}
var soongTarget Target
if target.Host {
soongTarget = config.BuildOSTarget
} else {
soongTarget = config.AndroidCommonTarget
if soongTarget.Os.Class != Device {
// kernel-build-tools seems to set the AndroidCommonTarget to a linux host
// target for some reason, disable device builds in that case.
continue
}
// change this to config.BuildOSTarget if we add host targets
soongTarget := config.AndroidCommonTarget
if soongTarget.Os.Class != Device {
// kernel-build-tools seems to set the AndroidCommonTarget to a linux host
// target for some reason, disable device builds in that case.
continue
}
result = append(result, cqueryKey{
label: target.Label,
label: target,
requestType: cquery.GetOutputFiles,
configKey: configKey{
arch: soongTarget.Arch.String(),

View file

@ -485,11 +485,55 @@ type ProductVariables struct {
CheckVendorSeappViolations *bool `json:",omitempty"`
// PartitionsVars are extra variables that are used to define the partition images. They should
// not be read from soong modules.
PartitionVars struct {
ProductDirectory string `json:",omitempty"`
} `json:",omitempty"`
// PartitionVarsForBazelMigrationOnlyDoNotUse are extra variables that are used to define the
// partition images. They should not be read from soong modules.
PartitionVarsForBazelMigrationOnlyDoNotUse PartitionVariables `json:",omitempty"`
}
type PartitionVariables struct {
ProductDirectory string `json:",omitempty"`
PartitionQualifiedVariables map[string]struct {
BuildingImage bool `json:",omitempty"`
BoardErofsCompressor string `json:",omitempty"`
BoardErofsCompressHints string `json:",omitempty"`
BoardErofsPclusterSize string `json:",omitempty"`
BoardExtfsInodeCount string `json:",omitempty"`
BoardExtfsRsvPct string `json:",omitempty"`
BoardF2fsSloadCompressFlags string `json:",omitempty"`
BoardFileSystemCompress string `json:",omitempty"`
BoardFileSystemType string `json:",omitempty"`
BoardJournalSize string `json:",omitempty"`
BoardPartitionReservedSize string `json:",omitempty"`
BoardPartitionSize string `json:",omitempty"`
BoardSquashfsBlockSize string `json:",omitempty"`
BoardSquashfsCompressor string `json:",omitempty"`
BoardSquashfsCompressorOpt string `json:",omitempty"`
BoardSquashfsDisable4kAlign string `json:",omitempty"`
ProductBaseFsPath string `json:",omitempty"`
ProductHeadroom string `json:",omitempty"`
ProductVerityPartition string `json:",omitempty"`
}
TargetUserimagesUseExt2 bool `json:",omitempty"`
TargetUserimagesUseExt3 bool `json:",omitempty"`
TargetUserimagesUseExt4 bool `json:",omitempty"`
TargetUserimagesSparseExtDisabled bool `json:",omitempty"`
TargetUserimagesSparseErofsDisabled bool `json:",omitempty"`
TargetUserimagesSparseSquashfsDisabled bool `json:",omitempty"`
TargetUserimagesSparseF2fsDisabled bool `json:",omitempty"`
BoardErofsCompressor string `json:",omitempty"`
BoardErofsCompressorHints string `json:",omitempty"`
BoardErofsPclusterSize string `json:",omitempty"`
BoardErofsShareDupBlocks string `json:",omitempty"`
BoardErofsUseLegacyCompression string `json:",omitempty"`
BoardExt4ShareDupBlocks string `json:",omitempty"`
BoardFlashLogicalBlockSize string `json:",omitempty"`
BoardFlashEraseBlockSize string `json:",omitempty"`
BoardUsesRecoveryAsBoot bool `json:",omitempty"`
BoardBuildGkiBootImageWithoutRamdisk bool `json:",omitempty"`
ProductUseDynamicPartitionSize bool `json:",omitempty"`
CopyImagesForTargetFilesZip bool `json:",omitempty"`
}
func boolPtr(v bool) *bool {

View file

@ -22,6 +22,16 @@ type createProductConfigFilesResult struct {
bp2buildTargets map[string]BazelTargets
}
type bazelLabel struct {
repo string
pkg string
target string
}
func (l *bazelLabel) String() string {
return fmt.Sprintf("@%s//%s:%s", l.repo, l.pkg, l.target)
}
func createProductConfigFiles(
ctx *CodegenContext,
metrics CodegenMetrics) (createProductConfigFilesResult, error) {
@ -54,8 +64,8 @@ func createProductConfigFiles(
}
currentProductFolder := fmt.Sprintf("build/bazel/products/%s", targetProduct)
if len(productVariables.PartitionVars.ProductDirectory) > 0 {
currentProductFolder = fmt.Sprintf("%s%s", productVariables.PartitionVars.ProductDirectory, targetProduct)
if len(productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory) > 0 {
currentProductFolder = fmt.Sprintf("%s%s", productVariables.PartitionVarsForBazelMigrationOnlyDoNotUse.ProductDirectory, targetProduct)
}
productReplacer := strings.NewReplacer(
@ -72,14 +82,22 @@ func createProductConfigFiles(
productsForTesting[i] = fmt.Sprintf(" \"@//build/bazel/tests/products:%s\",", productsForTesting[i])
}
productLabelsToVariables := make(map[string]*android.ProductVariables)
productLabelsToVariables[productReplacer.Replace("@//{PRODUCT_FOLDER}:{PRODUCT}")] = &productVariables
productLabelsToVariables := make(map[bazelLabel]*android.ProductVariables)
productLabelsToVariables[bazelLabel{
repo: "",
pkg: currentProductFolder,
target: targetProduct,
}] = &productVariables
for product, productVariablesStarlark := range productsForTestingMap {
productVariables, err := starlarkMapToProductVariables(productVariablesStarlark)
if err != nil {
return res, err
}
productLabelsToVariables["@//build/bazel/tests/products:"+product] = &productVariables
productLabelsToVariables[bazelLabel{
repo: "",
pkg: "build/bazel/tests/products",
target: product,
}] = &productVariables
}
res.bp2buildTargets = make(map[string]BazelTargets)
@ -194,7 +212,7 @@ build --host_platform @//{PRODUCT_FOLDER}:{PRODUCT}_darwin_x86_64
}
func platformMappingContent(
productLabelToVariables map[string]*android.ProductVariables,
productLabelToVariables map[bazelLabel]*android.ProductVariables,
soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
convertedModulePathMap map[string]string) (string, error) {
var result strings.Builder
@ -233,7 +251,7 @@ var bazelPlatformSuffixes = []string{
}
func platformMappingSingleProduct(
label string,
label bazelLabel,
productVariables *android.ProductVariables,
soongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions,
convertedModulePathMap map[string]string,
@ -251,7 +269,7 @@ func platformMappingSingleProduct(
for _, suffix := range bazelPlatformSuffixes {
result.WriteString(" ")
result.WriteString(label)
result.WriteString(label.String())
result.WriteString(suffix)
result.WriteString("\n")
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:always_use_prebuilt_sdks=%t\n", proptools.Bool(productVariables.Always_use_prebuilt_sdks)))
@ -272,7 +290,7 @@ func platformMappingSingleProduct(
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_name=%s\n", proptools.String(productVariables.DeviceName)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_page_size_agnostic=%t\n", proptools.Bool(productVariables.DevicePageSizeAgnostic)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_product=%s\n", proptools.String(productVariables.DeviceProduct)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_platform=%s\n", label))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:device_platform=%s\n", label.String()))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enable_cfi=%t\n", proptools.BoolDefault(productVariables.EnableCFI, true)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:enforce_vintf_manifest=%t\n", proptools.Bool(productVariables.Enforce_vintf_manifest)))
result.WriteString(fmt.Sprintf(" --//build/bazel/product_config:malloc_not_svelte=%t\n", proptools.Bool(productVariables.Malloc_not_svelte)))
@ -422,11 +440,14 @@ func starlarkMapToProductVariables(in map[string]starlark.Value) (android.Produc
return result, nil
}
func createTargets(productLabelsToVariables map[string]*android.ProductVariables, res map[string]BazelTargets) {
func createTargets(productLabelsToVariables map[bazelLabel]*android.ProductVariables, res map[string]BazelTargets) {
createGeneratedAndroidCertificateDirectories(productLabelsToVariables, res)
for label, variables := range productLabelsToVariables {
createSystemPartition(label, &variables.PartitionVarsForBazelMigrationOnlyDoNotUse, res)
}
}
func createGeneratedAndroidCertificateDirectories(productLabelsToVariables map[string]*android.ProductVariables, targets map[string]BazelTargets) {
func createGeneratedAndroidCertificateDirectories(productLabelsToVariables map[bazelLabel]*android.ProductVariables, targets map[string]BazelTargets) {
var allDefaultAppCertificateDirs []string
for _, productVariables := range productLabelsToVariables {
if proptools.String(productVariables.DefaultAppCertificate) != "" {
@ -454,3 +475,267 @@ func createGeneratedAndroidCertificateDirectories(productLabelsToVariables map[s
})
}
}
func createSystemPartition(platformLabel bazelLabel, variables *android.PartitionVariables, targets map[string]BazelTargets) {
if !variables.PartitionQualifiedVariables["system"].BuildingImage {
return
}
imageProps := generateImagePropDictionary(variables, "system")
imageProps["skip_fsck"] = "true"
var properties strings.Builder
for _, prop := range android.SortedKeys(imageProps) {
properties.WriteString(prop)
properties.WriteRune('=')
properties.WriteString(imageProps[prop])
properties.WriteRune('\n')
}
targets[platformLabel.pkg] = append(targets[platformLabel.pkg], BazelTarget{
name: "system_image",
packageName: platformLabel.pkg,
content: fmt.Sprintf(`partition(
name = "system_image",
base_staging_dir = "//build/bazel/bazel_sandwich:system_staging_dir",
base_staging_dir_file_list = "//build/bazel/bazel_sandwich:system_staging_dir_file_list",
root_dir = "//build/bazel/bazel_sandwich:root_staging_dir",
image_properties = """
%s
""",
type = "system",
)`, properties.String()),
ruleClass: "partition",
loads: []BazelLoad{{
file: "//build/bazel/rules/partitions:partition.bzl",
symbols: []BazelLoadSymbol{{
symbol: "partition",
}},
}},
}, BazelTarget{
name: "system_image_test",
packageName: platformLabel.pkg,
content: `partition_diff_test(
name = "system_image_test",
partition1 = "//build/bazel/bazel_sandwich:make_system_image",
partition2 = ":system_image",
)`,
ruleClass: "partition_diff_test",
loads: []BazelLoad{{
file: "//build/bazel/rules/partitions/diff:partition_diff.bzl",
symbols: []BazelLoadSymbol{{
symbol: "partition_diff_test",
}},
}},
}, BazelTarget{
name: "run_system_image_test",
packageName: platformLabel.pkg,
content: `run_test_in_build(
name = "run_system_image_test",
test = ":system_image_test",
)`,
ruleClass: "run_test_in_build",
loads: []BazelLoad{{
file: "//build/bazel/bazel_sandwich:run_test_in_build.bzl",
symbols: []BazelLoadSymbol{{
symbol: "run_test_in_build",
}},
}},
})
}
var allPartitionTypes = []string{
"system",
"vendor",
"cache",
"userdata",
"product",
"system_ext",
"oem",
"odm",
"vendor_dlkm",
"odm_dlkm",
"system_dlkm",
}
// An equivalent of make's generate-image-prop-dictionary function
func generateImagePropDictionary(variables *android.PartitionVariables, partitionType string) map[string]string {
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
ret := map[string]string{}
if partitionType == "system" {
if len(variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize) > 0 {
ret["system_other_size"] = variables.PartitionQualifiedVariables["system_other"].BoardPartitionSize
}
if len(partitionQualifiedVariables.ProductHeadroom) > 0 {
ret["system_headroom"] = partitionQualifiedVariables.ProductHeadroom
}
addCommonRoFlagsToImageProps(variables, partitionType, ret)
}
// TODO: other partition-specific logic
if variables.TargetUserimagesUseExt2 {
ret["fs_type"] = "ext2"
} else if variables.TargetUserimagesUseExt3 {
ret["fs_type"] = "ext3"
} else if variables.TargetUserimagesUseExt4 {
ret["fs_type"] = "ext4"
}
if !variables.TargetUserimagesSparseExtDisabled {
ret["extfs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseErofsDisabled {
ret["erofs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseSquashfsDisabled {
ret["squashfs_sparse_flag"] = "-s"
}
if !variables.TargetUserimagesSparseF2fsDisabled {
ret["f2fs_sparse_flag"] = "-S"
}
erofsCompressor := variables.BoardErofsCompressor
if len(erofsCompressor) == 0 && hasErofsPartition(variables) {
if len(variables.BoardErofsUseLegacyCompression) > 0 {
erofsCompressor = "lz4"
} else {
erofsCompressor = "lz4hc,9"
}
}
if len(erofsCompressor) > 0 {
ret["erofs_default_compressor"] = erofsCompressor
}
if len(variables.BoardErofsCompressorHints) > 0 {
ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints
}
if len(variables.BoardErofsCompressorHints) > 0 {
ret["erofs_default_compress_hints"] = variables.BoardErofsCompressorHints
}
if len(variables.BoardErofsPclusterSize) > 0 {
ret["erofs_pcluster_size"] = variables.BoardErofsPclusterSize
}
if len(variables.BoardErofsShareDupBlocks) > 0 {
ret["erofs_share_dup_blocks"] = variables.BoardErofsShareDupBlocks
}
if len(variables.BoardErofsUseLegacyCompression) > 0 {
ret["erofs_use_legacy_compression"] = variables.BoardErofsUseLegacyCompression
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
if len(variables.BoardFlashLogicalBlockSize) > 0 {
ret["flash_logical_block_size"] = variables.BoardFlashLogicalBlockSize
}
if len(variables.BoardFlashEraseBlockSize) > 0 {
ret["flash_erase_block_size"] = variables.BoardFlashEraseBlockSize
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
if len(variables.BoardExt4ShareDupBlocks) > 0 {
ret["ext4_share_dup_blocks"] = variables.BoardExt4ShareDupBlocks
}
for _, partitionType := range allPartitionTypes {
if qualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]; ok && len(qualifiedVariables.ProductVerityPartition) > 0 {
ret[partitionType+"_verity_block_device"] = qualifiedVariables.ProductVerityPartition
}
}
// TODO: Vboot
// TODO: AVB
if variables.BoardUsesRecoveryAsBoot {
ret["recovery_as_boot"] = "true"
}
if variables.BoardBuildGkiBootImageWithoutRamdisk {
ret["gki_boot_image_without_ramdisk"] = "true"
}
if variables.ProductUseDynamicPartitionSize {
ret["use_dynamic_partition_size"] = "true"
}
if variables.CopyImagesForTargetFilesZip {
ret["use_fixed_timestamp"] = "true"
}
return ret
}
// Soong equivalent of make's add-common-ro-flags-to-image-props
func addCommonRoFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) {
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
if len(partitionQualifiedVariables.BoardErofsCompressor) > 0 {
ret[partitionType+"_erofs_compressor"] = partitionQualifiedVariables.BoardErofsCompressor
}
if len(partitionQualifiedVariables.BoardErofsCompressHints) > 0 {
ret[partitionType+"_erofs_compress_hints"] = partitionQualifiedVariables.BoardErofsCompressHints
}
if len(partitionQualifiedVariables.BoardErofsPclusterSize) > 0 {
ret[partitionType+"_erofs_pcluster_size"] = partitionQualifiedVariables.BoardErofsPclusterSize
}
if len(partitionQualifiedVariables.BoardExtfsRsvPct) > 0 {
ret[partitionType+"_extfs_rsv_pct"] = partitionQualifiedVariables.BoardExtfsRsvPct
}
if len(partitionQualifiedVariables.BoardF2fsSloadCompressFlags) > 0 {
ret[partitionType+"_f2fs_sldc_flags"] = partitionQualifiedVariables.BoardF2fsSloadCompressFlags
}
if len(partitionQualifiedVariables.BoardFileSystemCompress) > 0 {
ret[partitionType+"_f2fs_compress"] = partitionQualifiedVariables.BoardFileSystemCompress
}
if len(partitionQualifiedVariables.BoardFileSystemType) > 0 {
ret[partitionType+"_fs_type"] = partitionQualifiedVariables.BoardFileSystemType
}
if len(partitionQualifiedVariables.BoardJournalSize) > 0 {
ret[partitionType+"_journal_size"] = partitionQualifiedVariables.BoardJournalSize
}
if len(partitionQualifiedVariables.BoardPartitionReservedSize) > 0 {
ret[partitionType+"_reserved_size"] = partitionQualifiedVariables.BoardPartitionReservedSize
}
if len(partitionQualifiedVariables.BoardPartitionSize) > 0 {
ret[partitionType+"_size"] = partitionQualifiedVariables.BoardPartitionSize
}
if len(partitionQualifiedVariables.BoardSquashfsBlockSize) > 0 {
ret[partitionType+"_squashfs_block_size"] = partitionQualifiedVariables.BoardSquashfsBlockSize
}
if len(partitionQualifiedVariables.BoardSquashfsCompressor) > 0 {
ret[partitionType+"_squashfs_compressor"] = partitionQualifiedVariables.BoardSquashfsCompressor
}
if len(partitionQualifiedVariables.BoardSquashfsCompressorOpt) > 0 {
ret[partitionType+"_squashfs_compressor_opt"] = partitionQualifiedVariables.BoardSquashfsCompressorOpt
}
if len(partitionQualifiedVariables.BoardSquashfsDisable4kAlign) > 0 {
ret[partitionType+"_squashfs_disable_4k_align"] = partitionQualifiedVariables.BoardSquashfsDisable4kAlign
}
if len(partitionQualifiedVariables.BoardPartitionSize) == 0 && len(partitionQualifiedVariables.BoardPartitionReservedSize) == 0 && len(partitionQualifiedVariables.ProductHeadroom) == 0 {
ret[partitionType+"_disable_sparse"] = "true"
}
addCommonFlagsToImageProps(variables, partitionType, ret)
}
func hasErofsPartition(variables *android.PartitionVariables) bool {
return variables.PartitionQualifiedVariables["product"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system_ext"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["odm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["vendor"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["vendor_dlkm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["odm_dlkm"].BoardFileSystemType == "erofs" ||
variables.PartitionQualifiedVariables["system_dlkm"].BoardFileSystemType == "erofs"
}
// Soong equivalent of make's add-common-flags-to-image-props
func addCommonFlagsToImageProps(variables *android.PartitionVariables, partitionType string, ret map[string]string) {
// The selinux_fc will be handled separately
partitionQualifiedVariables, ok := variables.PartitionQualifiedVariables[partitionType]
if !ok {
panic("Unknown partitionType: " + partitionType)
}
ret["building_"+partitionType+"_image"] = boolToMakeString(partitionQualifiedVariables.BuildingImage)
}
func boolToMakeString(b bool) string {
if b {
return "true"
}
return ""
}