Introduce PackagingBase

PackagingBase is the base struct that implements the basic packaging
functionalities. Here, packaging means grouping dependencies into an
output file where the built artifacts from the dependencies are placed
in a directory-like structure in the output file. The exact format of
the output file is irrelevant; it could be a filesystem image, tar.gz,
zip, or whatever.

PackagingBase is responsible for traversing the dependencies and copying
their outputs under the package root directory, which is expected to be
provided by the module type that includes PackagingBase. Then the
concrete module type is expected to do final step of converting the
package root directory into the output file of specific format.

Bug: 159685774
Bug: 172414391
Test: m dist out/dist/cvd-host_package.tar.gz
Change-Id: I5446eee4834ce3b6f0f5843d93bb330a26d42fe3

android/Android.bp

@@ -76,6 +76,7 @@ bootstrap_go_package {
         "neverallow_test.go",
         "onceper_test.go",
         "package_test.go",
+        "packaging_test.go",
         "path_properties_test.go",
         "paths_test.go",
         "prebuilt_test.go",

android/packaging.go

@@ -14,6 +14,13 @@
 
 package android
 
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/google/blueprint"
+)
+
 // PackagingSpec abstracts a request to place a built artifact at a certain path in a package.
 // A package can be the traditional <partition>.img, but isn't limited to those. Other examples could
 // be a new filesystem image that is a subset of system.img (e.g. for an Android-like mini OS running
@@ -32,3 +39,167 @@ type PackagingSpec struct {
 	// Whether relPathInPackage should be marked as executable or not
 	executable bool
 }
+
+type PackageModule interface {
+	Module
+	packagingBase() *PackagingBase
+
+	// AddDeps adds dependencies to the `deps` modules. This should be called in DepsMutator.
+	AddDeps(ctx BottomUpMutatorContext)
+
+	// CopyDepsToZip zips the built artifacts of the dependencies into the given zip file and
+	// returns zip entries in it.  This is expected to be called in GenerateAndroidBuildActions,
+	// followed by a build rule that unzips it and creates the final output (img, zip, tar.gz,
+	// etc.) from the extracted files
+	CopyDepsToZip(ctx ModuleContext, zipOut OutputPath) []string
+}
+
+// PackagingBase provides basic functionality for packaging dependencies. A module is expected to
+// include this struct and call InitPackageModule.
+type PackagingBase struct {
+	properties PackagingProperties
+
+	// Allows this module to skip missing dependencies. In most cases, this
+	// is not required, but for rare cases like when there's a dependency
+	// to a module which exists in certain repo checkouts, this is needed.
+	IgnoreMissingDependencies bool
+}
+
+type depsProperty struct {
+	// Modules to include in this package
+	Deps []string `android:"arch_variant"`
+}
+
+type packagingMultilibProperties struct {
+	First  depsProperty `android:"arch_variant"`
+	Common depsProperty `android:"arch_variant"`
+	Lib32  depsProperty `android:"arch_variant"`
+	Lib64  depsProperty `android:"arch_variant"`
+}
+
+type PackagingProperties struct {
+	Deps     []string                    `android:"arch_variant"`
+	Multilib packagingMultilibProperties `android:"arch_variant"`
+}
+
+type packagingDependencyTag struct{ blueprint.BaseDependencyTag }
+
+var depTag = packagingDependencyTag{}
+
+func InitPackageModule(p PackageModule) {
+	base := p.packagingBase()
+	p.AddProperties(&base.properties)
+}
+
+func (p *PackagingBase) packagingBase() *PackagingBase {
+	return p
+}
+
+// From deps and multilib.*.deps, select the dependencies that are for the given arch
+// deps is for the current archicture when this module is not configured for multi target.
+// When configured for multi target, deps is selected for each of the targets and is NOT
+// selected for the current architecture which would be Common.
+func (p *PackagingBase) getDepsForArch(ctx BaseModuleContext, arch ArchType) []string {
+	var ret []string
+	if arch == ctx.Target().Arch.ArchType && len(ctx.MultiTargets()) == 0 {
+		ret = append(ret, p.properties.Deps...)
+	} else if arch.Multilib == "lib32" {
+		ret = append(ret, p.properties.Multilib.Lib32.Deps...)
+	} else if arch.Multilib == "lib64" {
+		ret = append(ret, p.properties.Multilib.Lib64.Deps...)
+	} else if arch == Common {
+		ret = append(ret, p.properties.Multilib.Common.Deps...)
+	}
+	for i, t := range ctx.MultiTargets() {
+		if t.Arch.ArchType == arch {
+			ret = append(ret, p.properties.Deps...)
+			if i == 0 {
+				ret = append(ret, p.properties.Multilib.First.Deps...)
+			}
+		}
+	}
+	return FirstUniqueStrings(ret)
+}
+
+func (p *PackagingBase) getSupportedTargets(ctx BaseModuleContext) []Target {
+	var ret []Target
+	// The current and the common OS targets are always supported
+	ret = append(ret, ctx.Target())
+	if ctx.Arch().ArchType != Common {
+		ret = append(ret, Target{Os: ctx.Os(), Arch: Arch{ArchType: Common}})
+	}
+	// If this module is configured for multi targets, those should be supported as well
+	ret = append(ret, ctx.MultiTargets()...)
+	return ret
+}
+
+// See PackageModule.AddDeps
+func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext) {
+	for _, t := range p.getSupportedTargets(ctx) {
+		for _, dep := range p.getDepsForArch(ctx, t.Arch.ArchType) {
+			if p.IgnoreMissingDependencies && !ctx.OtherModuleExists(dep) {
+				continue
+			}
+			ctx.AddFarVariationDependencies(t.Variations(), depTag, dep)
+		}
+	}
+}
+
+// See PackageModule.CopyDepsToZip
+func (p *PackagingBase) CopyDepsToZip(ctx ModuleContext, zipOut OutputPath) (entries []string) {
+	var supportedArches []string
+	for _, t := range p.getSupportedTargets(ctx) {
+		supportedArches = append(supportedArches, t.Arch.ArchType.String())
+	}
+	m := make(map[string]PackagingSpec)
+	ctx.WalkDeps(func(child Module, parent Module) bool {
+		// Don't track modules with unsupported arch
+		// TODO(jiyong): remove this when aosp/1501613 lands.
+		if !InList(child.Target().Arch.ArchType.String(), supportedArches) {
+			return false
+		}
+		for _, ps := range child.PackagingSpecs() {
+			if _, ok := m[ps.relPathInPackage]; !ok {
+				m[ps.relPathInPackage] = ps
+			}
+		}
+		return true
+	})
+
+	builder := NewRuleBuilder()
+
+	dir := PathForModuleOut(ctx, ".zip").OutputPath
+	builder.Command().Text("rm").Flag("-rf").Text(dir.String())
+	builder.Command().Text("mkdir").Flag("-p").Text(dir.String())
+
+	seenDir := make(map[string]bool)
+	for _, k := range SortedStringKeys(m) {
+		ps := m[k]
+		destPath := dir.Join(ctx, ps.relPathInPackage).String()
+		destDir := filepath.Dir(destPath)
+		entries = append(entries, ps.relPathInPackage)
+		if _, ok := seenDir[destDir]; !ok {
+			seenDir[destDir] = true
+			builder.Command().Text("mkdir").Flag("-p").Text(destDir)
+		}
+		if ps.symlinkTarget == "" {
+			builder.Command().Text("cp").Input(ps.srcPath).Text(destPath)
+		} else {
+			builder.Command().Text("ln").Flag("-sf").Text(ps.symlinkTarget).Text(destPath)
+		}
+		if ps.executable {
+			builder.Command().Text("chmod").Flag("a+x").Text(destPath)
+		}
+	}
+
+	builder.Command().
+		BuiltTool(ctx, "soong_zip").
+		FlagWithOutput("-o ", zipOut).
+		FlagWithArg("-C ", dir.String()).
+		Flag("-L 0"). // no compression because this will be unzipped soon
+		FlagWithArg("-D ", dir.String())
+	builder.Command().Text("rm").Flag("-rf").Text(dir.String())
+
+	builder.Build(pctx, ctx, "zip_deps", fmt.Sprintf("Zipping deps for %s", ctx.ModuleName()))
+	return entries
+}

android/packaging_test.go

@@ -0,0 +1,188 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+	"reflect"
+	"testing"
+)
+
+// Module to be packaged
+type componentTestModule struct {
+	ModuleBase
+	props struct {
+		Deps []string
+	}
+}
+
+func componentTestModuleFactory() Module {
+	m := &componentTestModule{}
+	m.AddProperties(&m.props)
+	InitAndroidArchModule(m, HostAndDeviceSupported, MultilibBoth)
+	return m
+}
+
+func (m *componentTestModule) DepsMutator(ctx BottomUpMutatorContext) {
+	ctx.AddDependency(ctx.Module(), nil, m.props.Deps...)
+}
+
+func (m *componentTestModule) GenerateAndroidBuildActions(ctx ModuleContext) {
+	builtFile := PathForModuleOut(ctx, m.Name())
+	dir := ctx.Target().Arch.ArchType.Multilib
+	installDir := PathForModuleInstall(ctx, dir)
+	ctx.InstallFile(installDir, m.Name(), builtFile)
+}
+
+// Module that itself is a package
+type packageTestModule struct {
+	ModuleBase
+	PackagingBase
+
+	entries []string
+}
+
+func packageTestModuleFactory() Module {
+	module := &packageTestModule{}
+	InitPackageModule(module)
+	InitAndroidMultiTargetsArchModule(module, DeviceSupported, MultilibCommon)
+	return module
+}
+
+func (m *packageTestModule) DepsMutator(ctx BottomUpMutatorContext) {
+	m.AddDeps(ctx)
+}
+
+func (m *packageTestModule) GenerateAndroidBuildActions(ctx ModuleContext) {
+	zipFile := PathForModuleOut(ctx, "myzip.zip").OutputPath
+	m.entries = m.CopyDepsToZip(ctx, zipFile)
+}
+
+func runPackagingTest(t *testing.T, bp string, expected []string) {
+	t.Helper()
+
+	config := TestArchConfig(buildDir, nil, bp, nil)
+
+	ctx := NewTestArchContext(config)
+	ctx.RegisterModuleType("component", componentTestModuleFactory)
+	ctx.RegisterModuleType("package_module", packageTestModuleFactory)
+	ctx.Register()
+
+	_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+	FailIfErrored(t, errs)
+	_, errs = ctx.PrepareBuildActions(config)
+	FailIfErrored(t, errs)
+
+	p := ctx.ModuleForTests("package", "android_common").Module().(*packageTestModule)
+	actual := p.entries
+	actual = SortedUniqueStrings(actual)
+	expected = SortedUniqueStrings(expected)
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("\ngot: %v\nexpected: %v\n", actual, expected)
+	}
+}
+
+func TestPackagingBase(t *testing.T) {
+	runPackagingTest(t,
+		`
+		component {
+			name: "foo",
+		}
+
+		package_module {
+			name: "package",
+			deps: ["foo"],
+		}
+		`, []string{"lib64/foo"})
+
+	runPackagingTest(t,
+		`
+		component {
+			name: "foo",
+			deps: ["bar"],
+		}
+
+		component {
+			name: "bar",
+		}
+
+		package_module {
+			name: "package",
+			deps: ["foo"],
+		}
+		`, []string{"lib64/foo", "lib64/bar"})
+
+	runPackagingTest(t,
+		`
+		component {
+			name: "foo",
+			deps: ["bar"],
+		}
+
+		component {
+			name: "bar",
+		}
+
+		package_module {
+			name: "package",
+			deps: ["foo"],
+			compile_multilib: "both",
+		}
+		`, []string{"lib32/foo", "lib32/bar", "lib64/foo", "lib64/bar"})
+
+	runPackagingTest(t,
+		`
+		component {
+			name: "foo",
+		}
+
+		component {
+			name: "bar",
+			compile_multilib: "32",
+		}
+
+		package_module {
+			name: "package",
+			deps: ["foo"],
+			multilib: {
+				lib32: {
+					deps: ["bar"],
+				},
+			},
+			compile_multilib: "both",
+		}
+		`, []string{"lib32/foo", "lib32/bar", "lib64/foo"})
+
+	runPackagingTest(t,
+		`
+		component {
+			name: "foo",
+		}
+
+		component {
+			name: "bar",
+		}
+
+		package_module {
+			name: "package",
+			deps: ["foo"],
+			multilib: {
+				first: {
+					deps: ["bar"],
+				},
+			},
+			compile_multilib: "both",
+		}
+		`, []string{"lib32/foo", "lib64/foo", "lib64/bar"})
+}