platform_build_soong/android/rule_builder.go
Dan Willemsen 633c502295 Support RuleBuilder.Sbox to wrap commands in sbox
This essentially allows you to declare that everything in a directory
will be created by the rule, and we'll ensure that your command actually
writes out all of the claimed outputs, and remove any other files that
previously existed in that directory.

Test: built-in tests
Change-Id: I990dce2b3a0d89ebd2736ac1a0cadfb5864c6e73
2019-04-15 14:52:05 -07:00

624 lines
20 KiB
Go

// Copyright 2018 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"fmt"
"sort"
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
"android/soong/shared"
)
// RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
// graph.
type RuleBuilder struct {
commands []*RuleBuilderCommand
installs RuleBuilderInstalls
temporariesSet map[WritablePath]bool
restat bool
sbox bool
sboxOutDir WritablePath
missingDeps []string
}
// NewRuleBuilder returns a newly created RuleBuilder.
func NewRuleBuilder() *RuleBuilder {
return &RuleBuilder{
temporariesSet: make(map[WritablePath]bool),
}
}
// RuleBuilderInstall is a tuple of install from and to locations.
type RuleBuilderInstall struct {
From Path
To string
}
type RuleBuilderInstalls []RuleBuilderInstall
// String returns the RuleBuilderInstalls in the form used by $(call copy-many-files) in Make, a space separated
// list of from:to tuples.
func (installs RuleBuilderInstalls) String() string {
sb := strings.Builder{}
for i, install := range installs {
if i != 0 {
sb.WriteRune(' ')
}
sb.WriteString(install.From.String())
sb.WriteRune(':')
sb.WriteString(install.To)
}
return sb.String()
}
// MissingDeps adds modules to the list of missing dependencies. If MissingDeps
// is called with a non-empty input, any call to Build will result in a rule
// that will print an error listing the missing dependencies and fail.
// MissingDeps should only be called if Config.AllowMissingDependencies() is
// true.
func (r *RuleBuilder) MissingDeps(missingDeps []string) {
r.missingDeps = append(r.missingDeps, missingDeps...)
}
// Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
//
// Restat is not compatible with Sbox()
func (r *RuleBuilder) Restat() *RuleBuilder {
if r.sbox {
panic("Restat() is not compatible with Sbox()")
}
r.restat = true
return r
}
// Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output
// directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure
// that all outputs have been written, and will discard any output files that were not specified.
//
// Sbox is not compatible with Restat()
func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder {
if r.sbox {
panic("Sbox() may not be called more than once")
}
if len(r.commands) > 0 {
panic("Sbox() may not be called after Command()")
}
if r.restat {
panic("Sbox() is not compatible with Restat()")
}
r.sbox = true
r.sboxOutDir = outputDir
return r
}
// Install associates an output of the rule with an install location, which can be retrieved later using
// RuleBuilder.Installs.
func (r *RuleBuilder) Install(from Path, to string) {
r.installs = append(r.installs, RuleBuilderInstall{from, to})
}
// Command returns a new RuleBuilderCommand for the rule. The commands will be ordered in the rule by when they were
// created by this method. That can be mutated through their methods in any order, as long as the mutations do not
// race with any call to Build.
func (r *RuleBuilder) Command() *RuleBuilderCommand {
command := &RuleBuilderCommand{
sbox: r.sbox,
sboxOutDir: r.sboxOutDir,
}
r.commands = append(r.commands, command)
return command
}
// Temporary marks an output of a command as an intermediate file that will be used as an input to another command
// in the same rule, and should not be listed in Outputs.
func (r *RuleBuilder) Temporary(path WritablePath) {
r.temporariesSet[path] = true
}
// DeleteTemporaryFiles adds a command to the rule that deletes any outputs that have been marked using Temporary
// when the rule runs. DeleteTemporaryFiles should be called after all calls to Temporary.
func (r *RuleBuilder) DeleteTemporaryFiles() {
var temporariesList WritablePaths
for intermediate := range r.temporariesSet {
temporariesList = append(temporariesList, intermediate)
}
sort.Slice(temporariesList, func(i, j int) bool {
return temporariesList[i].String() < temporariesList[j].String()
})
r.Command().Text("rm").Flag("-f").Outputs(temporariesList)
}
// Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take input paths, such
// as RuleBuilderCommand.Input, RuleBuilderComand.Implicit, or RuleBuilderCommand.FlagWithInput. Inputs to a command
// that are also outputs of another command in the same RuleBuilder are filtered out.
func (r *RuleBuilder) Inputs() Paths {
outputs := r.outputSet()
depFiles := r.depFileSet()
inputs := make(map[string]Path)
for _, c := range r.commands {
for _, input := range c.inputs {
inputStr := input.String()
if _, isOutput := outputs[inputStr]; !isOutput {
if _, isDepFile := depFiles[inputStr]; !isDepFile {
inputs[input.String()] = input
}
}
}
}
var inputList Paths
for _, input := range inputs {
inputList = append(inputList, input)
}
sort.Slice(inputList, func(i, j int) bool {
return inputList[i].String() < inputList[j].String()
})
return inputList
}
func (r *RuleBuilder) outputSet() map[string]WritablePath {
outputs := make(map[string]WritablePath)
for _, c := range r.commands {
for _, output := range c.outputs {
outputs[output.String()] = output
}
}
return outputs
}
// Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take output paths, such
// as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or RuleBuilderCommand.FlagWithInput.
func (r *RuleBuilder) Outputs() WritablePaths {
outputs := r.outputSet()
var outputList WritablePaths
for _, output := range outputs {
if !r.temporariesSet[output] {
outputList = append(outputList, output)
}
}
sort.Slice(outputList, func(i, j int) bool {
return outputList[i].String() < outputList[j].String()
})
return outputList
}
func (r *RuleBuilder) depFileSet() map[string]WritablePath {
depFiles := make(map[string]WritablePath)
for _, c := range r.commands {
for _, depFile := range c.depFiles {
depFiles[depFile.String()] = depFile
}
}
return depFiles
}
// DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such
// as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile.
func (r *RuleBuilder) DepFiles() WritablePaths {
var depFiles WritablePaths
for _, c := range r.commands {
for _, depFile := range c.depFiles {
depFiles = append(depFiles, depFile)
}
}
return depFiles
}
// Installs returns the list of tuples passed to Install.
func (r *RuleBuilder) Installs() RuleBuilderInstalls {
return append(RuleBuilderInstalls(nil), r.installs...)
}
func (r *RuleBuilder) toolsSet() map[string]Path {
tools := make(map[string]Path)
for _, c := range r.commands {
for _, tool := range c.tools {
tools[tool.String()] = tool
}
}
return tools
}
// Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method.
func (r *RuleBuilder) Tools() Paths {
toolsSet := r.toolsSet()
var toolsList Paths
for _, tool := range toolsSet {
toolsList = append(toolsList, tool)
}
sort.Slice(toolsList, func(i, j int) bool {
return toolsList[i].String() < toolsList[j].String()
})
return toolsList
}
// Commands returns a slice containing a the built command line for each call to RuleBuilder.Command.
func (r *RuleBuilder) Commands() []string {
var commands []string
for _, c := range r.commands {
commands = append(commands, string(c.buf))
}
return commands
}
// BuilderContext is a subset of ModuleContext and SingletonContext.
type BuilderContext interface {
PathContext
Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule
Build(PackageContext, BuildParams)
}
var _ BuilderContext = ModuleContext(nil)
var _ BuilderContext = SingletonContext(nil)
func (r *RuleBuilder) depFileMergerCmd(ctx PathContext, depFiles WritablePaths) *RuleBuilderCommand {
return r.Command().
Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).
Inputs(depFiles.Paths())
}
// Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
// Outputs.
func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string, desc string) {
name = ninjaNameEscape(name)
if len(r.missingDeps) > 0 {
ctx.Build(pctx, BuildParams{
Rule: ErrorRule,
Outputs: r.Outputs(),
Description: desc,
Args: map[string]string{
"error": "missing dependencies: " + strings.Join(r.missingDeps, ", "),
},
})
return
}
var depFile WritablePath
var depFormat blueprint.Deps
if depFiles := r.DepFiles(); len(depFiles) > 0 {
depFile = depFiles[0]
depFormat = blueprint.DepsGCC
if len(depFiles) > 1 {
// Add a command locally that merges all depfiles together into the first depfile.
r.depFileMergerCmd(ctx, depFiles)
if r.sbox {
// Check for Rel() errors, as all depfiles should be in the output dir
for _, path := range depFiles[1:] {
Rel(ctx, r.sboxOutDir.String(), path.String())
}
}
}
}
tools := r.Tools()
commands := r.Commands()
outputs := r.Outputs()
if len(commands) == 0 {
return
}
if len(outputs) == 0 {
panic("No outputs specified from any Commands")
}
commandString := strings.Join(proptools.NinjaEscapeList(commands), " && ")
if r.sbox {
sboxOutputs := make([]string, len(outputs))
for i, output := range outputs {
sboxOutputs[i] = "__SBOX_OUT_DIR__/" + Rel(ctx, r.sboxOutDir.String(), output.String())
}
if depFile != nil {
sboxOutputs = append(sboxOutputs, "__SBOX_OUT_DIR__/"+Rel(ctx, r.sboxOutDir.String(), depFile.String()))
}
commandString = proptools.ShellEscape(commandString)
if !strings.HasPrefix(commandString, `'`) {
commandString = `'` + commandString + `'`
}
sboxCmd := &RuleBuilderCommand{}
sboxCmd.Tool(ctx.Config().HostToolPath(ctx, "sbox")).
Flag("-c").Text(commandString).
Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
Flag("--output-root").Text(r.sboxOutDir.String()).
Flags(sboxOutputs)
commandString = string(sboxCmd.buf)
tools = append(tools, sboxCmd.tools...)
}
// Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
// ImplicitOutputs. RuleBuilder never uses "$out", so the distinction between Outputs and ImplicitOutputs
// doesn't matter.
output := outputs[0]
implicitOutputs := outputs[1:]
ctx.Build(pctx, BuildParams{
Rule: ctx.Rule(pctx, name, blueprint.RuleParams{
Command: commandString,
CommandDeps: tools.Strings(),
Restat: r.restat,
}),
Implicits: r.Inputs(),
Output: output,
ImplicitOutputs: implicitOutputs,
Depfile: depFile,
Deps: depFormat,
Description: desc,
})
}
// RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the
// command and track dependencies. The methods mutate the RuleBuilderCommand in place, as well as return the
// RuleBuilderCommand, so they can be used chained or unchained. All methods that add text implicitly add a single
// space as a separator from the previous method.
type RuleBuilderCommand struct {
buf []byte
inputs Paths
outputs WritablePaths
depFiles WritablePaths
tools Paths
sbox bool
sboxOutDir WritablePath
}
func (c *RuleBuilderCommand) addInput(path Path) string {
if c.sbox {
if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel {
return "__SBOX_OUT_DIR__/" + rel
}
}
c.inputs = append(c.inputs, path)
return path.String()
}
func (c *RuleBuilderCommand) outputStr(path Path) string {
if c.sbox {
// Errors will be handled in RuleBuilder.Build where we have a context to report them
rel, _, _ := maybeRelErr(c.sboxOutDir.String(), path.String())
return "__SBOX_OUT_DIR__/" + rel
}
return path.String()
}
// Text adds the specified raw text to the command line. The text should not contain input or output paths or the
// rule will not have them listed in its dependencies or outputs.
func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand {
if len(c.buf) > 0 {
c.buf = append(c.buf, ' ')
}
c.buf = append(c.buf, text...)
return c
}
// Textf adds the specified formatted text to the command line. The text should not contain input or output paths or
// the rule will not have them listed in its dependencies or outputs.
func (c *RuleBuilderCommand) Textf(format string, a ...interface{}) *RuleBuilderCommand {
return c.Text(fmt.Sprintf(format, a...))
}
// Flag adds the specified raw text to the command line. The text should not contain input or output paths or the
// rule will not have them listed in its dependencies or outputs.
func (c *RuleBuilderCommand) Flag(flag string) *RuleBuilderCommand {
return c.Text(flag)
}
// Flags adds the specified raw text to the command line. The text should not contain input or output paths or the
// rule will not have them listed in its dependencies or outputs.
func (c *RuleBuilderCommand) Flags(flags []string) *RuleBuilderCommand {
for _, flag := range flags {
c.Text(flag)
}
return c
}
// FlagWithArg adds the specified flag and argument text to the command line, with no separator between them. The flag
// and argument should not contain input or output paths or the rule will not have them listed in its dependencies or
// outputs.
func (c *RuleBuilderCommand) FlagWithArg(flag, arg string) *RuleBuilderCommand {
return c.Text(flag + arg)
}
// FlagForEachArg adds the specified flag joined with each argument to the command line. The result is identical to
// calling FlagWithArg for argument.
func (c *RuleBuilderCommand) FlagForEachArg(flag string, args []string) *RuleBuilderCommand {
for _, arg := range args {
c.FlagWithArg(flag, arg)
}
return c
}
// FlagWithList adds the specified flag and list of arguments to the command line, with the arguments joined by sep
// and no separator between the flag and arguments. The flag and arguments should not contain input or output paths or
// the rule will not have them listed in its dependencies or outputs.
func (c *RuleBuilderCommand) FlagWithList(flag string, list []string, sep string) *RuleBuilderCommand {
return c.Text(flag + strings.Join(list, sep))
}
// Tool adds the specified tool path to the command line. The path will be also added to the dependencies returned by
// RuleBuilder.Tools.
func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand {
c.tools = append(c.tools, path)
return c.Text(path.String())
}
// Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
// RuleBuilder.Inputs.
func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand {
return c.Text(c.addInput(path))
}
// Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the
// dependencies returned by RuleBuilder.Inputs.
func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand {
for _, path := range paths {
c.Input(path)
}
return c
}
// Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the
// command line.
func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand {
c.addInput(path)
return c
}
// Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the
// command line.
func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand {
for _, path := range paths {
c.addInput(path)
}
return c
}
// Output adds the specified output path to the command line. The path will also be added to the outputs returned by
// RuleBuilder.Outputs.
func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand {
c.outputs = append(c.outputs, path)
return c.Text(c.outputStr(path))
}
// Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to
// the outputs returned by RuleBuilder.Outputs.
func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand {
for _, path := range paths {
c.Output(path)
}
return c
}
// DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command
// line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles are added to
// commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together.
func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand {
c.depFiles = append(c.depFiles, path)
return c.Text(c.outputStr(path))
}
// ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying
// the command line.
func (c *RuleBuilderCommand) ImplicitOutput(path WritablePath) *RuleBuilderCommand {
c.outputs = append(c.outputs, path)
return c
}
// ImplicitOutputs adds the specified output paths to the dependencies returned by RuleBuilder.Outputs without modifying
// the command line.
func (c *RuleBuilderCommand) ImplicitOutputs(paths WritablePaths) *RuleBuilderCommand {
c.outputs = append(c.outputs, paths...)
return c
}
// ImplicitDepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles without modifying
// the command line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles
// are added to commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the
// depfiles together.
func (c *RuleBuilderCommand) ImplicitDepFile(path WritablePath) *RuleBuilderCommand {
c.depFiles = append(c.depFiles, path)
return c
}
// FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path
// will also be added to the dependencies returned by RuleBuilder.Inputs.
func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand {
return c.Text(flag + c.addInput(path))
}
// FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep
// and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by
// RuleBuilder.Inputs.
func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand {
strs := make([]string, len(paths))
for i, path := range paths {
strs[i] = c.addInput(path)
}
return c.FlagWithList(flag, strs, sep)
}
// FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also
// be added to the dependencies returned by RuleBuilder.Inputs. The result is identical to calling FlagWithInput for
// each input path.
func (c *RuleBuilderCommand) FlagForEachInput(flag string, paths Paths) *RuleBuilderCommand {
for _, path := range paths {
c.FlagWithInput(flag, path)
}
return c
}
// FlagWithOutput adds the specified flag and output path to the command line, with no separator between them. The path
// will also be added to the outputs returned by RuleBuilder.Outputs.
func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand {
c.outputs = append(c.outputs, path)
return c.Text(flag + c.outputStr(path))
}
// FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path
// will also be added to the outputs returned by RuleBuilder.Outputs.
func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand {
c.depFiles = append(c.depFiles, path)
return c.Text(flag + c.outputStr(path))
}
// String returns the command line.
func (c *RuleBuilderCommand) String() string {
return string(c.buf)
}
func ninjaNameEscape(s string) string {
b := []byte(s)
escaped := false
for i, c := range b {
valid := (c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
(c == '_') ||
(c == '-') ||
(c == '.')
if !valid {
b[i] = '_'
escaped = true
}
}
if escaped {
s = string(b)
}
return s
}