tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Allow choosing which soong_build invocations to run in the debugger

Browse source 
When `SOONG_DELVE_STEPS` enviromnment variable is set in addition
to `SOONG_DELVE`, only given soong_build invocations ('steps') are
run in the debugger.
Also, make bootstrapBlueprint code table-driven.

Test: treehugger
Change-Id: Ia4016240ca9e88c2a85c6d06851c5bba30e7b2b5
This commit is contained in:
Sasha Smundak 2022-11-28 17:02:40 -08:00
parent 5c4729df93
commit 4cbe83ad5e
2 changed files with 175 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
README.md
View file

@ -609,15 +609,15 @@ To load the code of Soong in IntelliJ:
Content Root, then add the `build/blueprint` directory.
* Optional: also add the `external/golang-protobuf` directory. In practice,
IntelliJ seems to work well enough without this, too.
### Running Soong in a debugger
To make `soong_build` wait for a debugger connection, install `dlv` and then
start the build with `SOONG_DELVE=<listen addr>` in the environment.
For example:
```bash
SOONG_DELVE=5006 m nothing
```
Both the Android build driver (`soong_ui`) and Soong proper (`soong_build`) are
Go applications and can be debugged with the help of the standard Go debugger
called Delve. A client (e.g., IntelliJ IDEA) communicates with Delve via IP port
that Delve listens to (the port number is passed to it on invocation).
#### Debugging Android Build Driver ####
To make `soong_ui` wait for a debugger connection, use the `SOONG_UI_DELVE`
variable:
@ -625,11 +625,28 @@ variable:
SOONG_UI_DELVE=5006 m nothing
```
#### Debugging Soong Proper ####
setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
To make `soong_build` wait for a debugger connection, install `dlv` and then
start the build with `SOONG_DELVE=<listen addr>` in the environment.
For example:
```bash
SOONG_DELVE=5006 m nothing
```
Android build driver invokes `soong_build` multiple times, and by default each
invocation is run in the debugger. Setting `SOONG_DELVE_STEPS` controls which
invocations are run in the debugger, e.g., running
```bash
SOONG_DELVE=2345 SOONG_DELVE_STEPS='build,modulegraph' m
```
results in only `build` (main build step) and `modulegraph` being run in the debugger.
The allowed step names are `api_bp2build`, `bp2build_files`, `bp2build_workspace`,
`build`, `modulegraph`, `queryview`, `soong_docs`.
Note setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
is because in order to debug the binary, it needs to be built with debug
symbols.
#### Delve Troubleshooting ####
To test the debugger connection, run this command:
```
@ -648,15 +665,23 @@ using:
sudo sysctl -w kernel.yama.ptrace_scope=0
```
#### IntelliJ Setup ####
To connect to the process using IntelliJ:
* Run -> Edit Configurations...
* Choose "Go Remote" on the left
* Click on the "+" buttion on the top-left
* Give it a nice name and set "Host" to localhost and "Port" to the port in the
environment variable
* Give it a nice _name_ and set "Host" to `localhost` and "Port" to the port in the
environment variable (`SOONG_UI_DELVE` for `soong_ui`, `SOONG_DELVE` for
`soong_build`)
* Set the breakpoints where you want application to stop
* Run the build from the command line
* In IntelliJ, click Run -> Debug _name_
* Observe _Connecting..._ message in the debugger pane. It changes to
_Connected_ once the communication with the debugger has been established; the
terminal window where the build started will display
`API server listening at ...` message
Debugging works far worse than debugging Java, but is sometimes useful.
Sometimes the `dlv` process hangs on connection. A symptom of this is `dlv`
spinning a core or two. In that case, `kill -9` `dlv` and try again.

255
ui/build/soong.go
View file

@ -18,7 +18,6 @@ import (
"errors"
"fmt"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"strconv"
@ -56,13 +55,13 @@ const (
bootstrapEpoch = 1
)
func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
data, err := shared.EnvFileContents(envDeps)
if err != nil {
return err
}
return ioutil.WriteFile(envFile, data, 0644)
return os.WriteFile(envFile, data, 0644)
}
// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
@ -141,7 +140,7 @@ func writeEmptyFile(ctx Context, path string) {
if exists, err := fileExists(path); err != nil {
ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
} else if !exists {
err = ioutil.WriteFile(path, nil, 0666)
err = os.WriteFile(path, nil, 0666)
if err != nil {
ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
}
@ -157,24 +156,28 @@ func fileExists(path string) (bool, error) {
return true, nil
}
func primaryBuilderInvocation(
config Config,
name string,
output string,
specificArgs []string,
description string) bootstrap.PrimaryBuilderInvocation {
type PrimaryBuilderFactory struct {
name string
description string
config Config
output string
specificArgs []string
debugPort string
}
func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
commonArgs := make([]string, 0, 0)
if !config.skipSoongTests {
if !pb.config.skipSoongTests {
commonArgs = append(commonArgs, "-t")
}
commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list"))
commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
invocationEnv := make(map[string]string)
if os.Getenv("SOONG_DELVE") != "" {
if pb.debugPort != "" {
//debug mode
commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE"))
commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary())
commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
"--delve_path", shared.ResolveDelveBinary())
// GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
// is useful because the preemption happens by sending SIGURG to the OS
// thread hosting the goroutine in question and each signal results in
@ -188,26 +191,26 @@ func primaryBuilderInvocation(
}
var allArgs []string
allArgs = append(allArgs, specificArgs...)
allArgs = append(allArgs, pb.specificArgs...)
allArgs = append(allArgs,
"--globListDir", name,
"--globFile", config.NamedGlobFile(name))
"--globListDir", pb.name,
"--globFile", pb.config.NamedGlobFile(pb.name))
allArgs = append(allArgs, commonArgs...)
allArgs = append(allArgs, environmentArgs(config, name)...)
allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+name)
allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
}
if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
allArgs = append(allArgs, "--memprofile", profileMem+"."+name)
allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
}
allArgs = append(allArgs, "Android.bp")
return bootstrap.PrimaryBuilderInvocation{
Inputs: []string{"Android.bp"},
Outputs: []string{output},
Outputs: []string{pb.output},
Args: allArgs,
Description: description,
Description: pb.description,
// NB: Changing the value of this environment variable will not result in a
// rebuild. The bootstrap Ninja file will change, but apparently Ninja does
// not consider changing the pool specified in a statement a change that's
@ -270,90 +273,117 @@ func bootstrapBlueprint(ctx Context, config Config) {
if config.bazelStagingMode {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
}
mainSoongBuildInvocation := primaryBuilderInvocation(
config,
soongBuildTag,
config.SoongNinjaFile(),
mainSoongBuildExtraArgs,
fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
)
if config.BazelBuildEnabled() {
// Mixed builds call Bazel from soong_build and they therefore need the
// Bazel workspace to be available. Make that so by adding a dependency on
// the bp2build marker file to the action that invokes soong_build .
mainSoongBuildInvocation.OrderOnlyInputs = append(mainSoongBuildInvocation.OrderOnlyInputs,
config.Bp2BuildWorkspaceMarkerFile())
}
bp2buildInvocation := primaryBuilderInvocation(
config,
bp2buildFilesTag,
config.Bp2BuildFilesMarkerFile(),
[]string{
"--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
},
fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
)
bp2buildWorkspaceInvocation := primaryBuilderInvocation(
config,
bp2buildWorkspaceTag,
config.Bp2BuildWorkspaceMarkerFile(),
[]string{
"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
},
fmt.Sprintf("Creating Bazel symlink forest"),
)
bp2buildWorkspaceInvocation.Inputs = append(bp2buildWorkspaceInvocation.Inputs,
config.Bp2BuildFilesMarkerFile(), filepath.Join(config.FileListDir(), "bazel.list"))
jsonModuleGraphInvocation := primaryBuilderInvocation(
config,
jsonModuleGraphTag,
config.ModuleGraphFile(),
[]string{
"--module_graph_file", config.ModuleGraphFile(),
"--module_actions_file", config.ModuleActionsFile(),
},
fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
)
queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
queryviewInvocation := primaryBuilderInvocation(
config,
queryviewTag,
config.QueryviewMarkerFile(),
[]string{
"--bazel_queryview_dir", queryviewDir,
},
fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
)
// The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
// The final workspace will be generated in out/soong/api_bp2build
apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
apiBp2buildInvocation := primaryBuilderInvocation(
config,
apiBp2buildTag,
config.ApiBp2buildMarkerFile(),
[]string{
"--bazel_api_bp2build_dir", apiBp2buildDir,
},
fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
)
soongDocsInvocation := primaryBuilderInvocation(
config,
soongDocsTag,
config.SoongDocsHtml(),
[]string{
"--soong_docs", config.SoongDocsHtml(),
pbfs := []PrimaryBuilderFactory{
{
name: soongBuildTag,
description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
config: config,
output: config.SoongNinjaFile(),
specificArgs: mainSoongBuildExtraArgs,
},
fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
)
{
name: bp2buildFilesTag,
description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
config: config,
output: config.Bp2BuildFilesMarkerFile(),
specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()},
},
{
name: bp2buildWorkspaceTag,
description: "Creating Bazel symlink forest",
config: config,
output: config.Bp2BuildWorkspaceMarkerFile(),
specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()},
},
{
name: jsonModuleGraphTag,
description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
config: config,
output: config.ModuleGraphFile(),
specificArgs: []string{
"--module_graph_file", config.ModuleGraphFile(),
"--module_actions_file", config.ModuleActionsFile(),
},
},
{
name: queryviewTag,
description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
config: config,
output: config.QueryviewMarkerFile(),
specificArgs: []string{"--bazel_queryview_dir", queryviewDir},
},
{
name: apiBp2buildTag,
description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
config: config,
output: config.ApiBp2buildMarkerFile(),
specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir},
},
{
name: soongDocsTag,
description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
config: config,
output: config.SoongDocsHtml(),
specificArgs: []string{"--soong_docs", config.SoongDocsHtml()},
},
}
// Figure out which invocations will be run under the debugger:
// * SOONG_DELVE if set specifies listening port
// * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
debuggedInvocations := make(map[string]bool)
delvePort := os.Getenv("SOONG_DELVE")
if delvePort != "" {
if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
var validSteps []string
for _, pbf := range pbfs {
debuggedInvocations[pbf.name] = false
validSteps = append(validSteps, pbf.name)
}
for _, step := range strings.Split(steps, ",") {
if _, ok := debuggedInvocations[step]; ok {
debuggedInvocations[step] = true
} else {
ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
"Valid steps are %v", step, validSteps)
}
}
} else {
// SOONG_DELVE_STEPS is not set, run all steps in the debugger
for _, pbf := range pbfs {
debuggedInvocations[pbf.name] = true
}
}
}
var invocations []bootstrap.PrimaryBuilderInvocation
for _, pbf := range pbfs {
if debuggedInvocations[pbf.name] {
pbf.debugPort = delvePort
}
pbi := pbf.primaryBuilderInvocation()
// Some invocations require adjustment:
switch pbf.name {
case soongBuildTag:
if config.BazelBuildEnabled() {
// Mixed builds call Bazel from soong_build and they therefore need the
// Bazel workspace to be available. Make that so by adding a dependency on
// the bp2build marker file to the action that invokes soong_build .
pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
}
case bp2buildWorkspaceTag:
pbi.Inputs = append(pbi.Inputs,
config.Bp2BuildFilesMarkerFile(),
filepath.Join(config.FileListDir(), "bazel.list"))
}
invocations = append(invocations, pbi)
}
// The glob .ninja files are subninja'd. However, they are generated during
// the build itself so we write an empty file if the file does not exist yet
@ -362,11 +392,11 @@ func bootstrapBlueprint(ctx Context, config Config) {
writeEmptyFile(ctx, globFile)
}
var blueprintArgs bootstrap.Args
blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja")
blueprintArgs.EmptyNinjaFile = false
blueprintArgs := bootstrap.Args{
ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
EmptyNinjaFile: false,
}
blueprintCtx := blueprint.NewContext()
blueprintCtx.SetIgnoreUnknownModuleTypes(true)
@ -376,16 +406,9 @@ func bootstrapBlueprint(ctx Context, config Config) {
outDir: config.OutDir(),
runGoTests: !config.skipSoongTests,
// If we want to debug soong_build, we need to compile it for debugging
debugCompilation: os.Getenv("SOONG_DELVE") != "",
subninjas: bootstrapGlobFileList(config),
primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{
mainSoongBuildInvocation,
bp2buildInvocation,
bp2buildWorkspaceInvocation,
jsonModuleGraphInvocation,
queryviewInvocation,
apiBp2buildInvocation,
soongDocsInvocation},
debugCompilation: delvePort != "",
subninjas: bootstrapGlobFileList(config),
primaryBuilderInvocations: invocations,
}
// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little