Merge changes Icfc893c8,I40f03fc0

* changes:
  Allow debugging with SOONG_DELVE=<listen addr>
  Print ninja stdout live during the build

README.md

@@ -337,6 +337,19 @@ build/soong/scripts/setup_go_workspace_for_soong.sh
 This will bind mount the Soong source directories into the directory in the layout expected by
 the IDE.
 
+### Running Soong in a debugger
+
+To run the soong_build process in a debugger, install `dlv` and then start the build with
+`SOONG_DELVE=<listen addr>` in the environment.
+For examle:
+```bash
+SOONG_DELVE=:1234 m nothing
+```
+and then in another terminal:
+```
+dlv connect :1234
+```
+
 ## Contact
 
 Email android-building@googlegroups.com (external) for any questions, or see

android/env.go

@@ -16,6 +16,7 @@ package android
 
 import (
 	"os"
+	"os/exec"
 	"strings"
 
 	"android/soong/env"
@@ -29,8 +30,16 @@ import (
 // a manifest regeneration.
 
 var originalEnv map[string]string
+var SoongDelveListen string
+var SoongDelvePath string
 
 func init() {
+	// Delve support needs to read this environment variable very early, before NewConfig has created a way to
+	// access originalEnv with dependencies.  Store the value where soong_build can find it, it will manually
+	// ensure the dependencies are created.
+	SoongDelveListen = os.Getenv("SOONG_DELVE")
+	SoongDelvePath, _ = exec.LookPath("dlv")
+
 	originalEnv = make(map[string]string)
 	for _, env := range os.Environ() {
 		idx := strings.IndexRune(env, '=')
@@ -38,6 +47,8 @@ func init() {
 			originalEnv[env[:idx]] = env[idx+1:]
 		}
 	}
+	// Clear the environment to prevent use of os.Getenv(), which would not provide dependencies on environment
+	// variable values.  The environment is available through ctx.Config().Getenv, ctx.Config().IsEnvTrue, etc.
 	os.Clearenv()
 }
 

cmd/soong_build/main.go

@@ -18,7 +18,12 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
 
 	"github.com/google/blueprint/bootstrap"
 
@@ -50,6 +55,42 @@ func newNameResolver(config android.Config) *android.NameResolver {
 }
 
 func main() {
+	if android.SoongDelveListen != "" {
+		if android.SoongDelvePath == "" {
+			fmt.Fprintln(os.Stderr, "SOONG_DELVE is set but failed to find dlv")
+			os.Exit(1)
+		}
+		pid := strconv.Itoa(os.Getpid())
+		cmd := []string{android.SoongDelvePath,
+			"attach", pid,
+			"--headless",
+			"-l", android.SoongDelveListen,
+			"--api-version=2",
+			"--accept-multiclient",
+			"--log",
+		}
+
+		fmt.Println("Starting", strings.Join(cmd, " "))
+		dlv := exec.Command(cmd[0], cmd[1:]...)
+		dlv.Stdout = os.Stdout
+		dlv.Stderr = os.Stderr
+		dlv.Stdin = nil
+
+		// Put dlv into its own process group so we can kill it and the child process it starts.
+		dlv.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+
+		err := dlv.Start()
+		if err != nil {
+			// Print the error starting dlv and continue.
+			fmt.Println(err)
+		} else {
+			// Kill the process group for dlv when soong_build exits.
+			defer syscall.Kill(-dlv.Process.Pid, syscall.SIGKILL)
+			// Wait to give dlv a chance to connect and pause the process.
+			time.Sleep(time.Second)
+		}
+	}
+
 	flag.Parse()
 
 	// The top-level Blueprints file is passed as the first argument.
@@ -72,7 +113,17 @@ func main() {
 
 	ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
 
-	bootstrap.Main(ctx.Context, configuration, configuration.ConfigFileName, configuration.ProductVariablesFileName)
+	extraNinjaDeps := []string{configuration.ConfigFileName, configuration.ProductVariablesFileName}
+
+	// Read the SOONG_DELVE again through configuration so that there is a dependency on the environment variable
+	// and soong_build will rerun when it is set for the first time.
+	if listen := configuration.Getenv("SOONG_DELVE"); listen != "" {
+		// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
+		// enabled even if it completed successfully.
+		extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.BuildDir(), "always_rerun_for_delve"))
+	}
+
+	bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...)
 
 	if docFile != "" {
 		if err := writeDocs(ctx, docFile); err != nil {

ui/build/exec.go

@@ -15,7 +15,10 @@
 package build
 
 import (
+	"bufio"
+	"io"
 	"os/exec"
+	"strings"
 )
 
 // Cmd is a wrapper of os/exec.Cmd that integrates with the build context for
@@ -139,3 +142,34 @@ func (c *Cmd) RunAndPrintOrFatal() {
 	st.Finish()
 	c.reportError(err)
 }
+
+// RunAndStreamOrFatal will run the command, while running print
+// any output, then handle any errors with a call to ctx.Fatal
+func (c *Cmd) RunAndStreamOrFatal() {
+	out, err := c.StdoutPipe()
+	if err != nil {
+		c.ctx.Fatal(err)
+	}
+	c.Stderr = c.Stdout
+
+	st := c.ctx.Status.StartTool()
+
+	c.StartOrFatal()
+
+	buf := bufio.NewReaderSize(out, 2*1024*1024)
+	for {
+		// Attempt to read whole lines, but write partial lines that are too long to fit in the buffer or hit EOF
+		line, err := buf.ReadString('\n')
+		if line != "" {
+			st.Print(strings.TrimSuffix(line, "\n"))
+		} else if err == io.EOF {
+			break
+		} else if err != nil {
+			c.ctx.Fatal(err)
+		}
+	}
+
+	err = c.Wait()
+	st.Finish()
+	c.reportError(err)
+}

ui/build/ninja.go

@@ -103,7 +103,7 @@ func runNinja(ctx Context, config Config) {
 	}()
 
 	ctx.Status.Status("Starting ninja...")
-	cmd.RunAndPrintOrFatal()
+	cmd.RunAndStreamOrFatal()
 }
 
 type statusChecker struct {

ui/build/paths/config.go

@@ -81,6 +81,7 @@ var Configuration = map[string]PathConfig{
 	"bzip2":    Allowed,
 	"dd":       Allowed,
 	"diff":     Allowed,
+	"dlv":      Allowed,
 	"egrep":    Allowed,
 	"expr":     Allowed,
 	"find":     Allowed,

ui/build/sandbox_linux.go

@@ -162,6 +162,10 @@ func (c *Cmd) wrapSandbox() {
 		c.ctx.Printf("AllowBuildBrokenUsesNetwork: %v", c.Sandbox.AllowBuildBrokenUsesNetwork)
 		c.ctx.Printf("BuildBrokenUsesNetwork: %v", c.config.BuildBrokenUsesNetwork())
 		sandboxArgs = append(sandboxArgs, "-N")
+	} else if dlv, _ := c.config.Environment().Get("SOONG_DELVE"); dlv != "" {
+		// The debugger is enabled and soong_build will pause until a remote delve process connects, allow
+		// network connections.
+		sandboxArgs = append(sandboxArgs, "-N")
 	}
 
 	// Stop nsjail from parsing arguments

ui/build/soong.go

@@ -120,7 +120,7 @@ func runSoong(ctx Context, config Config) {
 			"--frontend_file", fifo,
 			"-f", filepath.Join(config.SoongOutDir(), file))
 		cmd.Sandbox = soongSandbox
-		cmd.RunAndPrintOrFatal()
+		cmd.RunAndStreamOrFatal()
 	}
 
 	ninja("minibootstrap", ".minibootstrap/build.ninja")