Add build wrapper to do stage selection before ninja

This wrapper script can be used instead of ninja to ensure the build
won't get stuck building the primary builder.

An example of when this would happen:

1. Do a successful build
2. Sync/make a change in the primary builder (soong, etc) that depends
   on a blueprint change.
3. The next build would notice that change, and rewind to the primary
   stage to rebuild the builder. That build would fail.
4. Sync/fix the blueprint code.
5. The next build would still fail. The bootstrap stage would need to
   be run in order to fix the primary stage, but we're still stuck in the
   primary stage. The only way to switch stages is to successfully
   complete everything required to choose the next stage.

This generally isn't a problem in the main stage, since there is no code
being built in the dependency chain leading up to stage selection.

Any existing wrappers (like soong) can execute this wrapper (optionally
skipping ninja execution) if they're worried about the above situation.
This isn't strictly required -- running ninja directly will still work
in most cases, you'll just need to re-run bootstrap.bash if you get into
a bad state.

Change-Id: I5901d7240a1daa388a786ceb1c8259502fc14058

.travis.yml

@@ -17,8 +17,8 @@ before_script:
 script:
     - go test ./...
     - mkdir stage
-    - cd stage 
+    - cd stage
     - ../bootstrap.bash
-    - ninja
+    - ./blueprint.bash
     - diff -us ../build.ninja.in .bootstrap/bootstrap.ninja.in
     - ../tests/test.sh

blueprint.bash

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# This script is intented to wrap the execution of ninja so that we
+# can do some checks before each ninja run.
+#
+# It can either be run with a standalone Blueprint checkout to generate
+# the minibp binary, or can be used by another script as part of a custom
+# Blueprint-based build system. When used by another script, the following
+# environment variables can be set to configure this script, which are
+# documented below:
+#
+#   BUILDDIR
+#   SKIP_NINJA
+#
+# When run in a standalone Blueprint checkout, bootstrap.bash will install
+# this script into the $BUILDDIR, where it may be executed.
+#
+# For embedding into a custom build system, the current directory when this
+# script executes should be the same directory that $BOOTSTRAP should be
+# called from.
+
+set -e
+
+# BUILDDIR should be set to the path to store build results. By default,
+# this is the directory containing this script, but can be set explicitly
+# if the custom build system only wants to install their own wrapper.
+[ -z "$BUILDDIR" ] && BUILDDIR=`dirname "${BASH_SOURCE[0]}"`
+
+# .blueprint.bootstrap provides saved values from the bootstrap.bash script:
+#
+#   BOOTSTRAP
+#   BOOTSTRAP_MANIFEST
+#
+# If it doesn't exist, we probably just need to re-run bootstrap.bash, which
+# ninja will do when switching stages. So just skip to ninja.
+if [ -f "${BUILDDIR}/.blueprint.bootstrap" ]; then
+    source "${BUILDDIR}/.blueprint.bootstrap"
+
+    # Pick the newer of .bootstrap/bootstrap.ninja.in or $BOOTSTRAP_MANIFEST,
+    # and copy it to .bootstrap/build.ninja.in
+    GEN_BOOTSTRAP_MANIFEST="${BUILDDIR}/.bootstrap/bootstrap.ninja.in"
+    if [ -f "${GEN_BOOTSTRAP_MANIFEST}" ]; then
+        if [ "${GEN_BOOTSTRAP_MANIFEST}" -nt "${BOOTSTRAP_MANIFEST}" ]; then
+            BOOTSTRAP_MANIFEST="${GEN_BOOTSTRAP_MANIFEST}"
+        fi
+    fi
+
+    # Copy the selected manifest to $BUILDDIR/.bootstrap/build.ninja.in
+    mkdir -p "${BUILDDIR}/.bootstrap"
+    cp "${BOOTSTRAP_MANIFEST}" "${BUILDDIR}/.bootstrap/build.ninja.in"
+
+    # Bootstrap it to $BUILDDIR/build.ninja
+    "${BOOTSTRAP}" -i "${BUILDDIR}/.bootstrap/build.ninja.in"
+fi
+
+# SKIP_NINJA can be used by wrappers that wish to run ninja themselves.
+if [ -z "$SKIP_NINJA" ]; then
+    ninja -C "${BUILDDIR}" "$@"
+else
+    exit 0
+fi

bootstrap.bash

@@ -10,6 +10,7 @@
 # their default values are set:
 #
 #   BOOTSTRAP
+#   WRAPPER
 #   SRCDIR
 #   BUILDDIR
 #   BOOTSTRAP_MANIFEST
@@ -28,7 +29,14 @@ EXTRA_ARGS=""
 # BOOTSTRAP should be set to the path of the bootstrap script.  It can be
 # either an absolute path or one relative to the build directory (which of
 # these is used should probably match what's used for SRCDIR).
-[ -z "$BOOTSTRAP" ] && BOOTSTRAP="${BASH_SOURCE[0]}"
+if [ -z "$BOOTSTRAP" ]; then
+    BOOTSTRAP="${BASH_SOURCE[0]}"
+
+    # WRAPPER should only be set if you want a ninja wrapper script to be
+    # installed into the builddir. It is set to blueprint's blueprint.bash
+    # only if BOOTSTRAP and WRAPPER are unset.
+    [ -z "$WRAPPER" ] && WRAPPER="`dirname "${BOOTSTRAP}"`/blueprint.bash"
+fi
 
 # SRCDIR should be set to the path of the root source directory.  It can be
 # either an absolute path or a path relative to the build directory.  Whether
@@ -128,3 +136,10 @@ sed -e "s|@@SrcDir@@|$SRCDIR|g"                        \
     -e "s|@@Bootstrap@@|$BOOTSTRAP|g"                  \
     -e "s|@@BootstrapManifest@@|$BOOTSTRAP_MANIFEST|g" \
     $IN > $BUILDDIR/build.ninja
+
+echo "BOOTSTRAP=\"${BOOTSTRAP}\"" > $BUILDDIR/.blueprint.bootstrap
+echo "BOOTSTRAP_MANIFEST=\"${BOOTSTRAP_MANIFEST}\"" >> $BUILDDIR/.blueprint.bootstrap
+
+if [ ! -z "$WRAPPER" ]; then
+    cp $WRAPPER $BUILDDIR/
+fi

tests/expected_wrapper_all

@@ -0,0 +1,2 @@
+Choosing primary.ninja.in for next stage
+Choosing main.ninja.in for next stage

tests/expected_wrapper_regen

@@ -0,0 +1,4 @@
+Stage bootstrap.ninja.in has changed, restarting
+Choosing bootstrap.ninja.in for next stage
+Choosing primary.ninja.in for next stage
+Choosing main.ninja.in for next stage

tests/expected_wrapper_start

@@ -0,0 +1,2 @@
+Choosing primary.ninja.in for next stage
+Choosing main.ninja.in for next stage

tests/expected_wrapper_start2

@@ -0,0 +1,2 @@
+Choosing primary.ninja.in for next stage
+Choosing main.ninja.in for next stage

tests/test.sh

@@ -29,7 +29,26 @@ function testcase()
   fi
 }
 
-
+# Run wrapper, filter the output, and compare against expectations
+# $1: Name of test
+function testcase_wrapper()
+{
+  echo -n "Running wrapper_$1..."
+  if ! ./blueprint.bash -v -d explain >log_wrapper_$1 2>&1; then
+    echo " Failed."
+    echo "Test wrapper_$1 Failed:" >>failed
+    tail log_wrapper_$1 >>failed
+    return
+  fi
+  grep -E "^(Choosing|Newer|Stage)" log_wrapper_$1 >test_wrapper_$1
+  if ! cmp -s test_wrapper_$1 ../tests/expected_wrapper_$1; then
+    echo " Failed."
+    echo "Test wrapper_$1 Failed:" >>failed
+    diff -u ../tests/expected_wrapper_$1 test_wrapper_$1 >>failed
+  else
+    echo " Passed."
+  fi
+}
 
 
 testcase start
@@ -85,8 +104,36 @@ sleep 2
 touch ../parser/parser_test.go
 testcase rebuild_test
 
+# Restart testing using the wrapper instead of going straight to ninja. This
+# will force each test to start in the correct bootstrap stage, so there are
+# less cases to test.
+cd ..
+rm -rf out.test
+mkdir -p out.test
+cd out.test
+../bootstrap.bash
 
+testcase_wrapper start
 
+# This test affects all bootstrap stages
+sleep 2
+touch ../Blueprints
+testcase_wrapper all
+
+# From now on, we're going to be modifying the build.ninja.in, so let's make our
+# own copy
+sleep 2
+../tests/bootstrap.bash -r
+
+sleep 2
+testcase_wrapper start2
+
+# This is similar to the last test, but incorporates a change into the source
+# build.ninja.in, so that we'll restart into the new version created by the
+# build.
+sleep 2
+echo "# test" >>src.build.ninja.in
+testcase_wrapper regen
 
 if [ -f failed ]; then
   cat failed