Document the bootstrap process.
This change adds a bunch of docs describing the bootstrap process. It also renames a couple things in the source to better correspond with the terminology used in the docs. On top of that it makes the standalone build to copy minibp from the .bootstrap/bin directory to the bin directory to make it easier to find. Change-Id: Ibae3e607b59e2caf7d764ec3153155fde208ec1e
This commit is contained in:
Jamie Gennis
parent
86179feee4
commit
7ab5f3c4ba
4 changed files with 191 additions and 51 deletions
|
|
@ -354,9 +354,9 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
topLevelBlueprints := filepath.Join("$SrcDir",
|
||||
filepath.Base(topLevelBlueprintsFile))
|
||||
|
||||
tmpNinjaFile := filepath.Join(bootstrapDir, "build.ninja.in")
|
||||
tmpNinjaDepFile := tmpNinjaFile + ".d"
|
||||
tmpBootstrapFile := filepath.Join(bootstrapDir, "bootstrap.ninja.in")
|
||||
mainNinjaFile := filepath.Join(bootstrapDir, "main.ninja.in")
|
||||
mainNinjaDepFile := mainNinjaFile + ".d"
|
||||
bootstrapNinjaFile := filepath.Join(bootstrapDir, "bootstrap.ninja.in")
|
||||
|
||||
if generatingBootstrapper(ctx.Config()) {
|
||||
// We're generating a bootstrapper Ninja file, so we need to set things
|
||||
|
|
@ -380,14 +380,14 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
blueprint.RuleParams{
|
||||
Command: fmt.Sprintf("%s %s -d %s -o $out $in",
|
||||
primaryBuilderFile, primaryBuilderExtraFlags,
|
||||
tmpNinjaDepFile),
|
||||
mainNinjaDepFile),
|
||||
Description: fmt.Sprintf("%s $out", primaryBuilderName),
|
||||
Depfile: tmpNinjaDepFile,
|
||||
Depfile: mainNinjaDepFile,
|
||||
})
|
||||
|
||||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: bigbp,
|
||||
Outputs: []string{tmpNinjaFile},
|
||||
Outputs: []string{mainNinjaFile},
|
||||
Inputs: []string{topLevelBlueprints},
|
||||
Implicits: []string{primaryBuilderFile},
|
||||
})
|
||||
|
|
@ -397,7 +397,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
// accomplish that we depend on a file that should never exist and
|
||||
// "build" it using Ninja's built-in phony rule.
|
||||
//
|
||||
// We also need to add an implicit dependency on tmpBootstrapFile so
|
||||
// We also need to add an implicit dependency on bootstrapNinjaFile so
|
||||
// that it gets generated as part of the bootstrap process.
|
||||
notAFile := filepath.Join(bootstrapDir, "notAFile")
|
||||
ctx.Build(blueprint.BuildParams{
|
||||
|
|
@ -408,8 +408,8 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: bootstrap,
|
||||
Outputs: []string{"build.ninja"},
|
||||
Inputs: []string{tmpNinjaFile},
|
||||
Implicits: []string{"$Bootstrap", notAFile, tmpBootstrapFile},
|
||||
Inputs: []string{mainNinjaFile},
|
||||
Implicits: []string{"$Bootstrap", notAFile, bootstrapNinjaFile},
|
||||
})
|
||||
|
||||
// Rebuild the bootstrap Ninja file using the minibp that we just built.
|
||||
|
|
@ -430,7 +430,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
|
||||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: minibp,
|
||||
Outputs: []string{tmpBootstrapFile},
|
||||
Outputs: []string{bootstrapNinjaFile},
|
||||
Inputs: []string{topLevelBlueprints},
|
||||
Implicits: []string{minibpFile},
|
||||
Args: map[string]string{
|
||||
|
|
@ -449,22 +449,26 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
// rule. We do this by depending on that file and then setting up a
|
||||
// phony rule to generate it that uses the depfile.
|
||||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: rebootstrap,
|
||||
Outputs: []string{"build.ninja"},
|
||||
Inputs: []string{"$BootstrapManifest"},
|
||||
Implicits: []string{"$Bootstrap", primaryBuilderFile, tmpNinjaFile},
|
||||
Rule: rebootstrap,
|
||||
Outputs: []string{"build.ninja"},
|
||||
Inputs: []string{"$BootstrapManifest"},
|
||||
Implicits: []string{
|
||||
"$Bootstrap",
|
||||
primaryBuilderFile,
|
||||
mainNinjaFile,
|
||||
},
|
||||
})
|
||||
|
||||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: phony,
|
||||
Outputs: []string{tmpNinjaFile},
|
||||
Outputs: []string{mainNinjaFile},
|
||||
Inputs: []string{topLevelBlueprints},
|
||||
Args: map[string]string{
|
||||
"depfile": tmpNinjaDepFile,
|
||||
"depfile": mainNinjaDepFile,
|
||||
},
|
||||
})
|
||||
|
||||
// If the bootstrap Ninja invocation caused a new tmpBootstrapFile to be
|
||||
// If the bootstrap Ninja invocation caused a new bootstrapNinjaFile to be
|
||||
// generated then that means we need to rebootstrap using it instead of
|
||||
// the current bootstrap manifest. We enable the Ninja "generator"
|
||||
// behavior so that Ninja doesn't invoke this build just because it's
|
||||
|
|
@ -472,11 +476,22 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: cp,
|
||||
Outputs: []string{"$BootstrapManifest"},
|
||||
Inputs: []string{tmpBootstrapFile},
|
||||
Inputs: []string{bootstrapNinjaFile},
|
||||
Args: map[string]string{
|
||||
"generator": "true",
|
||||
},
|
||||
})
|
||||
|
||||
if primaryBuilderName == "minibp" {
|
||||
// This is a standalone Blueprint build, so we copy the minibp
|
||||
// binary to the "bin" directory to make it easier to find.
|
||||
finalMinibp := filepath.Join("bin", primaryBuilderName)
|
||||
ctx.Build(blueprint.BuildParams{
|
||||
Rule: cp,
|
||||
Inputs: []string{primaryBuilderFile},
|
||||
Outputs: []string{finalMinibp},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
// These variables are the only only configuration needed by the boostrap
|
||||
// These variables are the only configuration needed by the boostrap
|
||||
// modules. They are always set to the variable name enclosed in "@@" so
|
||||
// that their values can be easily replaced in the generated Ninja file.
|
||||
SrcDir = blueprint.StaticVariable("SrcDir", "@@SrcDir@@")
|
||||
|
|
|
|||
|
|
@ -1,29 +1,154 @@
|
|||
/*
|
||||
|
||||
The Blueprint bootstrapping mechanism is intended to enable building a source
|
||||
tree using a Blueprint-based build system that is embedded (as source) in that
|
||||
source tree. The only prerequisites for performing such a build are:
|
||||
1. A Ninja binary
|
||||
2. A script interpreter (e.g. Bash or Python)
|
||||
3. A Go toolchain
|
||||
|
||||
The bootstrapping process is intended to be customized for the source tree in
|
||||
which it is embedded. As an initial example, we'll examine the bootstrapping
|
||||
system used to build the core Blueprint packages. Bootstrapping the core
|
||||
Blueprint packages involves two files:
|
||||
|
||||
bootstrap.bash: When this script is run it initializes the current
|
||||
working directory as a build output directory. It does
|
||||
this by first automatically determining the root source
|
||||
directory and Go build environment. It then uses those
|
||||
values to do a simple string replacement over the
|
||||
build.ninja.in file contents, and places the result into
|
||||
the current working directory.
|
||||
|
||||
build.ninja.in: This file is generated by passing all the Blueprint
|
||||
|
||||
files needed to construct the primary builder
|
||||
|
||||
|
||||
*/
|
||||
// The Blueprint bootstrapping mechanism is intended to enable building a source
|
||||
// tree using a Blueprint-based build system that is embedded (as source) in
|
||||
// that source tree. The only prerequisites for performing such a build are:
|
||||
//
|
||||
// 1. A Ninja binary
|
||||
// 2. A script interpreter (e.g. Bash or Python)
|
||||
// 3. A Go toolchain
|
||||
//
|
||||
// The Primary Builder
|
||||
//
|
||||
// As part of the bootstrapping process, a binary called the "primary builder"
|
||||
// is created. This primary builder is the binary that includes both the core
|
||||
// Blueprint library and the build logic specific to the source tree. It is
|
||||
// used to generate the Ninja file that describes how to build the entire source
|
||||
// tree.
|
||||
//
|
||||
// The primary builder must be a pure Go (i.e. no cgo) module built with the
|
||||
// module type 'bootstrap_go_binary'. It should have the 'primaryBuilder'
|
||||
// module property set to true in its Blueprints file. If more than one module
|
||||
// sets primaryBuilder to true the build will fail.
|
||||
//
|
||||
// The primary builder main function should look something like:
|
||||
//
|
||||
// package main
|
||||
//
|
||||
// import (
|
||||
// "blueprint"
|
||||
// "blueprint/bootstrap"
|
||||
// "flag"
|
||||
// "path/filepath"
|
||||
//
|
||||
// "my/custom/build/logic"
|
||||
// )
|
||||
//
|
||||
// func main() {
|
||||
// // The primary builder should use the global flag set because the
|
||||
// // bootstrap package registers its own flags there.
|
||||
// flag.Parse()
|
||||
//
|
||||
// // The top-level Blueprints file is passed as the first argument.
|
||||
// srcDir := filepath.Dir(flag.Arg(0))
|
||||
//
|
||||
// // Create the build context.
|
||||
// ctx := blueprint.NewContext()
|
||||
//
|
||||
// // Register custom module types
|
||||
// ctx.RegisterModuleType("foo", logic.FooModule)
|
||||
// ctx.RegisterModuleType("bar", logic.BarModule)
|
||||
//
|
||||
// // Register custom singletons
|
||||
// ctx.RegisterSingleton("baz", logic.NewBazSingleton())
|
||||
//
|
||||
// // Create and initialize the custom Config object.
|
||||
// config := logic.NewConfig(srcDir)
|
||||
//
|
||||
// // This call never returns
|
||||
// bootstrap.Main(ctx, config)
|
||||
// }
|
||||
//
|
||||
// Required Source Files
|
||||
//
|
||||
// There are three files that must be included in the source tree to facilitate
|
||||
// the build bootstrapping:
|
||||
//
|
||||
// 1. The top-level Blueprints file
|
||||
// 2. The bootstrap Ninja file template
|
||||
// 3. The bootstrap script
|
||||
//
|
||||
// The top-level Blueprints file describes how the entire source tree should be
|
||||
// built. It must have a 'subdirs' assignment that includes both the core
|
||||
// Blueprint library and the custom build logic for the source tree. It should
|
||||
// also include (either directly or through a subdirs entry) describe all the
|
||||
// modules to be built in the source tree.
|
||||
//
|
||||
// The bootstrap Ninja file template describes the build actions necessary to
|
||||
// build the primary builder for the source tree. This template contains a set
|
||||
// of placeholder Ninja variable values that get filled in by the bootstrap
|
||||
// script to create a usable Ninja file. It can be created by running the
|
||||
// minibp binary that gets created as part of the standalone Blueprint build.
|
||||
// Passing minibp the path to the top-level Blueprints file will cause it to
|
||||
// create a bootstrap Ninja file template named 'build.ninja.in'.
|
||||
//
|
||||
// The bootstrap script is a small script (or theoretically a compiled binary)
|
||||
// that is included in the source tree to begin the bootstrapping process. It
|
||||
// is responsible for providing filling in the bootstrap Ninja file template
|
||||
// with some basic information about the Go build environemnt and the path to
|
||||
// the root source directory. It does this by performing a simple string
|
||||
// substitution on the template file to produce a usable build.ninja file.
|
||||
//
|
||||
// The Bootstrapping Process
|
||||
//
|
||||
// A bootstrap-enabled build directory has two states, each with a corresponding
|
||||
// Ninja file. The states are referred to as the "bootstrap" state and the
|
||||
// "main" state. Changing the directory to a particular state means replacing
|
||||
// the build.ninja file one that will perform the build actions for the state.
|
||||
//
|
||||
// The bootstrapping process begins with the user running the bootstrap script
|
||||
// to initialize a new build directory. The script is run from the build
|
||||
// directory, and when run with no arguments it copies the source bootstrap
|
||||
// Ninja file into the build directory as "build.ninja". It also performs a set
|
||||
// of string substitutions on the file to configure it for the user's build
|
||||
// environment. Specifically, the following strings are substituted in the file:
|
||||
//
|
||||
// @@SrcDir@@ - The path to the root source directory (either
|
||||
// absolute or relative to the build dir)
|
||||
// @@GoRoot@@ - The path to the root directory of the Go toolchain
|
||||
// @@GoOS@@ - The OS string for the Go toolchain
|
||||
// @@GoArch@@ - The CPU architecture for the Go toolchain
|
||||
// @@GoChar@@ - The CPU arch character for the Go toolchain
|
||||
// @@Bootstrap@@ - The path to the bootstrap script
|
||||
// @@BootstrapManifest@@ - The path to the source bootstrap Ninja file
|
||||
//
|
||||
// Once the script completes the build directory is initialized in the bootstrap
|
||||
// build state. In this state, running Ninja may perform the following build
|
||||
// actions. Each one but the last can be skipped if its output is determined to
|
||||
// be up-to-date.
|
||||
//
|
||||
// - Build the minibp binary
|
||||
// - Run minibp to generate .bootstrap/bootstrap.ninja.in
|
||||
// - Build the primary builder binary
|
||||
// - Run the primary builder to generate .bootstrap/main.ninja.in
|
||||
// - Run the bootstrap script to "copy" .bootstrap/main.ninja.in to build.ninja
|
||||
//
|
||||
// The last of these build actions results in transitioning the build directory
|
||||
// to the main build state. In this state, the
|
||||
//
|
||||
// The main state (potentially) performs the following actions:
|
||||
// - Copy .bootstrap/bootstrap.ninja.in to the source bootstrap Ninja location
|
||||
// - Run the bootstrap script to "copy" the source bootstrap Ninja file to build.ninja
|
||||
// - Build all the non-bootstrap modules defined in Blueprints files
|
||||
//
|
||||
// Updating the Bootstrap Ninja File Template
|
||||
//
|
||||
// The main purpose of the bootstrap state is to generate the Ninja file for the
|
||||
// main state. The one additional thing it does is generate a new bootstrap
|
||||
// Ninja file template at .bootstrap/bootstrap.ninja.in. When generating this
|
||||
// file, minibp will compare the new bootstrap Ninja file contents with the
|
||||
// original (in the source tree). If the contents match, the new file will be
|
||||
// created with a timestamp that matches that of the original, indicating that
|
||||
// the original file in the source tree is up-to-date.
|
||||
//
|
||||
// This is done so that in the main state if the bootstrap Ninja file template
|
||||
// in the source tree is out of date it can be automatically updated. Note,
|
||||
// however, that we can't have the main state generate the new bootstrap Ninja
|
||||
// file template contents itself, because it may be using an older minibp.
|
||||
// Recall that minibp is only built during the bootstrap state (to break a
|
||||
// circular dependence), so if a new bootstrap Ninja file template were
|
||||
// generated then it could replace a new file (from an updated source tree) with
|
||||
// one generated using an old minibp.
|
||||
//
|
||||
// This scheme ensures that updates to the source tree are always incorporated
|
||||
// into the build process and that changes that require a new bootstrap Ninja
|
||||
// file template automatically update the template in the source tree.
|
||||
package bootstrap
|
||||
|
|
|
|||
|
|
@ -161,8 +161,8 @@ build .bootstrap/bin/minibp: g.bootstrap.cp .bootstrap/minibp/obj/a.out
|
|||
# GoType: blueprint/bootstrap.singleton
|
||||
|
||||
rule s.bootstrap.bigbp
|
||||
command = .bootstrap/bin/minibp -p -d .bootstrap/build.ninja.in.d -o ${out} ${in}
|
||||
depfile = .bootstrap/build.ninja.in.d
|
||||
command = .bootstrap/bin/minibp -p -d .bootstrap/main.ninja.in.d -o ${out} ${in}
|
||||
depfile = .bootstrap/main.ninja.in.d
|
||||
description = minibp ${out}
|
||||
|
||||
rule s.bootstrap.minibp
|
||||
|
|
@ -171,10 +171,10 @@ rule s.bootstrap.minibp
|
|||
description = minibp ${out}
|
||||
generator = true
|
||||
|
||||
build .bootstrap/build.ninja.in: s.bootstrap.bigbp $
|
||||
build .bootstrap/main.ninja.in: s.bootstrap.bigbp $
|
||||
${g.bootstrap.SrcDir}/Blueprints | .bootstrap/bin/minibp
|
||||
build .bootstrap/notAFile: phony
|
||||
build build.ninja: g.bootstrap.bootstrap .bootstrap/build.ninja.in | $
|
||||
build build.ninja: g.bootstrap.bootstrap .bootstrap/main.ninja.in | $
|
||||
${g.bootstrap.Bootstrap} .bootstrap/notAFile $
|
||||
.bootstrap/bootstrap.ninja.in
|
||||
build .bootstrap/bootstrap.ninja.in: s.bootstrap.minibp $
|
||||
|
|
|
|||
Loading…
Reference in a new issue