Remove bp2build and bazel from soong_ui
Bug: 315353489 Test: m blueprint_tests Change-Id: I9df31b18caaae24e3cf2994e56bb90b50523f11e
This commit is contained in:
Colin Cross
parent
d788b3e6cb
commit
8d411ff8f8
10 changed files with 41 additions and 913 deletions
|
|
@ -18,7 +18,6 @@ import (
|
|||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
|
@ -194,16 +193,13 @@ func main() {
|
|||
buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
|
||||
soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
|
||||
rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
|
||||
bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
|
||||
soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
|
||||
|
||||
metricsFiles := []string{
|
||||
buildErrorFile, // build error strings
|
||||
rbeMetricsFile, // high level metrics related to remote build execution.
|
||||
bp2buildMetricsFile, // high level metrics related to bp2build.
|
||||
soongMetricsFile, // high level metrics related to this build system.
|
||||
soongBuildMetricsFile, // high level metrics related to soong build(except bp2build)
|
||||
config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
|
||||
buildErrorFile, // build error strings
|
||||
rbeMetricsFile, // high level metrics related to remote build execution.
|
||||
soongMetricsFile, // high level metrics related to this build system.
|
||||
soongBuildMetricsFile, // high level metrics related to soong build
|
||||
}
|
||||
|
||||
os.MkdirAll(logsDir, 0777)
|
||||
|
|
@ -293,38 +289,12 @@ func preProductConfigSetup(buildCtx build.Context, config build.Config) {
|
|||
}
|
||||
}
|
||||
|
||||
removeBadTargetRename(buildCtx, config)
|
||||
|
||||
// Create a source finder.
|
||||
f := build.NewSourceFinder(buildCtx, config)
|
||||
defer f.Shutdown()
|
||||
build.FindSources(buildCtx, config, f)
|
||||
}
|
||||
|
||||
func removeBadTargetRename(ctx build.Context, config build.Config) {
|
||||
log := ctx.ContextImpl.Logger
|
||||
// find bad paths
|
||||
m, err := filepath.Glob(filepath.Join(config.OutDir(), "bazel", "output", "execroot", "__main__", "bazel-out", "mixed_builds_product-*", "bin", "tools", "metalava", "metalava"))
|
||||
if err != nil {
|
||||
log.Fatalf("Glob for invalid file failed %s", err)
|
||||
}
|
||||
for _, f := range m {
|
||||
info, err := os.Stat(f)
|
||||
if err != nil {
|
||||
log.Fatalf("Stat of invalid file %q failed %s", f, err)
|
||||
}
|
||||
// if it's a directory, leave it, but remove the files
|
||||
if !info.IsDir() {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
log.Fatalf("Remove of invalid file %q failed %s", f, err)
|
||||
} else {
|
||||
log.Verbosef("Removed %q", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func dumpVar(ctx build.Context, config build.Config, args []string) {
|
||||
flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
|
||||
flags.SetOutput(ctx.Writer)
|
||||
|
|
@ -607,81 +577,6 @@ func getCommand(args []string) (*command, []string, error) {
|
|||
return nil, nil, fmt.Errorf("Command not found: %q\nDid you mean one of these: %q", args[1], listFlags())
|
||||
}
|
||||
|
||||
// For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary.
|
||||
func populateExternalDistDir(ctx build.Context, config build.Config) {
|
||||
// Make sure that internalDistDirPath and externalDistDirPath are both absolute paths, so we can compare them
|
||||
var err error
|
||||
var internalDistDirPath string
|
||||
var externalDistDirPath string
|
||||
if internalDistDirPath, err = filepath.Abs(config.DistDir()); err != nil {
|
||||
ctx.Fatalf("Unable to find absolute path of %s: %s", internalDistDirPath, err)
|
||||
}
|
||||
if externalDistDirPath, err = filepath.Abs(config.RealDistDir()); err != nil {
|
||||
ctx.Fatalf("Unable to find absolute path of %s: %s", externalDistDirPath, err)
|
||||
}
|
||||
if externalDistDirPath == internalDistDirPath {
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure the internal DIST_DIR actually exists before trying to read from it
|
||||
if _, err = os.Stat(internalDistDirPath); os.IsNotExist(err) {
|
||||
ctx.Println("Skipping Bazel dist dir migration - nothing to do!")
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure the external DIST_DIR actually exists before trying to write to it
|
||||
if err = os.MkdirAll(externalDistDirPath, 0755); err != nil {
|
||||
ctx.Fatalf("Unable to make directory %s: %s", externalDistDirPath, err)
|
||||
}
|
||||
|
||||
ctx.Println("Populating external DIST_DIR...")
|
||||
|
||||
populateExternalDistDirHelper(ctx, config, internalDistDirPath, externalDistDirPath)
|
||||
}
|
||||
|
||||
func populateExternalDistDirHelper(ctx build.Context, config build.Config, internalDistDirPath string, externalDistDirPath string) {
|
||||
files, err := ioutil.ReadDir(internalDistDirPath)
|
||||
if err != nil {
|
||||
ctx.Fatalf("Can't read internal distdir %s: %s", internalDistDirPath, err)
|
||||
}
|
||||
for _, f := range files {
|
||||
internalFilePath := filepath.Join(internalDistDirPath, f.Name())
|
||||
externalFilePath := filepath.Join(externalDistDirPath, f.Name())
|
||||
|
||||
if f.IsDir() {
|
||||
// Moving a directory - check if there is an existing directory to merge with
|
||||
externalLstat, err := os.Lstat(externalFilePath)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
ctx.Fatalf("Can't lstat external %s: %s", externalDistDirPath, err)
|
||||
}
|
||||
// Otherwise, if the error was os.IsNotExist, that's fine and we fall through to the rename at the bottom
|
||||
} else {
|
||||
if externalLstat.IsDir() {
|
||||
// Existing dir - try to merge the directories?
|
||||
populateExternalDistDirHelper(ctx, config, internalFilePath, externalFilePath)
|
||||
continue
|
||||
} else {
|
||||
// Existing file being replaced with a directory. Delete the existing file...
|
||||
if err := os.RemoveAll(externalFilePath); err != nil {
|
||||
ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Moving a file (not a dir) - delete any existing file or directory
|
||||
if err := os.RemoveAll(externalFilePath); err != nil {
|
||||
ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The actual move - do a rename instead of a copy in order to save disk space.
|
||||
if err := os.Rename(internalFilePath, externalFilePath); err != nil {
|
||||
ctx.Fatalf("Unable to rename %s -> %s due to error %s", internalFilePath, externalFilePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setMaxFiles(ctx build.Context) {
|
||||
var limits syscall.Rlimit
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ bootstrap_go_package {
|
|||
"soong-ui-tracer",
|
||||
],
|
||||
srcs: [
|
||||
"bazel_metrics.go",
|
||||
"build.go",
|
||||
"cleanbuild.go",
|
||||
"config.go",
|
||||
|
|
@ -75,7 +74,6 @@ bootstrap_go_package {
|
|||
"proc_sync_test.go",
|
||||
"rbe_test.go",
|
||||
"staging_snapshot_test.go",
|
||||
"upload_test.go",
|
||||
"util_test.go",
|
||||
],
|
||||
darwin: {
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
// Copyright 2023 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package build
|
||||
|
||||
// This file contains functionality to parse bazel profile data into
|
||||
// a bazel_metrics proto, defined in build/soong/ui/metrics/bazel_metrics_proto
|
||||
// These metrics are later uploaded in upload.go
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"android/soong/shared"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto"
|
||||
)
|
||||
|
||||
func parseTimingToNanos(str string) int64 {
|
||||
millisString := removeDecimalPoint(str)
|
||||
timingMillis, _ := strconv.ParseInt(millisString, 10, 64)
|
||||
return timingMillis * 1000000
|
||||
}
|
||||
|
||||
func parsePercentageToTenThousandths(str string) int32 {
|
||||
percentageString := removeDecimalPoint(str)
|
||||
//remove the % at the end of the string
|
||||
percentage := strings.ReplaceAll(percentageString, "%", "")
|
||||
percentagePortion, _ := strconv.ParseInt(percentage, 10, 32)
|
||||
return int32(percentagePortion)
|
||||
}
|
||||
|
||||
func removeDecimalPoint(numString string) string {
|
||||
// The format is always 0.425 or 10.425
|
||||
return strings.ReplaceAll(numString, ".", "")
|
||||
}
|
||||
|
||||
func parseTotal(line string) int64 {
|
||||
words := strings.Fields(line)
|
||||
timing := words[3]
|
||||
return parseTimingToNanos(timing)
|
||||
}
|
||||
|
||||
func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming {
|
||||
words := strings.Fields(line)
|
||||
getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) {
|
||||
// Sample lines include:
|
||||
// Total launch phase time 0.011 s 2.59%
|
||||
// Total target pattern evaluation phase time 0.011 s 2.59%
|
||||
var beginning int
|
||||
var end int
|
||||
for ind, word := range words {
|
||||
if word == "Total" {
|
||||
beginning = ind + 1
|
||||
} else if beginning > 0 && word == "phase" {
|
||||
end = ind
|
||||
break
|
||||
}
|
||||
}
|
||||
phaseName := strings.Join(words[beginning:end], " ")
|
||||
|
||||
// end is now "phase" - advance by 2 for timing and 4 for percentage
|
||||
percentageString := words[end+4]
|
||||
timingString := words[end+2]
|
||||
timing := parseTimingToNanos(timingString)
|
||||
percentagePortion := parsePercentageToTenThousandths(percentageString)
|
||||
return phaseName, timing, percentagePortion
|
||||
}
|
||||
|
||||
phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words)
|
||||
phaseTiming := bazel_metrics_proto.PhaseTiming{}
|
||||
phaseTiming.DurationNanos = &timing
|
||||
phaseTiming.PortionOfBuildTime = &portion
|
||||
|
||||
phaseTiming.PhaseName = &phaseName
|
||||
return phaseTiming
|
||||
}
|
||||
|
||||
// This method takes a file created by bazel's --analyze-profile mode and
|
||||
// writes bazel metrics data to the provided filepath.
|
||||
func ProcessBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context, config Config) {
|
||||
if bazelProfileFile == "" {
|
||||
return
|
||||
}
|
||||
|
||||
readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics {
|
||||
//serialize the proto, write it
|
||||
bazelMetrics := bazel_metrics_proto.BazelMetrics{}
|
||||
|
||||
file, err := os.ReadFile(filepath)
|
||||
if err != nil {
|
||||
ctx.Fatalln("Error reading metrics file\n", err)
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(string(file)))
|
||||
scanner.Split(bufio.ScanLines)
|
||||
|
||||
var phaseTimings []*bazel_metrics_proto.PhaseTiming
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "Total run time") {
|
||||
total := parseTotal(line)
|
||||
bazelMetrics.Total = &total
|
||||
} else if strings.HasPrefix(line, "Total") {
|
||||
phaseTiming := parsePhaseTiming(line)
|
||||
phaseTimings = append(phaseTimings, &phaseTiming)
|
||||
}
|
||||
}
|
||||
bazelMetrics.PhaseTimings = phaseTimings
|
||||
bazelMetrics.BesId = proto.String(config.besId)
|
||||
|
||||
return bazelMetrics
|
||||
}
|
||||
|
||||
if _, err := os.Stat(bazelProfileFile); err != nil {
|
||||
// We can assume bazel didn't run if the profile doesn't exist
|
||||
return
|
||||
}
|
||||
bazelProto := readBazelProto(bazelProfileFile)
|
||||
bazelProto.ExitCode = proto.Int32(config.bazelExitCode)
|
||||
shared.Save(&bazelProto, bazelMetricsFile)
|
||||
}
|
||||
|
|
@ -135,22 +135,6 @@ const (
|
|||
RunBuildTests = 1 << iota
|
||||
)
|
||||
|
||||
// checkBazelMode fails the build if there are conflicting arguments for which bazel
|
||||
// build mode to use.
|
||||
func checkBazelMode(ctx Context, config Config) {
|
||||
count := 0
|
||||
if config.bazelProdMode {
|
||||
count++
|
||||
}
|
||||
if config.bazelStagingMode {
|
||||
count++
|
||||
}
|
||||
if count > 1 {
|
||||
ctx.Fatalln("Conflicting bazel mode.\n" +
|
||||
"Do not specify more than one of --bazel-mode and --bazel-mode-staging ")
|
||||
}
|
||||
}
|
||||
|
||||
// checkProblematicFiles fails the build if existing Android.mk or CleanSpec.mk files are found at the root of the tree.
|
||||
func checkProblematicFiles(ctx Context) {
|
||||
files := []string{"Android.mk", "CleanSpec.mk"}
|
||||
|
|
@ -262,8 +246,6 @@ func Build(ctx Context, config Config) {
|
|||
|
||||
defer waitForDist(ctx)
|
||||
|
||||
checkBazelMode(ctx, config)
|
||||
|
||||
// checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree.
|
||||
checkProblematicFiles(ctx)
|
||||
|
||||
|
|
|
|||
|
|
@ -72,7 +72,6 @@ type configImpl struct {
|
|||
checkbuild bool
|
||||
dist bool
|
||||
jsonModuleGraph bool
|
||||
bp2build bool
|
||||
queryview bool
|
||||
reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
|
||||
soongDocs bool
|
||||
|
|
@ -87,9 +86,7 @@ type configImpl struct {
|
|||
skipMetricsUpload bool
|
||||
buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
|
||||
buildFromSourceStub bool
|
||||
ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
|
||||
bazelExitCode int32 // For b runs - necessary for updating NonZeroExit
|
||||
besId string // For b runs, to identify the BuildEventService logs
|
||||
ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
|
||||
|
||||
// From the product config
|
||||
katiArgs []string
|
||||
|
|
@ -108,16 +105,11 @@ type configImpl struct {
|
|||
|
||||
pathReplaced bool
|
||||
|
||||
bazelProdMode bool
|
||||
bazelStagingMode bool
|
||||
|
||||
// Set by multiproduct_kati
|
||||
emptyNinjaFile bool
|
||||
|
||||
metricsUploader string
|
||||
|
||||
bazelForceEnabledModules string
|
||||
|
||||
includeTags []string
|
||||
sourceRootDirs []string
|
||||
|
||||
|
|
@ -450,11 +442,6 @@ func NewConfig(ctx Context, args ...string) Config {
|
|||
}
|
||||
}
|
||||
|
||||
bpd := ret.BazelMetricsDir()
|
||||
if err := os.RemoveAll(bpd); err != nil {
|
||||
ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
|
||||
}
|
||||
|
||||
c := Config{ret}
|
||||
storeConfigMetrics(ctx, c)
|
||||
return c
|
||||
|
|
@ -535,12 +522,10 @@ func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig
|
|||
|
||||
func buildConfig(config Config) *smpb.BuildConfig {
|
||||
c := &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(config.ForceUseGoma()),
|
||||
UseGoma: proto.Bool(config.UseGoma()),
|
||||
UseRbe: proto.Bool(config.UseRBE()),
|
||||
BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
|
||||
NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
|
||||
ForceUseGoma: proto.Bool(config.ForceUseGoma()),
|
||||
UseGoma: proto.Bool(config.UseGoma()),
|
||||
UseRbe: proto.Bool(config.UseRBE()),
|
||||
NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
|
||||
}
|
||||
c.Targets = append(c.Targets, config.arguments...)
|
||||
|
||||
|
|
@ -794,10 +779,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
|
|||
c.reportMkMetrics = true
|
||||
} else if arg == "--multitree-build" {
|
||||
c.multitreeBuild = true
|
||||
} else if arg == "--bazel-mode" {
|
||||
c.bazelProdMode = true
|
||||
} else if arg == "--bazel-mode-staging" {
|
||||
c.bazelStagingMode = true
|
||||
} else if arg == "--search-api-dir" {
|
||||
c.searchApiDir = true
|
||||
} else if strings.HasPrefix(arg, "--ninja_weight_source=") {
|
||||
|
|
@ -832,8 +813,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
|
|||
buildCmd = strings.TrimPrefix(buildCmd, "\"")
|
||||
buildCmd = strings.TrimSuffix(buildCmd, "\"")
|
||||
ctx.Metrics.SetBuildCommand([]string{buildCmd})
|
||||
} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
|
||||
c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
|
||||
} else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
|
||||
buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
|
||||
val, err := strconv.ParseInt(buildTimeStr, 10, 64)
|
||||
|
|
@ -878,8 +857,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
|
|||
c.dist = true
|
||||
} else if arg == "json-module-graph" {
|
||||
c.jsonModuleGraph = true
|
||||
} else if arg == "bp2build" {
|
||||
c.bp2build = true
|
||||
} else if arg == "queryview" {
|
||||
c.queryview = true
|
||||
} else if arg == "soong_docs" {
|
||||
|
|
@ -976,13 +953,12 @@ func (c *configImpl) SoongBuildInvocationNeeded() bool {
|
|||
return true
|
||||
}
|
||||
|
||||
if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() {
|
||||
if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() {
|
||||
// Command line was empty, the default Ninja target is built
|
||||
return true
|
||||
}
|
||||
|
||||
// bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
|
||||
if c.Dist() && !c.Bp2Build() {
|
||||
if c.Dist() {
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -1012,14 +988,6 @@ func (c *configImpl) NinjaArgs() []string {
|
|||
return c.ninjaArgs
|
||||
}
|
||||
|
||||
func (c *configImpl) BazelOutDir() string {
|
||||
return filepath.Join(c.OutDir(), "bazel")
|
||||
}
|
||||
|
||||
func (c *configImpl) bazelOutputBase() string {
|
||||
return filepath.Join(c.BazelOutDir(), "output")
|
||||
}
|
||||
|
||||
func (c *configImpl) SoongOutDir() string {
|
||||
return filepath.Join(c.OutDir(), "soong")
|
||||
}
|
||||
|
|
@ -1058,14 +1026,6 @@ func (c *configImpl) UsedEnvFile(tag string) string {
|
|||
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
|
||||
}
|
||||
|
||||
func (c *configImpl) Bp2BuildFilesMarkerFile() string {
|
||||
return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
|
||||
}
|
||||
|
||||
func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
|
||||
return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
|
||||
}
|
||||
|
||||
func (c *configImpl) SoongDocsHtml() string {
|
||||
return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
|
||||
}
|
||||
|
|
@ -1111,10 +1071,6 @@ func (c *configImpl) JsonModuleGraph() bool {
|
|||
return c.jsonModuleGraph
|
||||
}
|
||||
|
||||
func (c *configImpl) Bp2Build() bool {
|
||||
return c.bp2build
|
||||
}
|
||||
|
||||
func (c *configImpl) Queryview() bool {
|
||||
return c.queryview
|
||||
}
|
||||
|
|
@ -1306,7 +1262,7 @@ func (c *configImpl) canSupportRBE() bool {
|
|||
|
||||
func (c *configImpl) UseRBE() bool {
|
||||
// These alternate modes of running Soong do not use RBE / reclient.
|
||||
if c.Bp2Build() || c.Queryview() || c.JsonModuleGraph() {
|
||||
if c.Queryview() || c.JsonModuleGraph() {
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
@ -1323,10 +1279,6 @@ func (c *configImpl) UseRBE() bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func (c *configImpl) BazelBuildEnabled() bool {
|
||||
return c.bazelProdMode || c.bazelStagingMode
|
||||
}
|
||||
|
||||
func (c *configImpl) StartRBE() bool {
|
||||
if !c.UseRBE() {
|
||||
return false
|
||||
|
|
@ -1679,12 +1631,6 @@ func (c *configImpl) LogsDir() string {
|
|||
return absDir
|
||||
}
|
||||
|
||||
// BazelMetricsDir returns the <logs dir>/bazel_metrics directory
|
||||
// where the bazel profiles are located.
|
||||
func (c *configImpl) BazelMetricsDir() string {
|
||||
return filepath.Join(c.LogsDir(), "bazel_metrics")
|
||||
}
|
||||
|
||||
// MkFileMetrics returns the file path for make-related metrics.
|
||||
func (c *configImpl) MkMetrics() string {
|
||||
return filepath.Join(c.LogsDir(), "mk_metrics.pb")
|
||||
|
|
@ -1698,28 +1644,6 @@ func (c *configImpl) EmptyNinjaFile() bool {
|
|||
return c.emptyNinjaFile
|
||||
}
|
||||
|
||||
func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
|
||||
return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
|
||||
}
|
||||
|
||||
func (c *configImpl) IsPersistentBazelEnabled() bool {
|
||||
return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
|
||||
}
|
||||
|
||||
// GetBazeliskBazelVersion returns the Bazel version to use for this build,
|
||||
// or the empty string if the current canonical prod Bazel should be used.
|
||||
// This environment variable should only be set to debug the build system.
|
||||
// The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
|
||||
// handle downloading and invoking the correct Bazel binary.
|
||||
func (c *configImpl) GetBazeliskBazelVersion() string {
|
||||
value, _ := c.Environment().Get("USE_BAZEL_VERSION")
|
||||
return value
|
||||
}
|
||||
|
||||
func (c *configImpl) BazelModulesForceEnabledByFlag() string {
|
||||
return c.bazelForceEnabledModules
|
||||
}
|
||||
|
||||
func (c *configImpl) SkipMetricsUpload() bool {
|
||||
return c.skipMetricsUpload
|
||||
}
|
||||
|
|
@ -1737,10 +1661,6 @@ func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time
|
|||
return time.UnixMilli(c.buildStartedTime)
|
||||
}
|
||||
|
||||
func (c *configImpl) BazelExitCode() int32 {
|
||||
return c.bazelExitCode
|
||||
}
|
||||
|
||||
func GetMetricsUploader(topDir string, env *Environment) string {
|
||||
if p, ok := env.Get("METRICS_UPLOADER"); ok {
|
||||
metricsUploader := filepath.Join(topDir, p)
|
||||
|
|
|
|||
|
|
@ -1017,157 +1017,57 @@ func TestBuildConfig(t *testing.T) {
|
|||
name string
|
||||
environ Environment
|
||||
arguments []string
|
||||
useBazel bool
|
||||
bazelProdMode bool
|
||||
bazelStagingMode bool
|
||||
expectedBuildConfig *smpb.BuildConfig
|
||||
}{
|
||||
{
|
||||
name: "none set",
|
||||
environ: Environment{},
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "force use goma",
|
||||
environ: Environment{"FORCE_USE_GOMA=1"},
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(true),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
ForceUseGoma: proto.Bool(true),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "use goma",
|
||||
environ: Environment{"USE_GOMA=1"},
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(true),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(true),
|
||||
UseRbe: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "use rbe",
|
||||
environ: Environment{"USE_RBE=1"},
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(true),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "disable mixed builds",
|
||||
environ: Environment{"BUILD_BROKEN_DISABLE_BAZEL=1"},
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(true),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "use bazel as ninja",
|
||||
environ: Environment{},
|
||||
useBazel: true,
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bazel mixed build from prod mode",
|
||||
environ: Environment{},
|
||||
bazelProdMode: true,
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(true),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bazel mixed build from staging mode",
|
||||
environ: Environment{},
|
||||
bazelStagingMode: true,
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(true),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "specified targets",
|
||||
environ: Environment{},
|
||||
useBazel: true,
|
||||
arguments: []string{"droid", "dist"},
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(false),
|
||||
BazelMixedBuild: proto.Bool(false),
|
||||
Targets: []string{"droid", "dist"},
|
||||
ForceDisableBazelMixedBuild: proto.Bool(false),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "all set",
|
||||
environ: Environment{
|
||||
"FORCE_USE_GOMA=1",
|
||||
"USE_GOMA=1",
|
||||
"USE_RBE=1",
|
||||
"BUILD_BROKEN_DISABLE_BAZEL=1",
|
||||
},
|
||||
useBazel: true,
|
||||
bazelProdMode: true,
|
||||
expectedBuildConfig: &smpb.BuildConfig{
|
||||
ForceUseGoma: proto.Bool(true),
|
||||
UseGoma: proto.Bool(true),
|
||||
UseRbe: proto.Bool(true),
|
||||
BazelMixedBuild: proto.Bool(true),
|
||||
ForceDisableBazelMixedBuild: proto.Bool(true),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
ForceUseGoma: proto.Bool(false),
|
||||
UseGoma: proto.Bool(false),
|
||||
UseRbe: proto.Bool(true),
|
||||
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := testContext()
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := &configImpl{
|
||||
environ: &tc.environ,
|
||||
bazelProdMode: tc.bazelProdMode,
|
||||
bazelStagingMode: tc.bazelStagingMode,
|
||||
arguments: tc.arguments,
|
||||
environ: &tc.environ,
|
||||
arguments: tc.arguments,
|
||||
}
|
||||
config := Config{c}
|
||||
checkBazelMode(ctx, config)
|
||||
actualBuildConfig := buildConfig(config)
|
||||
if expected := tc.expectedBuildConfig; !proto.Equal(expected, actualBuildConfig) {
|
||||
t.Errorf("Build config mismatch.\n"+
|
||||
|
|
|
|||
|
|
@ -74,10 +74,6 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
|
|||
"AndroidProducts.mk",
|
||||
// General Soong build definitions, using the Blueprint syntax.
|
||||
"Android.bp",
|
||||
// Bazel build definitions.
|
||||
"BUILD.bazel",
|
||||
// Bazel build definitions.
|
||||
"BUILD",
|
||||
// Kati clean definitions.
|
||||
"CleanSpec.mk",
|
||||
// Ownership definition.
|
||||
|
|
@ -85,13 +81,11 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
|
|||
// Test configuration for modules in directories that contain this
|
||||
// file.
|
||||
"TEST_MAPPING",
|
||||
// Bazel top-level file to mark a directory as a Bazel workspace.
|
||||
"WORKSPACE",
|
||||
// METADATA file of packages
|
||||
"METADATA",
|
||||
},
|
||||
// Bazel Starlark configuration files and all .mk files for product/board configuration.
|
||||
IncludeSuffixes: []string{".bzl", ".mk"},
|
||||
// .mk files for product/board configuration.
|
||||
IncludeSuffixes: []string{".mk"},
|
||||
}
|
||||
dumpDir := config.FileListDir()
|
||||
f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
|
||||
|
|
@ -111,17 +105,6 @@ func androidBpSearchDirs(config Config) []string {
|
|||
return dirs
|
||||
}
|
||||
|
||||
// Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree.
|
||||
func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
|
||||
matches := []string{}
|
||||
for _, foundName := range entries.FileNames {
|
||||
if foundName == "BUILD.bazel" || foundName == "BUILD" || foundName == "WORKSPACE" || strings.HasSuffix(foundName, ".bzl") {
|
||||
matches = append(matches, foundName)
|
||||
}
|
||||
}
|
||||
return entries.DirNames, matches
|
||||
}
|
||||
|
||||
func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
|
||||
matches := []string{}
|
||||
for _, foundName := range entries.FileNames {
|
||||
|
|
@ -177,13 +160,6 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
|
|||
ctx.Fatalf("Could not export product list: %v", err)
|
||||
}
|
||||
|
||||
// Recursively look for all Bazel related files.
|
||||
bazelFiles := f.FindMatching(".", findBazelFiles)
|
||||
err = dumpListToFile(ctx, config, bazelFiles, filepath.Join(dumpDir, "bazel.list"))
|
||||
if err != nil {
|
||||
ctx.Fatalf("Could not export bazel BUILD list: %v", err)
|
||||
}
|
||||
|
||||
// Recursively look for all OWNERS files.
|
||||
owners := f.FindNamedAt(".", "OWNERS")
|
||||
err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list"))
|
||||
|
|
|
|||
|
|
@ -44,12 +44,10 @@ const (
|
|||
availableEnvFile = "soong.environment.available"
|
||||
usedEnvFile = "soong.environment.used"
|
||||
|
||||
soongBuildTag = "build"
|
||||
bp2buildFilesTag = "bp2build_files"
|
||||
bp2buildWorkspaceTag = "bp2build_workspace"
|
||||
jsonModuleGraphTag = "modulegraph"
|
||||
queryviewTag = "queryview"
|
||||
soongDocsTag = "soong_docs"
|
||||
soongBuildTag = "build"
|
||||
jsonModuleGraphTag = "modulegraph"
|
||||
queryviewTag = "queryview"
|
||||
soongDocsTag = "soong_docs"
|
||||
|
||||
// bootstrapEpoch is used to determine if an incremental build is incompatible with the current
|
||||
// version of bootstrap and needs cleaning before continuing the build. Increment this for
|
||||
|
|
@ -275,7 +273,6 @@ func bootstrapEpochCleanup(ctx Context, config Config) {
|
|||
func bootstrapGlobFileList(config Config) []string {
|
||||
return []string{
|
||||
config.NamedGlobFile(getGlobPathName(config)),
|
||||
config.NamedGlobFile(bp2buildFilesTag),
|
||||
config.NamedGlobFile(jsonModuleGraphTag),
|
||||
config.NamedGlobFile(queryviewTag),
|
||||
config.NamedGlobFile(soongDocsTag),
|
||||
|
|
@ -295,18 +292,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
|
|||
if config.EmptyNinjaFile() {
|
||||
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
|
||||
}
|
||||
if config.bazelProdMode {
|
||||
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
|
||||
}
|
||||
if config.bazelStagingMode {
|
||||
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
|
||||
}
|
||||
if config.IsPersistentBazelEnabled() {
|
||||
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
|
||||
}
|
||||
if len(config.bazelForceEnabledModules) > 0 {
|
||||
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
|
||||
}
|
||||
if config.MultitreeBuild() {
|
||||
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build")
|
||||
}
|
||||
|
|
@ -327,24 +312,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
|
|||
output: config.SoongNinjaFile(),
|
||||
specificArgs: mainSoongBuildExtraArgs,
|
||||
},
|
||||
{
|
||||
name: bp2buildFilesTag,
|
||||
description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
|
||||
config: config,
|
||||
output: config.Bp2BuildFilesMarkerFile(),
|
||||
specificArgs: append(baseArgs,
|
||||
"--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: bp2buildWorkspaceTag,
|
||||
description: "Creating Bazel symlink forest",
|
||||
config: config,
|
||||
output: config.Bp2BuildWorkspaceMarkerFile(),
|
||||
specificArgs: append(baseArgs,
|
||||
"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: jsonModuleGraphTag,
|
||||
description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
|
||||
|
|
@ -410,22 +377,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
|
|||
pbf.debugPort = delvePort
|
||||
}
|
||||
pbi := pbf.primaryBuilderInvocation()
|
||||
// Some invocations require adjustment:
|
||||
switch pbf.name {
|
||||
case soongBuildTag:
|
||||
if config.BazelBuildEnabled() {
|
||||
// Mixed builds call Bazel from soong_build and they therefore need the
|
||||
// Bazel workspace to be available. Make that so by adding a dependency on
|
||||
// the bp2build marker file to the action that invokes soong_build .
|
||||
pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
|
||||
}
|
||||
case bp2buildWorkspaceTag:
|
||||
pbi.Inputs = append(pbi.Inputs,
|
||||
config.Bp2BuildFilesMarkerFile(),
|
||||
filepath.Join(config.FileListDir(), "bazel.list"))
|
||||
case bp2buildFilesTag:
|
||||
pbi.Inputs = append(pbi.Inputs, filepath.Join(config.FileListDir(), "METADATA.list"))
|
||||
}
|
||||
invocations = append(invocations, pbi)
|
||||
}
|
||||
|
||||
|
|
@ -603,16 +554,7 @@ func runSoong(ctx Context, config Config) {
|
|||
|
||||
soongBuildEnv := config.Environment().Copy()
|
||||
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
|
||||
// For Bazel mixed builds.
|
||||
soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
|
||||
// Bazel's HOME var is set to an output subdirectory which doesn't exist. This
|
||||
// prevents Bazel from file I/O in the actual user HOME directory.
|
||||
soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
|
||||
soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
|
||||
soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
|
||||
soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
|
||||
soongBuildEnv.Set("LOG_DIR", config.LogsDir())
|
||||
soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
|
||||
|
||||
// For Soong bootstrapping tests
|
||||
if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
|
||||
|
|
@ -630,13 +572,9 @@ func runSoong(ctx Context, config Config) {
|
|||
|
||||
checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag))
|
||||
|
||||
if config.BazelBuildEnabled() || config.Bp2Build() {
|
||||
checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
|
||||
} else {
|
||||
// Remove bazel files in the event that bazel is disabled for the build.
|
||||
// These files may have been left over from a previous bazel-enabled build.
|
||||
cleanBazelFiles(config)
|
||||
}
|
||||
// Remove bazel files in the event that bazel is disabled for the build.
|
||||
// These files may have been left over from a previous bazel-enabled build.
|
||||
cleanBazelFiles(config)
|
||||
|
||||
if config.JsonModuleGraph() {
|
||||
checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
|
||||
|
|
@ -658,14 +596,6 @@ func runSoong(ctx Context, config Config) {
|
|||
ctx.BeginTrace(metrics.RunSoong, "bootstrap")
|
||||
defer ctx.EndTrace()
|
||||
|
||||
if config.IsPersistentBazelEnabled() {
|
||||
bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"), config.GetBazeliskBazelVersion())
|
||||
if err := bazelProxy.Start(); err != nil {
|
||||
ctx.Fatalf("Failed to create bazel proxy")
|
||||
}
|
||||
defer bazelProxy.Close()
|
||||
}
|
||||
|
||||
fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
|
||||
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
|
||||
defer nr.Close()
|
||||
|
|
@ -709,10 +639,6 @@ func runSoong(ctx Context, config Config) {
|
|||
targets = append(targets, config.ModuleGraphFile())
|
||||
}
|
||||
|
||||
if config.Bp2Build() {
|
||||
targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
|
||||
}
|
||||
|
||||
if config.Queryview() {
|
||||
targets = append(targets, config.QueryviewMarkerFile())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,47 +15,16 @@
|
|||
package build
|
||||
|
||||
import (
|
||||
"android/soong/ui/metrics"
|
||||
"android/soong/ui/status"
|
||||
"bufio"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"android/soong/ui/metrics"
|
||||
"android/soong/ui/status"
|
||||
)
|
||||
|
||||
var (
|
||||
// bazel output paths are in __main__/bazel-out/<config-specific-path>/bin
|
||||
bazelOutputPathRegexOnce sync.Once
|
||||
bazelOutputPathRegexp *regexp.Regexp
|
||||
)
|
||||
|
||||
func bazelOutputPathPattern(config Config) *regexp.Regexp {
|
||||
bazelOutputPathRegexOnce.Do(func() {
|
||||
// Bazel output files are in <Bazel output base>/execroot/__main__/bazel-out/<config>/bin
|
||||
bazelOutRoot := filepath.Join(regexp.QuoteMeta(config.bazelOutputBase()), "execroot", "__main__", "bazel-out")
|
||||
bazelOutputPathRegexp = regexp.MustCompile(bazelOutRoot + "/[^/]+/bin")
|
||||
})
|
||||
return bazelOutputPathRegexp
|
||||
}
|
||||
|
||||
func ignoreBazelPath(config Config, path string) bool {
|
||||
bazelRoot := filepath.Join(config.bazelOutputBase(), "execroot")
|
||||
// Don't check bazel output regexp unless it is Bazel path
|
||||
if strings.HasPrefix(path, bazelRoot) {
|
||||
bazelOutputRegexp := bazelOutputPathPattern(config)
|
||||
// if the file is a bazel path that is _not_ a Bazel generated file output, we rely on Bazel to
|
||||
// ensure the paths to exist. If it _is_ a Bazel output path, we expect that it should be built
|
||||
// by Ninja.
|
||||
return !bazelOutputRegexp.MatchString(path)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Checks for files in the out directory that have a rule that depends on them but no rule to
|
||||
// create them. This catches a common set of build failures where a rule to generate a file is
|
||||
// deleted (either by deleting a module in an Android.mk file, or by modifying the build system
|
||||
|
|
@ -128,9 +97,6 @@ func testForDanglingRules(ctx Context, config Config) {
|
|||
continue
|
||||
}
|
||||
|
||||
if ignoreBazelPath(config, line) {
|
||||
continue
|
||||
}
|
||||
danglingRules[line] = true
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,299 +0,0 @@
|
|||
// Copyright 2020 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package build
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"android/soong/ui/logger"
|
||||
)
|
||||
|
||||
func writeBazelProfileFile(dir string) error {
|
||||
contents := `
|
||||
|
||||
=== PHASE SUMMARY INFORMATION ===
|
||||
|
||||
Total launch phase time 1.193 s 15.77%
|
||||
Total init phase time 1.092 s 14.44%
|
||||
Total target pattern evaluation phase time 0.580 s 7.67%
|
||||
Total interleaved loading-and-analysis phase time 3.646 s 48.21%
|
||||
Total preparation phase time 0.022 s 0.30%
|
||||
Total execution phase time 0.993 s 13.13%
|
||||
Total finish phase time 0.036 s 0.48%
|
||||
---------------------------------------------------------------------
|
||||
Total run time 7.563 s 100.00%
|
||||
|
||||
Critical path (178 ms):
|
||||
Time Percentage Description
|
||||
178 ms 100.00% action 'BazelWorkspaceStatusAction stable-status.txt'
|
||||
|
||||
`
|
||||
file := filepath.Join(dir, "bazel_metrics.txt")
|
||||
return os.WriteFile(file, []byte(contents), 0666)
|
||||
}
|
||||
|
||||
func TestPruneMetricsFiles(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
|
||||
dirs := []string{
|
||||
filepath.Join(rootDir, "d1"),
|
||||
filepath.Join(rootDir, "d1", "d2"),
|
||||
filepath.Join(rootDir, "d1", "d2", "d3"),
|
||||
}
|
||||
|
||||
files := []string{
|
||||
filepath.Join(rootDir, "d1", "f1"),
|
||||
filepath.Join(rootDir, "d1", "d2", "f1"),
|
||||
filepath.Join(rootDir, "d1", "d2", "d3", "f1"),
|
||||
}
|
||||
|
||||
for _, d := range dirs {
|
||||
if err := os.MkdirAll(d, 0777); err != nil {
|
||||
t.Fatalf("got %v, expecting nil error for making directory %q", err, d)
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
if err := ioutil.WriteFile(f, []byte{}, 0777); err != nil {
|
||||
t.Fatalf("got %v, expecting nil error on writing file %q", err, f)
|
||||
}
|
||||
}
|
||||
|
||||
want := []string{
|
||||
filepath.Join(rootDir, "d1", "f1"),
|
||||
filepath.Join(rootDir, "d1", "d2", "f1"),
|
||||
filepath.Join(rootDir, "d1", "d2", "d3", "f1"),
|
||||
}
|
||||
|
||||
got := pruneMetricsFiles([]string{rootDir})
|
||||
|
||||
sort.Strings(got)
|
||||
sort.Strings(want)
|
||||
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %q, want %q after pruning metrics files", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMetrics(t *testing.T) {
|
||||
ctx := testContext()
|
||||
tests := []struct {
|
||||
description string
|
||||
uploader string
|
||||
createFiles bool
|
||||
files []string
|
||||
}{{
|
||||
description: "no metrics uploader",
|
||||
}, {
|
||||
description: "non-existent metrics files no upload",
|
||||
uploader: "echo",
|
||||
files: []string{"metrics_file_1", "metrics_file_2", "metrics_file_3, bazel_metrics.pb"},
|
||||
}, {
|
||||
description: "trigger upload",
|
||||
uploader: "echo",
|
||||
createFiles: true,
|
||||
files: []string{"metrics_file_1", "metrics_file_2, bazel_metrics.pb"},
|
||||
}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.description, func(t *testing.T) {
|
||||
defer logger.Recover(func(err error) {
|
||||
t.Fatalf("got unexpected error: %v", err)
|
||||
})
|
||||
|
||||
outDir, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create out directory: %v", outDir)
|
||||
}
|
||||
defer os.RemoveAll(outDir)
|
||||
|
||||
// Supply our own tmpDir to delete the temp dir once the test is done.
|
||||
orgTmpDir := tmpDir
|
||||
tmpDir = func(string, string) (string, error) {
|
||||
retDir := filepath.Join(outDir, "tmp_upload_dir")
|
||||
if err := os.Mkdir(retDir, 0755); err != nil {
|
||||
t.Fatalf("failed to create temporary directory %q: %v", retDir, err)
|
||||
}
|
||||
return retDir, nil
|
||||
}
|
||||
defer func() { tmpDir = orgTmpDir }()
|
||||
|
||||
metricsUploadDir := filepath.Join(outDir, ".metrics_uploader")
|
||||
if err := os.Mkdir(metricsUploadDir, 0755); err != nil {
|
||||
t.Fatalf("failed to create %q directory for oauth valid check: %v", metricsUploadDir, err)
|
||||
}
|
||||
|
||||
var metricsFiles []string
|
||||
if tt.createFiles {
|
||||
for _, f := range tt.files {
|
||||
filename := filepath.Join(outDir, f)
|
||||
metricsFiles = append(metricsFiles, filename)
|
||||
if err := ioutil.WriteFile(filename, []byte("test file"), 0644); err != nil {
|
||||
t.Fatalf("failed to create a fake metrics file %q for uploading: %v", filename, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := writeBazelProfileFile(outDir); err != nil {
|
||||
t.Fatalf("failed to create bazel profile file in dir: %v", outDir)
|
||||
}
|
||||
|
||||
config := Config{&configImpl{
|
||||
environ: &Environment{
|
||||
"OUT_DIR=" + outDir,
|
||||
},
|
||||
buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10),
|
||||
metricsUploader: tt.uploader,
|
||||
}}
|
||||
|
||||
UploadMetrics(ctx, config, false, time.Now(), metricsFiles...)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMetricsErrors(t *testing.T) {
|
||||
ctx := testContext()
|
||||
tests := []struct {
|
||||
description string
|
||||
tmpDir string
|
||||
tmpDirErr error
|
||||
expectedErr string
|
||||
}{{
|
||||
description: "getTmpDir returned error",
|
||||
tmpDirErr: errors.New("getTmpDir failed"),
|
||||
expectedErr: "getTmpDir failed",
|
||||
}, {
|
||||
description: "copyFile operation error",
|
||||
tmpDir: "/fake_dir",
|
||||
expectedErr: "failed to copy",
|
||||
}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.description, func(t *testing.T) {
|
||||
defer logger.Recover(func(err error) {
|
||||
got := err.Error()
|
||||
if !strings.Contains(got, tt.expectedErr) {
|
||||
t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr)
|
||||
}
|
||||
})
|
||||
|
||||
outDir, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create out directory: %v", outDir)
|
||||
}
|
||||
defer os.RemoveAll(outDir)
|
||||
|
||||
orgTmpDir := tmpDir
|
||||
tmpDir = func(string, string) (string, error) {
|
||||
return tt.tmpDir, tt.tmpDirErr
|
||||
}
|
||||
defer func() { tmpDir = orgTmpDir }()
|
||||
|
||||
metricsFile := filepath.Join(outDir, "metrics_file_1")
|
||||
if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil {
|
||||
t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err)
|
||||
}
|
||||
|
||||
config := Config{&configImpl{
|
||||
environ: &Environment{
|
||||
"OUT_DIR=/bad",
|
||||
},
|
||||
metricsUploader: "echo",
|
||||
}}
|
||||
|
||||
UploadMetrics(ctx, config, true, time.Now(), metricsFile)
|
||||
t.Errorf("got nil, expecting %q as a failure", tt.expectedErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePercentageToTenThousandths(t *testing.T) {
|
||||
// 2.59% should be returned as 259 - representing 259/10000 of the build
|
||||
percentage := parsePercentageToTenThousandths("2.59%")
|
||||
if percentage != 259 {
|
||||
t.Errorf("Expected percentage to be returned as ten-thousandths. Expected 259, have %d\n", percentage)
|
||||
}
|
||||
|
||||
// Test without a leading digit
|
||||
percentage = parsePercentageToTenThousandths(".52%")
|
||||
if percentage != 52 {
|
||||
t.Errorf("Expected percentage to be returned as ten-thousandths. Expected 52, have %d\n", percentage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTimingToNanos(t *testing.T) {
|
||||
// This parses from seconds (with millis precision) and returns nanos
|
||||
timingNanos := parseTimingToNanos("0.111")
|
||||
if timingNanos != 111000000 {
|
||||
t.Errorf("Error parsing timing. Expected 111000, have %d\n", timingNanos)
|
||||
}
|
||||
|
||||
// Test without a leading digit
|
||||
timingNanos = parseTimingToNanos(".112")
|
||||
if timingNanos != 112000000 {
|
||||
t.Errorf("Error parsing timing. Expected 112000, have %d\n", timingNanos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePhaseTiming(t *testing.T) {
|
||||
// Sample lines include:
|
||||
// Total launch phase time 0.011 s 2.59%
|
||||
// Total target pattern evaluation phase time 0.012 s 4.59%
|
||||
|
||||
line1 := "Total launch phase time 0.011 s 2.59%"
|
||||
timing := parsePhaseTiming(line1)
|
||||
|
||||
if timing.GetPhaseName() != "launch" {
|
||||
t.Errorf("Failed to parse phase name. Expected launch, have %s\n", timing.GetPhaseName())
|
||||
} else if timing.GetDurationNanos() != 11000000 {
|
||||
t.Errorf("Failed to parse duration nanos. Expected 11000000, have %d\n", timing.GetDurationNanos())
|
||||
} else if timing.GetPortionOfBuildTime() != 259 {
|
||||
t.Errorf("Failed to parse portion of build time. Expected 259, have %d\n", timing.GetPortionOfBuildTime())
|
||||
}
|
||||
|
||||
// Test with a multiword phase name
|
||||
line2 := "Total target pattern evaluation phase time 0.012 s 4.59%"
|
||||
|
||||
timing = parsePhaseTiming(line2)
|
||||
if timing.GetPhaseName() != "target pattern evaluation" {
|
||||
t.Errorf("Failed to parse phase name. Expected target pattern evaluation, have %s\n", timing.GetPhaseName())
|
||||
} else if timing.GetDurationNanos() != 12000000 {
|
||||
t.Errorf("Failed to parse duration nanos. Expected 12000000, have %d\n", timing.GetDurationNanos())
|
||||
} else if timing.GetPortionOfBuildTime() != 459 {
|
||||
t.Errorf("Failed to parse portion of build time. Expected 459, have %d\n", timing.GetPortionOfBuildTime())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTotal(t *testing.T) {
|
||||
// Total line is in the form of:
|
||||
// Total run time 7.563 s 100.00%
|
||||
|
||||
line := "Total run time 7.563 s 100.00%"
|
||||
|
||||
total := parseTotal(line)
|
||||
|
||||
// Only the seconds field is parsed, as nanos
|
||||
if total != 7563000000 {
|
||||
t.Errorf("Failed to parse total build time. Expected 7563000000, have %d\n", total)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue