tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Report mkfile metrics with --mk-metrics"

Browse source
This commit is contained in:
Christopher Parsons 2022-03-03 21:51:29 +00:00 committed by Gerrit Code Review
commit cb59ea7b2f
7 changed files with 300 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
ui/build/config.go
View file

@ -35,10 +35,10 @@ import (
)
const (
envConfigDir = "vendor/google/tools/soong_config"
jsonSuffix = "json"
envConfigDir = "vendor/google/tools/soong_config"
jsonSuffix = "json"
configFetcher = "vendor/google/tools/soong/expconfigfetcher"
configFetcher = "vendor/google/tools/soong/expconfigfetcher"
envConfigFetchTimeout = 10 * time.Second
)
@ -62,6 +62,7 @@ type configImpl struct {
jsonModuleGraph bool
bp2build bool
queryview bool
reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
soongDocs bool
skipConfig bool
skipKati bool
@ -155,7 +156,7 @@ func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error
}
configExists := false
outConfigFilePath := filepath.Join(config.OutDir(), envConfigName + jsonSuffix)
outConfigFilePath := filepath.Join(config.OutDir(), envConfigName+jsonSuffix)
if _, err := os.Stat(outConfigFilePath); err == nil {
configExists = true
}
@ -711,6 +712,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
c.skipConfig = true
} else if arg == "--skip-soong-tests" {
c.skipSoongTests = true
} else if arg == "--mk-metrics" {
c.reportMkMetrics = true
} else if len(arg) > 0 && arg[0] == '-' {
parseArgNum := func(def int) int {
if len(arg) > 2 {
@ -1381,6 +1384,11 @@ func (c *configImpl) BazelMetricsDir() string {
return filepath.Join(c.LogsDir(), "bazel_metrics")
}
// MkFileMetrics returns the file path for make-related metrics.
func (c *configImpl) MkMetrics() string {
return filepath.Join(c.LogsDir(), "mk_metrics.pb")
}
func (c *configImpl) SetEmptyNinjaFile(v bool) {
c.emptyNinjaFile = v
}

11
ui/build/finder.go
View file

@ -138,6 +138,17 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
ctx.Fatalf("Could not export module list: %v", err)
}
// Gate collecting/reporting mk metrics on builds that specifically request
// it, as identifying the total number of mk files adds 4-5ms onto null
// builds.
if config.reportMkMetrics {
androidMksTotal := f.FindNamedAt(".", "Android.mk")
ctx.Metrics.SetToplevelMakefiles(len(androidMks))
ctx.Metrics.SetTotalMakefiles(len(androidMksTotal))
ctx.Metrics.DumpMkMetrics(config.MkMetrics())
}
// Stop searching a subdirectory recursively after finding a CleanSpec.mk.
cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))

15
ui/metrics/Android.bp
View file

@ -21,9 +21,10 @@ bootstrap_go_package {
pkgPath: "android/soong/ui/metrics",
deps: [
"golang-protobuf-proto",
"soong-ui-bp2build_metrics_proto",
"soong-ui-metrics_upload_proto",
"soong-ui-metrics_proto",
"soong-ui-bp2build_metrics_proto",
"soong-ui-mk_metrics_proto",
"soong-ui-tracer",
"soong-shared",
],
@ -71,3 +72,15 @@ bootstrap_go_package {
"bp2build_metrics_proto/bp2build_metrics.pb.go",
],
}
bootstrap_go_package {
name: "soong-ui-mk_metrics_proto",
pkgPath: "android/soong/ui/metrics/mk_metrics_proto",
deps: [
"golang-protobuf-reflect-protoreflect",
"golang-protobuf-runtime-protoimpl",
],
srcs: [
"mk_metrics_proto/mk_metrics.pb.go",
],
}

32
ui/metrics/metrics.go
View file

@ -41,6 +41,7 @@ import (
"google.golang.org/protobuf/proto"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
mk_metrics_proto "android/soong/ui/metrics/mk_metrics_proto"
)
const (
@ -62,14 +63,22 @@ const (
Total = "total"
)
// Metrics is a struct that stores collected metrics during the course
// of a build which later is dumped to a MetricsBase protobuf file.
// See ui/metrics/metrics_proto/metrics.proto for further details
// on what information is collected.
// Metrics is a struct that stores collected metrics during the course of a
// build. It is later dumped to protobuf files. See underlying metrics protos
// for further details on what information is collected.
type Metrics struct {
// The protobuf message that is later written to the file.
// Protobuf containing various top-level build metrics. These include:
// 1. Build identifiers (ex: branch ID, requested product, hostname,
// originating command)
// 2. Per-subprocess top-level metrics (ex: ninja process IO and runtime).
// Note that, since these metrics are reported by soong_ui, there is little
// insight that can be provided into performance breakdowns of individual
// subprocesses.
metrics soong_metrics_proto.MetricsBase
// Protobuf containing metrics pertaining to number of makefiles in a build.
mkMetrics mk_metrics_proto.MkMetrics
// A list of pending build events.
EventTracer *EventTracer
}
@ -78,11 +87,24 @@ type Metrics struct {
func New() (metrics *Metrics) {
m := &Metrics{
metrics: soong_metrics_proto.MetricsBase{},
mkMetrics: mk_metrics_proto.MkMetrics{},
EventTracer: &EventTracer{},
}
return m
}
func (m *Metrics) SetTotalMakefiles(total int) {
m.mkMetrics.TotalMakefiles = uint32(total)
}
func (m *Metrics) SetToplevelMakefiles(total int) {
m.mkMetrics.ToplevelMakefiles = uint32(total)
}
func (m *Metrics) DumpMkMetrics(outPath string) {
shared.Save(&m.mkMetrics, outPath)
}
// SetTimeMetrics stores performance information from an executed block of
// code.
func (m *Metrics) SetTimeMetrics(perf soong_metrics_proto.PerfInfo) {

177
ui/metrics/mk_metrics_proto/mk_metrics.pb.go Normal file
View file

@ -0,0 +1,177 @@
// Copyright 2022 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc v3.9.1
// source: mk_metrics.proto
package mk_metrics_proto
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// Contains metrics pertaining to makefiles.
type MkMetrics struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Total number of mk files present in the workspace.
TotalMakefiles uint32 `protobuf:"varint,1,opt,name=totalMakefiles,proto3" json:"totalMakefiles,omitempty"`
// Number of top-level mk files present in the workspace.
// A mk file is "top level" if there are no mk files in its parent
// direrctories.
// This value is equivalent to the number of entries in Android.mk.list.
ToplevelMakefiles uint32 `protobuf:"varint,2,opt,name=toplevelMakefiles,proto3" json:"toplevelMakefiles,omitempty"`
}
func (x *MkMetrics) Reset() {
*x = MkMetrics{}
if protoimpl.UnsafeEnabled {
mi := &file_mk_metrics_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *MkMetrics) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MkMetrics) ProtoMessage() {}
func (x *MkMetrics) ProtoReflect() protoreflect.Message {
mi := &file_mk_metrics_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MkMetrics.ProtoReflect.Descriptor instead.
func (*MkMetrics) Descriptor() ([]byte, []int) {
return file_mk_metrics_proto_rawDescGZIP(), []int{0}
}
func (x *MkMetrics) GetTotalMakefiles() uint32 {
if x != nil {
return x.TotalMakefiles
}
return 0
}
func (x *MkMetrics) GetToplevelMakefiles() uint32 {
if x != nil {
return x.ToplevelMakefiles
}
return 0
}
var File_mk_metrics_proto protoreflect.FileDescriptor
var file_mk_metrics_proto_rawDesc = []byte{
0x0a, 0x10, 0x6d, 0x6b, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x12, 0x16, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f,
0x6d, 0x6b, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0x61, 0x0a, 0x09, 0x4d, 0x6b,
0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c,
0x4d, 0x61, 0x6b, 0x65, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52,
0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x61, 0x6b, 0x65, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12,
0x2c, 0x0a, 0x11, 0x74, 0x6f, 0x70, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x4d, 0x61, 0x6b, 0x65, 0x66,
0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x74, 0x6f, 0x70, 0x6c,
0x65, 0x76, 0x65, 0x6c, 0x4d, 0x61, 0x6b, 0x65, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x42, 0x2b, 0x5a,
0x29, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75,
0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x6d, 0x6b, 0x5f, 0x6d, 0x65, 0x74,
0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_mk_metrics_proto_rawDescOnce sync.Once
file_mk_metrics_proto_rawDescData = file_mk_metrics_proto_rawDesc
)
func file_mk_metrics_proto_rawDescGZIP() []byte {
file_mk_metrics_proto_rawDescOnce.Do(func() {
file_mk_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_mk_metrics_proto_rawDescData)
})
return file_mk_metrics_proto_rawDescData
}
var file_mk_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_mk_metrics_proto_goTypes = []interface{}{
(*MkMetrics)(nil), // 0: soong_build_mk_metrics.MkMetrics
}
var file_mk_metrics_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_mk_metrics_proto_init() }
func file_mk_metrics_proto_init() {
if File_mk_metrics_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_mk_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*MkMetrics); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_mk_metrics_proto_rawDesc,
NumEnums: 0,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_mk_metrics_proto_goTypes,
DependencyIndexes: file_mk_metrics_proto_depIdxs,
MessageInfos: file_mk_metrics_proto_msgTypes,
}.Build()
File_mk_metrics_proto = out.File
file_mk_metrics_proto_rawDesc = nil
file_mk_metrics_proto_goTypes = nil
file_mk_metrics_proto_depIdxs = nil
}

30
ui/metrics/mk_metrics_proto/mk_metrics.proto Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2022 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package soong_build_mk_metrics;
option go_package = "android/soong/ui/metrics/mk_metrics_proto";
// Contains metrics pertaining to makefiles.
message MkMetrics {
// Total number of mk files present in the workspace.
uint32 totalMakefiles = 1;
// Number of top-level mk files present in the workspace.
// A mk file is "top level" if there are no mk files in its parent
// direrctories.
// This value is equivalent to the number of entries in Android.mk.list.
uint32 toplevelMakefiles = 2;
}

29
ui/metrics/mk_metrics_proto/regen.sh Executable file
View file

@ -0,0 +1,29 @@
#!/bin/bash -e
# Copyright 2022 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Generates the golang source file of the mk_metrics.proto protobuf file.
function die() { echo "ERROR: $1" >&2; exit 1; }
readonly error_msg="Maybe you need to run 'lunch aosp_arm-eng && m aprotoc blueprint_tools'?"
if ! hash aprotoc &>/dev/null; then
die "could not find aprotoc. ${error_msg}"
fi
if ! aprotoc --go_out=paths=source_relative:. mk_metrics.proto; then
die "build failed. ${error_msg}"
fi