From 74b43998e75b4ce825e12c5907a8588159dea684 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Wed, 11 Mar 2020 08:21:05 -0700 Subject: [PATCH] Add build progress in Soong UI Soong UI generates a proto file named build_progress.pb in $(OUT_DIR) output directory that contains build action numbers (how many are executing, finished and total) during the course of a build. This is for external systems that invokes the Platform Build Systems and would like to know the completion status. Bug: b/150401146 Test: Wrote a bash script that continuously read the build_progress.pb file and computed the build completed percentage while building the aosp_arm-eng target. Compared the percentage between the Soong output console and the one reported by the bash script. Change-Id: I7c7347bc8e41958093892d8e2731c4f4169937dd --- cmd/soong_ui/main.go | 1 + ui/status/Android.bp | 10 ++ .../build_progress_proto/build_progress.pb.go | 115 ++++++++++++++++++ .../build_progress.proto} | 9 +- .../regen.sh | 2 +- .../completion_proto/build_completion.pb.go | 105 ---------------- ui/status/log.go | 74 +++++++++++ 7 files changed, 207 insertions(+), 109 deletions(-) create mode 100644 ui/status/build_progress_proto/build_progress.pb.go rename ui/status/{completion_proto/build_completion.proto => build_progress_proto/build_progress.proto} (87%) rename ui/status/{completion_proto => build_progress_proto}/regen.sh (82%) delete mode 100644 ui/status/completion_proto/build_completion.pb.go diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index db61fba5c..1d94f0218 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -173,6 +173,7 @@ func main() { stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log"))) stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error"))) stat.AddOutput(status.NewCriticalPath(log)) + stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb"))) buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024)) buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v", diff --git a/ui/status/Android.bp b/ui/status/Android.bp index ec929b342..19e5a2ad3 100644 --- a/ui/status/Android.bp +++ b/ui/status/Android.bp @@ -20,6 +20,7 @@ bootstrap_go_package { "soong-ui-logger", "soong-ui-status-ninja_frontend", "soong-ui-status-build_error_proto", + "soong-ui-status-build_progress_proto", ], srcs: [ "critical_path.go", @@ -53,3 +54,12 @@ bootstrap_go_package { "build_error_proto/build_error.pb.go", ], } + +bootstrap_go_package { + name: "soong-ui-status-build_progress_proto", + pkgPath: "android/soong/ui/status/build_progress_proto", + deps: ["golang-protobuf-proto"], + srcs: [ + "build_progress_proto/build_progress.pb.go", + ], +} diff --git a/ui/status/build_progress_proto/build_progress.pb.go b/ui/status/build_progress_proto/build_progress.pb.go new file mode 100644 index 000000000..f63c157b7 --- /dev/null +++ b/ui/status/build_progress_proto/build_progress.pb.go @@ -0,0 +1,115 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: build_progress.proto + +package soong_build_progress_proto + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type BuildProgress struct { + // Total number of actions in a build. The total actions will increase + // and might decrease during the course of a build. + TotalActions *uint64 `protobuf:"varint,1,opt,name=total_actions,json=totalActions" json:"total_actions,omitempty"` + // Total number of completed build actions. This value will never decrease + // and finished_actions <= total_actions. At one point of the build, the + // finished_actions will be equal to total_actions. This may not represent + // that the build is completed as the total_actions may be increased for + // additional counted work or is doing non-counted work. + FinishedActions *uint64 `protobuf:"varint,2,opt,name=finished_actions,json=finishedActions" json:"finished_actions,omitempty"` + // Total number of current actions being executed during a course of a + // build and current_actions + finished_actions <= total_actions. + CurrentActions *uint64 `protobuf:"varint,3,opt,name=current_actions,json=currentActions" json:"current_actions,omitempty"` + // Total number of actions that reported as a failure. + FailedActions *uint64 `protobuf:"varint,4,opt,name=failed_actions,json=failedActions" json:"failed_actions,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildProgress) Reset() { *m = BuildProgress{} } +func (m *BuildProgress) String() string { return proto.CompactTextString(m) } +func (*BuildProgress) ProtoMessage() {} +func (*BuildProgress) Descriptor() ([]byte, []int) { + return fileDescriptor_a8a463f8e30dab2e, []int{0} +} + +func (m *BuildProgress) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildProgress.Unmarshal(m, b) +} +func (m *BuildProgress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildProgress.Marshal(b, m, deterministic) +} +func (m *BuildProgress) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildProgress.Merge(m, src) +} +func (m *BuildProgress) XXX_Size() int { + return xxx_messageInfo_BuildProgress.Size(m) +} +func (m *BuildProgress) XXX_DiscardUnknown() { + xxx_messageInfo_BuildProgress.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildProgress proto.InternalMessageInfo + +func (m *BuildProgress) GetTotalActions() uint64 { + if m != nil && m.TotalActions != nil { + return *m.TotalActions + } + return 0 +} + +func (m *BuildProgress) GetFinishedActions() uint64 { + if m != nil && m.FinishedActions != nil { + return *m.FinishedActions + } + return 0 +} + +func (m *BuildProgress) GetCurrentActions() uint64 { + if m != nil && m.CurrentActions != nil { + return *m.CurrentActions + } + return 0 +} + +func (m *BuildProgress) GetFailedActions() uint64 { + if m != nil && m.FailedActions != nil { + return *m.FailedActions + } + return 0 +} + +func init() { + proto.RegisterType((*BuildProgress)(nil), "soong_build_progress.BuildProgress") +} + +func init() { proto.RegisterFile("build_progress.proto", fileDescriptor_a8a463f8e30dab2e) } + +var fileDescriptor_a8a463f8e30dab2e = []byte{ + // 165 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x49, 0x2a, 0xcd, 0xcc, + 0x49, 0x89, 0x2f, 0x28, 0xca, 0x4f, 0x2f, 0x4a, 0x2d, 0x2e, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, + 0x17, 0x12, 0x29, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, 0x47, 0x95, 0x53, 0x5a, 0xcf, 0xc8, 0xc5, 0xeb, + 0x04, 0x12, 0x0a, 0x80, 0x8a, 0x08, 0x29, 0x73, 0xf1, 0x96, 0xe4, 0x97, 0x24, 0xe6, 0xc4, 0x27, + 0x26, 0x97, 0x64, 0xe6, 0xe7, 0x15, 0x4b, 0x30, 0x2a, 0x30, 0x6a, 0xb0, 0x04, 0xf1, 0x80, 0x05, + 0x1d, 0x21, 0x62, 0x42, 0x9a, 0x5c, 0x02, 0x69, 0x99, 0x79, 0x99, 0xc5, 0x19, 0xa9, 0x29, 0x70, + 0x75, 0x4c, 0x60, 0x75, 0xfc, 0x30, 0x71, 0x98, 0x52, 0x75, 0x2e, 0xfe, 0xe4, 0xd2, 0xa2, 0xa2, + 0xd4, 0xbc, 0x12, 0xb8, 0x4a, 0x66, 0xb0, 0x4a, 0x3e, 0xa8, 0x30, 0x4c, 0xa1, 0x2a, 0x17, 0x5f, + 0x5a, 0x62, 0x66, 0x0e, 0x92, 0x89, 0x2c, 0x60, 0x75, 0xbc, 0x10, 0x51, 0xa8, 0x32, 0x27, 0x99, + 0x28, 0x29, 0x6c, 0x3e, 0x89, 0x07, 0xfb, 0x12, 0x10, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x6e, 0xc1, + 0xef, 0xfc, 0x00, 0x00, 0x00, +} diff --git a/ui/status/completion_proto/build_completion.proto b/ui/status/build_progress_proto/build_progress.proto similarity index 87% rename from ui/status/completion_proto/build_completion.proto rename to ui/status/build_progress_proto/build_progress.proto index 939545ed9..d78060aa1 100644 --- a/ui/status/completion_proto/build_completion.proto +++ b/ui/status/build_progress_proto/build_progress.proto @@ -14,10 +14,10 @@ syntax = "proto2"; -package soong_build_completion_status; -option go_package = "soong_build_completion_status_proto"; +package soong_build_progress; +option go_package = "soong_build_progress_proto"; -message BuildCompletionStatus { +message BuildProgress { // Total number of actions in a build. The total actions will increase // and might decrease during the course of a build. optional uint64 total_actions = 1; @@ -32,4 +32,7 @@ message BuildCompletionStatus { // Total number of current actions being executed during a course of a // build and current_actions + finished_actions <= total_actions. optional uint64 current_actions = 3; + + // Total number of actions that reported as a failure. + optional uint64 failed_actions = 4; } diff --git a/ui/status/completion_proto/regen.sh b/ui/status/build_progress_proto/regen.sh similarity index 82% rename from ui/status/completion_proto/regen.sh rename to ui/status/build_progress_proto/regen.sh index 652df08e0..572785d62 100755 --- a/ui/status/completion_proto/regen.sh +++ b/ui/status/build_progress_proto/regen.sh @@ -12,6 +12,6 @@ if ! hash aprotoc &>/dev/null; then die "could not find aprotoc. ${error_msg}" fi -if ! aprotoc --go_out=paths=source_relative:. build_completion.proto; then +if ! aprotoc --go_out=paths=source_relative:. build_progress.proto; then die "build failed. ${error_msg}" fi diff --git a/ui/status/completion_proto/build_completion.pb.go b/ui/status/completion_proto/build_completion.pb.go deleted file mode 100644 index 526e19a63..000000000 --- a/ui/status/completion_proto/build_completion.pb.go +++ /dev/null @@ -1,105 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: build_completion.proto - -package soong_build_completion_status_proto - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type BuildCompletionStatus struct { - // Total number of actions in a build. The total actions will increase - // and might decrease during the course of a build. - TotalActions *uint64 `protobuf:"varint,1,opt,name=total_actions,json=totalActions" json:"total_actions,omitempty"` - // Total number of completed build actions. This value will never decrease - // and finished_actions <= total_actions. At one point of the build, the - // finished_actions will be equal to total_actions. This may not represent - // that the build is completed as the total_actions may be increased for - // additional counted work or is doing non-counted work. - FinishedActions *uint64 `protobuf:"varint,2,opt,name=finished_actions,json=finishedActions" json:"finished_actions,omitempty"` - // Total number of current actions being executed during a course of a - // build and current_actions + finished_actions <= total_actions. - CurrentActions *uint64 `protobuf:"varint,3,opt,name=current_actions,json=currentActions" json:"current_actions,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *BuildCompletionStatus) Reset() { *m = BuildCompletionStatus{} } -func (m *BuildCompletionStatus) String() string { return proto.CompactTextString(m) } -func (*BuildCompletionStatus) ProtoMessage() {} -func (*BuildCompletionStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_7f03c01d09a4e764, []int{0} -} - -func (m *BuildCompletionStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_BuildCompletionStatus.Unmarshal(m, b) -} -func (m *BuildCompletionStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_BuildCompletionStatus.Marshal(b, m, deterministic) -} -func (m *BuildCompletionStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_BuildCompletionStatus.Merge(m, src) -} -func (m *BuildCompletionStatus) XXX_Size() int { - return xxx_messageInfo_BuildCompletionStatus.Size(m) -} -func (m *BuildCompletionStatus) XXX_DiscardUnknown() { - xxx_messageInfo_BuildCompletionStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_BuildCompletionStatus proto.InternalMessageInfo - -func (m *BuildCompletionStatus) GetTotalActions() uint64 { - if m != nil && m.TotalActions != nil { - return *m.TotalActions - } - return 0 -} - -func (m *BuildCompletionStatus) GetFinishedActions() uint64 { - if m != nil && m.FinishedActions != nil { - return *m.FinishedActions - } - return 0 -} - -func (m *BuildCompletionStatus) GetCurrentActions() uint64 { - if m != nil && m.CurrentActions != nil { - return *m.CurrentActions - } - return 0 -} - -func init() { - proto.RegisterType((*BuildCompletionStatus)(nil), "soong_build_completion_status.BuildCompletionStatus") -} - -func init() { proto.RegisterFile("build_completion.proto", fileDescriptor_7f03c01d09a4e764) } - -var fileDescriptor_7f03c01d09a4e764 = []byte{ - // 158 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4b, 0x2a, 0xcd, 0xcc, - 0x49, 0x89, 0x4f, 0xce, 0xcf, 0x2d, 0xc8, 0x49, 0x2d, 0xc9, 0xcc, 0xcf, 0xd3, 0x2b, 0x28, 0xca, - 0x2f, 0xc9, 0x17, 0x92, 0x2d, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, 0x47, 0x97, 0x8d, 0x2f, 0x2e, 0x49, - 0x2c, 0x29, 0x2d, 0x56, 0x9a, 0xc0, 0xc8, 0x25, 0xea, 0x04, 0x92, 0x73, 0x86, 0x4b, 0x05, 0x83, - 0x65, 0x84, 0x94, 0xb9, 0x78, 0x4b, 0xf2, 0x4b, 0x12, 0x73, 0xe2, 0x13, 0x93, 0x41, 0xa2, 0xc5, - 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x2c, 0x41, 0x3c, 0x60, 0x41, 0x47, 0x88, 0x98, 0x90, 0x26, 0x97, - 0x40, 0x5a, 0x66, 0x5e, 0x66, 0x71, 0x46, 0x6a, 0x0a, 0x5c, 0x1d, 0x13, 0x58, 0x1d, 0x3f, 0x4c, - 0x1c, 0xa6, 0x54, 0x9d, 0x8b, 0x3f, 0xb9, 0xb4, 0xa8, 0x28, 0x35, 0xaf, 0x04, 0xae, 0x92, 0x19, - 0xac, 0x92, 0x0f, 0x2a, 0x0c, 0x55, 0xe8, 0xa4, 0x1a, 0xa5, 0x8c, 0xd7, 0xcd, 0xf1, 0x60, 0x8f, - 0x01, 0x02, 0x00, 0x00, 0xff, 0xff, 0x13, 0x08, 0x7b, 0x38, 0xf1, 0x00, 0x00, 0x00, -} diff --git a/ui/status/log.go b/ui/status/log.go index d40724809..4e276416d 100644 --- a/ui/status/log.go +++ b/ui/status/log.go @@ -20,12 +20,14 @@ import ( "fmt" "io" "io/ioutil" + "os" "strings" "github.com/golang/protobuf/proto" "android/soong/ui/logger" "android/soong/ui/status/build_error_proto" + "android/soong/ui/status/build_progress_proto" ) type verboseLog struct { @@ -198,3 +200,75 @@ func (e *errorProtoLog) Message(level MsgLevel, message string) { func (e *errorProtoLog) Write(p []byte) (int, error) { return 0, errors.New("not supported") } + +type buildProgressLog struct { + filename string + log logger.Logger + failedActions uint64 +} + +func NewBuildProgressLog(log logger.Logger, filename string) StatusOutput { + return &buildProgressLog{ + filename: filename, + log: log, + failedActions: 0, + } +} + +func (b *buildProgressLog) StartAction(action *Action, counts Counts) { + b.updateCounters(counts) +} + +func (b *buildProgressLog) FinishAction(result ActionResult, counts Counts) { + if result.Error != nil { + b.failedActions++ + } + b.updateCounters(counts) +} + +func (b *buildProgressLog) Flush() { + //Not required. +} + +func (b *buildProgressLog) Message(level MsgLevel, message string) { + // Not required. +} + +func (b *buildProgressLog) Write(p []byte) (int, error) { + return 0, errors.New("not supported") +} + +func (b *buildProgressLog) updateCounters(counts Counts) { + err := writeToFile( + &soong_build_progress_proto.BuildProgress{ + CurrentActions: proto.Uint64(uint64(counts.RunningActions)), + FinishedActions: proto.Uint64(uint64(counts.FinishedActions)), + TotalActions: proto.Uint64(uint64(counts.TotalActions)), + FailedActions: proto.Uint64(b.failedActions), + }, + b.filename, + ) + if err != nil { + b.log.Printf("Failed to write file %s: %v\n", b.filename, err) + } +} + +func writeToFile(pb proto.Message, outputPath string) (err error) { + data, err := proto.Marshal(pb) + if err != nil { + return err + } + + tempPath := outputPath + ".tmp" + err = ioutil.WriteFile(tempPath, []byte(data), 0644) + if err != nil { + return err + } + + err = os.Rename(tempPath, outputPath) + if err != nil { + return err + } + + return nil +}