Add build progress in Soong UI

Soong UI generates a proto file named build_progress.pb in $(OUT_DIR)
output directory that contains build action numbers (how many are executing,
finished and total) during the course of a build. This is for external
systems that invokes the Platform Build Systems and would like to know
the completion status.

Bug: b/150401146
Test: Wrote a bash script that continuously read the
build_progress.pb file and computed the build completed percentage
while building the aosp_arm-eng target. Compared the percentage between
the Soong output console and the one reported by the bash script.

Change-Id: I7c7347bc8e41958093892d8e2731c4f4169937dd

cmd/soong_ui/main.go

@@ -173,6 +173,7 @@ func main() {
 	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
 	stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error")))
 	stat.AddOutput(status.NewCriticalPath(log))
+	stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb")))
 
 	buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024))
 	buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",

ui/status/Android.bp

@@ -20,6 +20,7 @@ bootstrap_go_package {
         "soong-ui-logger",
         "soong-ui-status-ninja_frontend",
         "soong-ui-status-build_error_proto",
+        "soong-ui-status-build_progress_proto",
     ],
     srcs: [
         "critical_path.go",
@@ -53,3 +54,12 @@ bootstrap_go_package {
         "build_error_proto/build_error.pb.go",
     ],
 }
+
+bootstrap_go_package {
+    name: "soong-ui-status-build_progress_proto",
+    pkgPath: "android/soong/ui/status/build_progress_proto",
+    deps: ["golang-protobuf-proto"],
+    srcs: [
+        "build_progress_proto/build_progress.pb.go",
+    ],
+}

ui/status/build_progress_proto/build_progress.pb.go

@@ -0,0 +1,115 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: build_progress.proto
+
+package soong_build_progress_proto
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type BuildProgress struct {
+	// Total number of actions in a build. The total actions will increase
+	// and might decrease during the course of a build.
+	TotalActions *uint64 `protobuf:"varint,1,opt,name=total_actions,json=totalActions" json:"total_actions,omitempty"`
+	// Total number of completed build actions. This value will never decrease
+	// and finished_actions <= total_actions. At one point of the build, the
+	// finished_actions will be equal to total_actions. This may not represent
+	// that the build is completed as the total_actions may be increased for
+	// additional counted work or is doing non-counted work.
+	FinishedActions *uint64 `protobuf:"varint,2,opt,name=finished_actions,json=finishedActions" json:"finished_actions,omitempty"`
+	// Total number of current actions being executed during a course of a
+	// build and current_actions + finished_actions <= total_actions.
+	CurrentActions *uint64 `protobuf:"varint,3,opt,name=current_actions,json=currentActions" json:"current_actions,omitempty"`
+	// Total number of actions that reported as a failure.
+	FailedActions        *uint64  `protobuf:"varint,4,opt,name=failed_actions,json=failedActions" json:"failed_actions,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *BuildProgress) Reset()         { *m = BuildProgress{} }
+func (m *BuildProgress) String() string { return proto.CompactTextString(m) }
+func (*BuildProgress) ProtoMessage()    {}
+func (*BuildProgress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a8a463f8e30dab2e, []int{0}
+}
+
+func (m *BuildProgress) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_BuildProgress.Unmarshal(m, b)
+}
+func (m *BuildProgress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_BuildProgress.Marshal(b, m, deterministic)
+}
+func (m *BuildProgress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_BuildProgress.Merge(m, src)
+}
+func (m *BuildProgress) XXX_Size() int {
+	return xxx_messageInfo_BuildProgress.Size(m)
+}
+func (m *BuildProgress) XXX_DiscardUnknown() {
+	xxx_messageInfo_BuildProgress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BuildProgress proto.InternalMessageInfo
+
+func (m *BuildProgress) GetTotalActions() uint64 {
+	if m != nil && m.TotalActions != nil {
+		return *m.TotalActions
+	}
+	return 0
+}
+
+func (m *BuildProgress) GetFinishedActions() uint64 {
+	if m != nil && m.FinishedActions != nil {
+		return *m.FinishedActions
+	}
+	return 0
+}
+
+func (m *BuildProgress) GetCurrentActions() uint64 {
+	if m != nil && m.CurrentActions != nil {
+		return *m.CurrentActions
+	}
+	return 0
+}
+
+func (m *BuildProgress) GetFailedActions() uint64 {
+	if m != nil && m.FailedActions != nil {
+		return *m.FailedActions
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterType((*BuildProgress)(nil), "soong_build_progress.BuildProgress")
+}
+
+func init() { proto.RegisterFile("build_progress.proto", fileDescriptor_a8a463f8e30dab2e) }
+
+var fileDescriptor_a8a463f8e30dab2e = []byte{
+	// 165 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x49, 0x2a, 0xcd, 0xcc,
+	0x49, 0x89, 0x2f, 0x28, 0xca, 0x4f, 0x2f, 0x4a, 0x2d, 0x2e, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9,
+	0x17, 0x12, 0x29, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, 0x47, 0x95, 0x53, 0x5a, 0xcf, 0xc8, 0xc5, 0xeb,
+	0x04, 0x12, 0x0a, 0x80, 0x8a, 0x08, 0x29, 0x73, 0xf1, 0x96, 0xe4, 0x97, 0x24, 0xe6, 0xc4, 0x27,
+	0x26, 0x97, 0x64, 0xe6, 0xe7, 0x15, 0x4b, 0x30, 0x2a, 0x30, 0x6a, 0xb0, 0x04, 0xf1, 0x80, 0x05,
+	0x1d, 0x21, 0x62, 0x42, 0x9a, 0x5c, 0x02, 0x69, 0x99, 0x79, 0x99, 0xc5, 0x19, 0xa9, 0x29, 0x70,
+	0x75, 0x4c, 0x60, 0x75, 0xfc, 0x30, 0x71, 0x98, 0x52, 0x75, 0x2e, 0xfe, 0xe4, 0xd2, 0xa2, 0xa2,
+	0xd4, 0xbc, 0x12, 0xb8, 0x4a, 0x66, 0xb0, 0x4a, 0x3e, 0xa8, 0x30, 0x4c, 0xa1, 0x2a, 0x17, 0x5f,
+	0x5a, 0x62, 0x66, 0x0e, 0x92, 0x89, 0x2c, 0x60, 0x75, 0xbc, 0x10, 0x51, 0xa8, 0x32, 0x27, 0x99,
+	0x28, 0x29, 0x6c, 0x3e, 0x89, 0x07, 0xfb, 0x12, 0x10, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x6e, 0xc1,
+	0xef, 0xfc, 0x00, 0x00, 0x00,
+}

ui/status/build_progress_proto/build_progress.proto

@@ -14,10 +14,10 @@
 
 syntax = "proto2";
 
-package soong_build_completion_status;
+package soong_build_progress;
-option go_package = "soong_build_completion_status_proto";
+option go_package = "soong_build_progress_proto";
 
-message BuildCompletionStatus {
+message BuildProgress {
   // Total number of actions in a build. The total actions will increase
   // and might decrease during the course of a build.
   optional uint64 total_actions = 1;
@@ -32,4 +32,7 @@ message BuildCompletionStatus {
   // Total number of current actions being executed during a course of a
   // build and current_actions + finished_actions <= total_actions.
   optional uint64 current_actions = 3;
+
+  // Total number of actions that reported as a failure.
+  optional uint64 failed_actions = 4;
 }

ui/status/build_progress_proto/regen.sh

@@ -12,6 +12,6 @@ if ! hash aprotoc &>/dev/null; then
   die "could not find aprotoc. ${error_msg}"
 fi
 
-if ! aprotoc --go_out=paths=source_relative:. build_completion.proto; then
+if ! aprotoc --go_out=paths=source_relative:. build_progress.proto; then
   die "build failed. ${error_msg}"
 fi

ui/status/completion_proto/build_completion.pb.go

@@ -1,105 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: build_completion.proto
-
-package soong_build_completion_status_proto
-
-import (
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	math "math"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type BuildCompletionStatus struct {
-	// Total number of actions in a build. The total actions will increase
-	// and might decrease during the course of a build.
-	TotalActions *uint64 `protobuf:"varint,1,opt,name=total_actions,json=totalActions" json:"total_actions,omitempty"`
-	// Total number of completed build actions. This value will never decrease
-	// and finished_actions <= total_actions. At one point of the build, the
-	// finished_actions will be equal to total_actions. This may not represent
-	// that the build is completed as the total_actions may be increased for
-	// additional counted work or is doing non-counted work.
-	FinishedActions *uint64 `protobuf:"varint,2,opt,name=finished_actions,json=finishedActions" json:"finished_actions,omitempty"`
-	// Total number of current actions being executed during a course of a
-	// build and current_actions + finished_actions <= total_actions.
-	CurrentActions       *uint64  `protobuf:"varint,3,opt,name=current_actions,json=currentActions" json:"current_actions,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *BuildCompletionStatus) Reset()         { *m = BuildCompletionStatus{} }
-func (m *BuildCompletionStatus) String() string { return proto.CompactTextString(m) }
-func (*BuildCompletionStatus) ProtoMessage()    {}
-func (*BuildCompletionStatus) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7f03c01d09a4e764, []int{0}
-}
-
-func (m *BuildCompletionStatus) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_BuildCompletionStatus.Unmarshal(m, b)
-}
-func (m *BuildCompletionStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_BuildCompletionStatus.Marshal(b, m, deterministic)
-}
-func (m *BuildCompletionStatus) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_BuildCompletionStatus.Merge(m, src)
-}
-func (m *BuildCompletionStatus) XXX_Size() int {
-	return xxx_messageInfo_BuildCompletionStatus.Size(m)
-}
-func (m *BuildCompletionStatus) XXX_DiscardUnknown() {
-	xxx_messageInfo_BuildCompletionStatus.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_BuildCompletionStatus proto.InternalMessageInfo
-
-func (m *BuildCompletionStatus) GetTotalActions() uint64 {
-	if m != nil && m.TotalActions != nil {
-		return *m.TotalActions
-	}
-	return 0
-}
-
-func (m *BuildCompletionStatus) GetFinishedActions() uint64 {
-	if m != nil && m.FinishedActions != nil {
-		return *m.FinishedActions
-	}
-	return 0
-}
-
-func (m *BuildCompletionStatus) GetCurrentActions() uint64 {
-	if m != nil && m.CurrentActions != nil {
-		return *m.CurrentActions
-	}
-	return 0
-}
-
-func init() {
-	proto.RegisterType((*BuildCompletionStatus)(nil), "soong_build_completion_status.BuildCompletionStatus")
-}
-
-func init() { proto.RegisterFile("build_completion.proto", fileDescriptor_7f03c01d09a4e764) }
-
-var fileDescriptor_7f03c01d09a4e764 = []byte{
-	// 158 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4b, 0x2a, 0xcd, 0xcc,
-	0x49, 0x89, 0x4f, 0xce, 0xcf, 0x2d, 0xc8, 0x49, 0x2d, 0xc9, 0xcc, 0xcf, 0xd3, 0x2b, 0x28, 0xca,
-	0x2f, 0xc9, 0x17, 0x92, 0x2d, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, 0x47, 0x97, 0x8d, 0x2f, 0x2e, 0x49,
-	0x2c, 0x29, 0x2d, 0x56, 0x9a, 0xc0, 0xc8, 0x25, 0xea, 0x04, 0x92, 0x73, 0x86, 0x4b, 0x05, 0x83,
-	0x65, 0x84, 0x94, 0xb9, 0x78, 0x4b, 0xf2, 0x4b, 0x12, 0x73, 0xe2, 0x13, 0x93, 0x41, 0xa2, 0xc5,
-	0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x2c, 0x41, 0x3c, 0x60, 0x41, 0x47, 0x88, 0x98, 0x90, 0x26, 0x97,
-	0x40, 0x5a, 0x66, 0x5e, 0x66, 0x71, 0x46, 0x6a, 0x0a, 0x5c, 0x1d, 0x13, 0x58, 0x1d, 0x3f, 0x4c,
-	0x1c, 0xa6, 0x54, 0x9d, 0x8b, 0x3f, 0xb9, 0xb4, 0xa8, 0x28, 0x35, 0xaf, 0x04, 0xae, 0x92, 0x19,
-	0xac, 0x92, 0x0f, 0x2a, 0x0c, 0x55, 0xe8, 0xa4, 0x1a, 0xa5, 0x8c, 0xd7, 0xcd, 0xf1, 0x60, 0x8f,
-	0x01, 0x02, 0x00, 0x00, 0xff, 0xff, 0x13, 0x08, 0x7b, 0x38, 0xf1, 0x00, 0x00, 0x00,
-}

ui/status/log.go

@@ -20,12 +20,14 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/golang/protobuf/proto"
 
 	"android/soong/ui/logger"
 	"android/soong/ui/status/build_error_proto"
+	"android/soong/ui/status/build_progress_proto"
 )
 
 type verboseLog struct {
@@ -198,3 +200,75 @@ func (e *errorProtoLog) Message(level MsgLevel, message string) {
 func (e *errorProtoLog) Write(p []byte) (int, error) {
 	return 0, errors.New("not supported")
 }
+
+type buildProgressLog struct {
+	filename      string
+	log           logger.Logger
+	failedActions uint64
+}
+
+func NewBuildProgressLog(log logger.Logger, filename string) StatusOutput {
+	return &buildProgressLog{
+		filename:      filename,
+		log:           log,
+		failedActions: 0,
+	}
+}
+
+func (b *buildProgressLog) StartAction(action *Action, counts Counts) {
+	b.updateCounters(counts)
+}
+
+func (b *buildProgressLog) FinishAction(result ActionResult, counts Counts) {
+	if result.Error != nil {
+		b.failedActions++
+	}
+	b.updateCounters(counts)
+}
+
+func (b *buildProgressLog) Flush() {
+	//Not required.
+}
+
+func (b *buildProgressLog) Message(level MsgLevel, message string) {
+	// Not required.
+}
+
+func (b *buildProgressLog) Write(p []byte) (int, error) {
+	return 0, errors.New("not supported")
+}
+
+func (b *buildProgressLog) updateCounters(counts Counts) {
+	err := writeToFile(
+		&soong_build_progress_proto.BuildProgress{
+			CurrentActions:  proto.Uint64(uint64(counts.RunningActions)),
+			FinishedActions: proto.Uint64(uint64(counts.FinishedActions)),
+			TotalActions:    proto.Uint64(uint64(counts.TotalActions)),
+			FailedActions:   proto.Uint64(b.failedActions),
+		},
+		b.filename,
+	)
+	if err != nil {
+		b.log.Printf("Failed to write file %s: %v\n", b.filename, err)
+	}
+}
+
+func writeToFile(pb proto.Message, outputPath string) (err error) {
+	data, err := proto.Marshal(pb)
+	if err != nil {
+		return err
+	}
+
+	tempPath := outputPath + ".tmp"
+	err = ioutil.WriteFile(tempPath, []byte(data), 0644)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename(tempPath, outputPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}