Merge changes I9512642d,I6548889c,I8db5198f

* changes:
  Add build_test.sh, split common parts of soong_ui.bash
  Allow specifying a build variant
  Improve multiproduct_kati output

build_test.bash

@@ -0,0 +1,34 @@
+#!/bin/bash -eu
+#
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This file is used in our continous build infrastructure to run a variety of
+# tests related to the build system.
+#
+# Currently, it's used to build and run multiproduct_kati, so it'll attempt
+# to build ninja files for every product in the tree. I expect this to
+# evolve as we find interesting things to test or track performance for.
+#
+
+# To track how long we took to startup. %N isn't supported on Darwin, but
+# that's detected in the Go code, which skips calculating the startup time.
+export TRACE_BEGIN_SOONG=$(date +%s%N)
+
+export TOP=$(cd $(dirname ${BASH_SOURCE[0]})/../..; PWD= /bin/pwd)
+source "${TOP}/build/soong/cmd/microfactory/microfactory.bash"
+
+build_go multiproduct_kati android/soong/cmd/multiproduct_kati
+exec "$(getoutdir)/multiproduct_kati" "$@"

cmd/microfactory/microfactory.bash

@@ -0,0 +1,88 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set of utility functions to build and run go code with microfactory
+#
+# Inputs:
+#  ${TOP}: The top of the android source tree
+#  ${OUT_DIR}: The output directory location (defaults to ${TOP}/out)
+#  ${OUT_DIR_COMMON_BASE}: Change the default out directory to
+#    ${OUT_DIR_COMMON_BASE}/$(basename ${TOP})
+
+# Ensure GOROOT is set to the in-tree version.
+case $(uname) in
+    Linux)
+        export GOROOT="${TOP}/prebuilts/go/linux-x86/"
+        ;;
+    Darwin)
+        export GOROOT="${TOP}/prebuilts/go/darwin-x86/"
+        ;;
+    *) echo "unknown OS:" $(uname) >&2 && exit 1;;
+esac
+
+# Find the output directory
+function getoutdir
+{
+    local out_dir="${OUT_DIR-}"
+    if [ -z "${out_dir}" ]; then
+        if [ "${OUT_DIR_COMMON_BASE-}" ]; then
+            out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${TOP})"
+        else
+            out_dir="${TOP}/out"
+        fi
+    fi
+    echo "${out_dir}"
+}
+
+# Bootstrap microfactory from source if necessary and use it to build the
+# requested binary.
+#
+# Arguments:
+#  $1: name of the requested binary
+#  $2: package name
+function build_go
+{
+    # Increment when microfactory changes enough that it cannot rebuild itself.
+    # For example, if we use a new command line argument that doesn't work on older versions.
+    local mf_version=2
+
+    local mf_src="${TOP}/build/soong/cmd/microfactory"
+
+    local out_dir=$(getoutdir)
+    local mf_bin="${out_dir}/microfactory_$(uname)"
+    local mf_version_file="${out_dir}/.microfactory_$(uname)_version"
+    local built_bin="${out_dir}/$1"
+    local from_src=1
+
+    if [ -f "${mf_bin}" ] && [ -f "${mf_version_file}" ]; then
+        if [ "${mf_version}" -eq "$(cat "${mf_version_file}")" ]; then
+            from_src=0
+        fi
+    fi
+
+    local mf_cmd
+    if [ $from_src -eq 1 ]; then
+        mf_cmd="${GOROOT}/bin/go run ${mf_src}/microfactory.go"
+    else
+        mf_cmd="${mf_bin}"
+    fi
+
+    ${mf_cmd} -s "${mf_src}" -b "${mf_bin}" \
+            -pkg-path "android/soong=${TOP}/build/soong" -trimpath "${TOP}/build/soong" \
+            -o "${built_bin}" $2
+
+    if [ $from_src -eq 1 ]; then
+        echo "${mf_version}" >"${mf_version_file}"
+    fi
+}

cmd/multiproduct_kati/main.go

@@ -51,11 +51,86 @@ var outDir = flag.String("out", "", "path to store output directories (defaults
 var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
 var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
 
+var buildVariant = flag.String("variant", "eng", "build variant to use")
+
 type Product struct {
 	ctx    build.Context
 	config build.Config
 }
 
+type Status struct {
+	cur    int
+	total  int
+	failed int
+
+	ctx           build.Context
+	haveBlankLine bool
+	smartTerminal bool
+
+	lock sync.Mutex
+}
+
+func NewStatus(ctx build.Context) *Status {
+	return &Status{
+		ctx:           ctx,
+		haveBlankLine: true,
+		smartTerminal: ctx.IsTerminal(),
+	}
+}
+
+func (s *Status) SetTotal(total int) {
+	s.total = total
+}
+
+func (s *Status) Fail(product string, err error) {
+	s.Finish(product)
+
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.smartTerminal && !s.haveBlankLine {
+		fmt.Fprintln(s.ctx.Stdout())
+		s.haveBlankLine = true
+	}
+
+	s.failed++
+	fmt.Fprintln(s.ctx.Stderr(), "FAILED:", product)
+	s.ctx.Verboseln("FAILED:", product)
+	s.ctx.Println(err)
+}
+
+func (s *Status) Finish(product string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.cur++
+	line := fmt.Sprintf("[%d/%d] %s", s.cur, s.total, product)
+
+	if s.smartTerminal {
+		if max, ok := s.ctx.TermWidth(); ok {
+			if len(line) > max {
+				line = line[:max]
+			}
+		}
+
+		fmt.Fprint(s.ctx.Stdout(), "\r", line, "\x1b[K")
+		s.haveBlankLine = false
+	} else {
+		s.ctx.Println(line)
+	}
+}
+
+func (s *Status) Finished() int {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if !s.haveBlankLine {
+		fmt.Fprintln(s.ctx.Stdout())
+		s.haveBlankLine = true
+	}
+	return s.failed
+}
+
 func main() {
 	log := logger.New(os.Stderr)
 	defer log.Cleanup()
@@ -80,7 +155,7 @@ func main() {
 		StdioInterface: build.StdioImpl{},
 	}}
 
-	failed := false
+	status := NewStatus(buildCtx)
 
 	config := build.NewConfig(buildCtx)
 	if *outDir == "" {
@@ -94,7 +169,7 @@ func main() {
 
 		if !*keep {
 			defer func() {
-				if !failed {
+				if status.Finished() == 0 {
 					os.RemoveAll(*outDir)
 				}
 			}()
@@ -114,8 +189,9 @@ func main() {
 	products := strings.Fields(vars["all_named_products"])
 	log.Verbose("Got product list:", products)
 
+	status.SetTotal(len(products))
+
 	var wg sync.WaitGroup
-	errs := make(chan error, len(products))
 	productConfigs := make(chan Product, len(products))
 
 	// Run the product config for every product in parallel
@@ -124,7 +200,7 @@ func main() {
 		go func(product string) {
 			defer wg.Done()
 			defer logger.Recover(func(err error) {
-				errs <- fmt.Errorf("Error building %s: %v", product, err)
+				status.Fail(product, err)
 			})
 
 			productOutDir := filepath.Join(config.OutDir(), product)
@@ -151,7 +227,7 @@ func main() {
 
 			productConfig := build.NewConfig(productCtx)
 			productConfig.Environment().Set("OUT_DIR", productOutDir)
-			productConfig.Lunch(productCtx, product, "eng")
+			productConfig.Lunch(productCtx, product, *buildVariant)
 
 			build.Build(productCtx, productConfig, build.BuildProductConfig)
 			productConfigs <- Product{productCtx, productConfig}
@@ -171,7 +247,7 @@ func main() {
 			for product := range productConfigs {
 				func() {
 					defer logger.Recover(func(err error) {
-						errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err)
+						status.Fail(product.config.TargetProduct(), err)
 					})
 
 					buildWhat := 0
@@ -185,22 +261,14 @@ func main() {
 					if !*keep {
 						os.RemoveAll(product.config.OutDir())
 					}
-					log.Println("Finished running for", product.config.TargetProduct())
+					status.Finish(product.config.TargetProduct())
 				}()
 			}
 		}()
 	}
-	go func() {
+	wg2.Wait()
-		wg2.Wait()
-		close(errs)
-	}()
 
-	for err := range errs {
+	if count := status.Finished(); count > 0 {
-		failed = true
+		log.Fatalln(count, "products failed")
-		log.Print(err)
-	}
-
-	if failed {
-		log.Fatalln("Failed")
 	}
 }

soong_ui.bash

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # To track how long we took to startup. %N isn't supported on Darwin, but
-# that's detected in the Go code, and skip calculating the startup time.
+# that's detected in the Go code, which skips calculating the startup time.
 export TRACE_BEGIN_SOONG=$(date +%s%N)
 
 # Function to find top of the source tree (if $TOP isn't set) by walking up the
@@ -47,63 +47,8 @@ function gettop
     fi
 }
 
-# Bootstrap microfactory from source if necessary and use it to build the
-# soong_ui binary, then run soong_ui.
-function run_go
-{
-    # Increment when microfactory changes enough that it cannot rebuild itself.
-    # For example, if we use a new command line argument that doesn't work on older versions.
-    local mf_version=2
-
-    local mf_src="${TOP}/build/soong/cmd/microfactory"
-
-    local out_dir="${OUT_DIR-}"
-    if [ -z "${out_dir}" ]; then
-        if [ "${OUT_DIR_COMMON_BASE-}" ]; then
-            out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${TOP})"
-        else
-            out_dir="${TOP}/out"
-        fi
-    fi
-
-    local mf_bin="${out_dir}/microfactory_$(uname)"
-    local mf_version_file="${out_dir}/.microfactory_$(uname)_version"
-    local soong_ui_bin="${out_dir}/soong_ui"
-    local from_src=1
-
-    if [ -f "${mf_bin}" ] && [ -f "${mf_version_file}" ]; then
-        if [ "${mf_version}" -eq "$(cat "${mf_version_file}")" ]; then
-            from_src=0
-        fi
-    fi
-
-    local mf_cmd
-    if [ $from_src -eq 1 ]; then
-        mf_cmd="${GOROOT}/bin/go run ${mf_src}/microfactory.go"
-    else
-        mf_cmd="${mf_bin}"
-    fi
-
-    ${mf_cmd} -s "${mf_src}" -b "${mf_bin}" \
-            -pkg-path "android/soong=${TOP}/build/soong" -trimpath "${TOP}/build/soong" \
-            -o "${soong_ui_bin}" android/soong/cmd/soong_ui
-
-    if [ $from_src -eq 1 ]; then
-        echo "${mf_version}" >"${mf_version_file}"
-    fi
-
-    exec "${out_dir}/soong_ui" "$@"
-}
-
 export TOP=$(gettop)
-case $(uname) in
+source build/soong/cmd/microfactory/microfactory.bash
-    Linux)
-        export GOROOT="${TOP}/prebuilts/go/linux-x86/"
-        ;;
-    Darwin)
-        export GOROOT="${TOP}/prebuilts/go/darwin-x86/"
-        ;;
-    *) echo "unknown OS:" $(uname) >&2 && exit 1;;
-esac
 
-run_go "$@"
+build_go soong_ui android/soong/cmd/soong_ui
+exec "$(getoutdir)/soong_ui" "$@"

ui/build/context.go

@@ -102,3 +102,7 @@ func (c ContextImpl) IsTerminal() bool {
 	}
 	return false
 }
+
+func (c ContextImpl) TermWidth() (int, bool) {
+	return termWidth(c.Stdout())
+}