diff --git a/build_test.bash b/build_test.bash new file mode 100755 index 000000000..f83336667 --- /dev/null +++ b/build_test.bash @@ -0,0 +1,34 @@ +#!/bin/bash -eu +# +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file is used in our continous build infrastructure to run a variety of +# tests related to the build system. +# +# Currently, it's used to build and run multiproduct_kati, so it'll attempt +# to build ninja files for every product in the tree. I expect this to +# evolve as we find interesting things to test or track performance for. +# + +# To track how long we took to startup. %N isn't supported on Darwin, but +# that's detected in the Go code, which skips calculating the startup time. +export TRACE_BEGIN_SOONG=$(date +%s%N) + +export TOP=$(cd $(dirname ${BASH_SOURCE[0]})/../..; PWD= /bin/pwd) +source "${TOP}/build/soong/cmd/microfactory/microfactory.bash" + +build_go multiproduct_kati android/soong/cmd/multiproduct_kati +exec "$(getoutdir)/multiproduct_kati" "$@" diff --git a/cmd/microfactory/microfactory.bash b/cmd/microfactory/microfactory.bash new file mode 100644 index 000000000..7489fe398 --- /dev/null +++ b/cmd/microfactory/microfactory.bash @@ -0,0 +1,88 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set of utility functions to build and run go code with microfactory +# +# Inputs: +# ${TOP}: The top of the android source tree +# ${OUT_DIR}: The output directory location (defaults to ${TOP}/out) +# ${OUT_DIR_COMMON_BASE}: Change the default out directory to +# ${OUT_DIR_COMMON_BASE}/$(basename ${TOP}) + +# Ensure GOROOT is set to the in-tree version. +case $(uname) in + Linux) + export GOROOT="${TOP}/prebuilts/go/linux-x86/" + ;; + Darwin) + export GOROOT="${TOP}/prebuilts/go/darwin-x86/" + ;; + *) echo "unknown OS:" $(uname) >&2 && exit 1;; +esac + +# Find the output directory +function getoutdir +{ + local out_dir="${OUT_DIR-}" + if [ -z "${out_dir}" ]; then + if [ "${OUT_DIR_COMMON_BASE-}" ]; then + out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${TOP})" + else + out_dir="${TOP}/out" + fi + fi + echo "${out_dir}" +} + +# Bootstrap microfactory from source if necessary and use it to build the +# requested binary. +# +# Arguments: +# $1: name of the requested binary +# $2: package name +function build_go +{ + # Increment when microfactory changes enough that it cannot rebuild itself. + # For example, if we use a new command line argument that doesn't work on older versions. + local mf_version=2 + + local mf_src="${TOP}/build/soong/cmd/microfactory" + + local out_dir=$(getoutdir) + local mf_bin="${out_dir}/microfactory_$(uname)" + local mf_version_file="${out_dir}/.microfactory_$(uname)_version" + local built_bin="${out_dir}/$1" + local from_src=1 + + if [ -f "${mf_bin}" ] && [ -f "${mf_version_file}" ]; then + if [ "${mf_version}" -eq "$(cat "${mf_version_file}")" ]; then + from_src=0 + fi + fi + + local mf_cmd + if [ $from_src -eq 1 ]; then + mf_cmd="${GOROOT}/bin/go run ${mf_src}/microfactory.go" + else + mf_cmd="${mf_bin}" + fi + + ${mf_cmd} -s "${mf_src}" -b "${mf_bin}" \ + -pkg-path "android/soong=${TOP}/build/soong" -trimpath "${TOP}/build/soong" \ + -o "${built_bin}" $2 + + if [ $from_src -eq 1 ]; then + echo "${mf_version}" >"${mf_version_file}" + fi +} diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 3aa5a8767..b12628e76 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -51,11 +51,86 @@ var outDir = flag.String("out", "", "path to store output directories (defaults var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)") var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)") +var buildVariant = flag.String("variant", "eng", "build variant to use") + type Product struct { ctx build.Context config build.Config } +type Status struct { + cur int + total int + failed int + + ctx build.Context + haveBlankLine bool + smartTerminal bool + + lock sync.Mutex +} + +func NewStatus(ctx build.Context) *Status { + return &Status{ + ctx: ctx, + haveBlankLine: true, + smartTerminal: ctx.IsTerminal(), + } +} + +func (s *Status) SetTotal(total int) { + s.total = total +} + +func (s *Status) Fail(product string, err error) { + s.Finish(product) + + s.lock.Lock() + defer s.lock.Unlock() + + if s.smartTerminal && !s.haveBlankLine { + fmt.Fprintln(s.ctx.Stdout()) + s.haveBlankLine = true + } + + s.failed++ + fmt.Fprintln(s.ctx.Stderr(), "FAILED:", product) + s.ctx.Verboseln("FAILED:", product) + s.ctx.Println(err) +} + +func (s *Status) Finish(product string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.cur++ + line := fmt.Sprintf("[%d/%d] %s", s.cur, s.total, product) + + if s.smartTerminal { + if max, ok := s.ctx.TermWidth(); ok { + if len(line) > max { + line = line[:max] + } + } + + fmt.Fprint(s.ctx.Stdout(), "\r", line, "\x1b[K") + s.haveBlankLine = false + } else { + s.ctx.Println(line) + } +} + +func (s *Status) Finished() int { + s.lock.Lock() + defer s.lock.Unlock() + + if !s.haveBlankLine { + fmt.Fprintln(s.ctx.Stdout()) + s.haveBlankLine = true + } + return s.failed +} + func main() { log := logger.New(os.Stderr) defer log.Cleanup() @@ -80,7 +155,7 @@ func main() { StdioInterface: build.StdioImpl{}, }} - failed := false + status := NewStatus(buildCtx) config := build.NewConfig(buildCtx) if *outDir == "" { @@ -94,7 +169,7 @@ func main() { if !*keep { defer func() { - if !failed { + if status.Finished() == 0 { os.RemoveAll(*outDir) } }() @@ -114,8 +189,9 @@ func main() { products := strings.Fields(vars["all_named_products"]) log.Verbose("Got product list:", products) + status.SetTotal(len(products)) + var wg sync.WaitGroup - errs := make(chan error, len(products)) productConfigs := make(chan Product, len(products)) // Run the product config for every product in parallel @@ -124,7 +200,7 @@ func main() { go func(product string) { defer wg.Done() defer logger.Recover(func(err error) { - errs <- fmt.Errorf("Error building %s: %v", product, err) + status.Fail(product, err) }) productOutDir := filepath.Join(config.OutDir(), product) @@ -151,7 +227,7 @@ func main() { productConfig := build.NewConfig(productCtx) productConfig.Environment().Set("OUT_DIR", productOutDir) - productConfig.Lunch(productCtx, product, "eng") + productConfig.Lunch(productCtx, product, *buildVariant) build.Build(productCtx, productConfig, build.BuildProductConfig) productConfigs <- Product{productCtx, productConfig} @@ -171,7 +247,7 @@ func main() { for product := range productConfigs { func() { defer logger.Recover(func(err error) { - errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err) + status.Fail(product.config.TargetProduct(), err) }) buildWhat := 0 @@ -185,22 +261,14 @@ func main() { if !*keep { os.RemoveAll(product.config.OutDir()) } - log.Println("Finished running for", product.config.TargetProduct()) + status.Finish(product.config.TargetProduct()) }() } }() } - go func() { - wg2.Wait() - close(errs) - }() + wg2.Wait() - for err := range errs { - failed = true - log.Print(err) - } - - if failed { - log.Fatalln("Failed") + if count := status.Finished(); count > 0 { + log.Fatalln(count, "products failed") } } diff --git a/soong_ui.bash b/soong_ui.bash index e3997cf45..105af9f48 100755 --- a/soong_ui.bash +++ b/soong_ui.bash @@ -15,7 +15,7 @@ # limitations under the License. # To track how long we took to startup. %N isn't supported on Darwin, but -# that's detected in the Go code, and skip calculating the startup time. +# that's detected in the Go code, which skips calculating the startup time. export TRACE_BEGIN_SOONG=$(date +%s%N) # Function to find top of the source tree (if $TOP isn't set) by walking up the @@ -47,63 +47,8 @@ function gettop fi } -# Bootstrap microfactory from source if necessary and use it to build the -# soong_ui binary, then run soong_ui. -function run_go -{ - # Increment when microfactory changes enough that it cannot rebuild itself. - # For example, if we use a new command line argument that doesn't work on older versions. - local mf_version=2 - - local mf_src="${TOP}/build/soong/cmd/microfactory" - - local out_dir="${OUT_DIR-}" - if [ -z "${out_dir}" ]; then - if [ "${OUT_DIR_COMMON_BASE-}" ]; then - out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${TOP})" - else - out_dir="${TOP}/out" - fi - fi - - local mf_bin="${out_dir}/microfactory_$(uname)" - local mf_version_file="${out_dir}/.microfactory_$(uname)_version" - local soong_ui_bin="${out_dir}/soong_ui" - local from_src=1 - - if [ -f "${mf_bin}" ] && [ -f "${mf_version_file}" ]; then - if [ "${mf_version}" -eq "$(cat "${mf_version_file}")" ]; then - from_src=0 - fi - fi - - local mf_cmd - if [ $from_src -eq 1 ]; then - mf_cmd="${GOROOT}/bin/go run ${mf_src}/microfactory.go" - else - mf_cmd="${mf_bin}" - fi - - ${mf_cmd} -s "${mf_src}" -b "${mf_bin}" \ - -pkg-path "android/soong=${TOP}/build/soong" -trimpath "${TOP}/build/soong" \ - -o "${soong_ui_bin}" android/soong/cmd/soong_ui - - if [ $from_src -eq 1 ]; then - echo "${mf_version}" >"${mf_version_file}" - fi - - exec "${out_dir}/soong_ui" "$@" -} - export TOP=$(gettop) -case $(uname) in - Linux) - export GOROOT="${TOP}/prebuilts/go/linux-x86/" - ;; - Darwin) - export GOROOT="${TOP}/prebuilts/go/darwin-x86/" - ;; - *) echo "unknown OS:" $(uname) >&2 && exit 1;; -esac +source build/soong/cmd/microfactory/microfactory.bash -run_go "$@" +build_go soong_ui android/soong/cmd/soong_ui +exec "$(getoutdir)/soong_ui" "$@" diff --git a/ui/build/context.go b/ui/build/context.go index f85bb6c0c..52a337d48 100644 --- a/ui/build/context.go +++ b/ui/build/context.go @@ -102,3 +102,7 @@ func (c ContextImpl) IsTerminal() bool { } return false } + +func (c ContextImpl) TermWidth() (int, bool) { + return termWidth(c.Stdout()) +}