Add a Go replacement for our top-level Make wrapper am: 1e70446251

am: 42a007560c

Change-Id: Ia8c859a02a91a402a9ef6de1bd267f849d8c7000

Android.bp

@@ -14,6 +14,7 @@ subdirs = [
     "androidmk",
     "cmd/*",
     "third_party/zip",
+    "ui/*",
 ]
 
 bootstrap_go_package {

cmd/soong_ui/Android.bp

@@ -0,0 +1,24 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+blueprint_go_binary {
+    name: "soong_ui",
+    deps: [
+        "soong-ui-build",
+        "soong-ui-logger",
+    ],
+    srcs: [
+        "main.go",
+    ],
+}

cmd/soong_ui/main.go

@@ -0,0 +1,87 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"android/soong/ui/build"
+	"android/soong/ui/logger"
+)
+
+func indexList(s string, list []string) int {
+	for i, l := range list {
+		if l == s {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func inList(s string, list []string) bool {
+	return indexList(s, list) != -1
+}
+
+func main() {
+	log := logger.New(os.Stderr)
+	defer log.Cleanup()
+
+	if len(os.Args) < 2 || !inList("--make-mode", os.Args) {
+		log.Fatalln("The `soong` native UI is not yet available.")
+	}
+
+	// Precondition: the current directory is the top of the source tree
+	if _, err := os.Stat("build/soong/root.bp"); err != nil {
+		if os.IsNotExist(err) {
+			log.Fatalln("soong_ui should run from the root of the source directory: build/soong/root.bp not found")
+		}
+		log.Fatalln("Error verifying tree state:", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	build.SetupSignals(log, cancel, log.Cleanup)
+
+	buildCtx := &build.ContextImpl{
+		Context:        ctx,
+		Logger:         log,
+		StdioInterface: build.StdioImpl{},
+	}
+	config := build.NewConfig(buildCtx, os.Args[1:]...)
+
+	log.SetVerbose(config.IsVerbose())
+	if err := os.MkdirAll(config.OutDir(), 0777); err != nil {
+		log.Fatalf("Error creating out directory: %v", err)
+	}
+	log.SetOutput(filepath.Join(config.OutDir(), "build.log"))
+
+	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
+		if !strings.HasSuffix(start, "N") {
+			if start_time, err := strconv.ParseUint(start, 10, 64); err == nil {
+				log.Verbosef("Took %dms to start up.",
+					time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds())
+			}
+		}
+	}
+
+	build.Build(buildCtx, config, build.BuildAll)
+}

ui/build/Android.bp

@@ -0,0 +1,36 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+bootstrap_go_package {
+    name: "soong-ui-build",
+    pkgPath: "android/soong/ui/build",
+    deps: [
+        "soong-ui-logger",
+    ],
+    srcs: [
+        "build.go",
+        "config.go",
+        "context.go",
+        "environment.go",
+        "kati.go",
+        "make.go",
+        "ninja.go",
+        "signal.go",
+        "soong.go",
+        "util.go",
+    ],
+    testSrcs: [
+        "environment_test.go",
+    ],
+}

ui/build/build.go

@@ -0,0 +1,105 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"text/template"
+)
+
+// Ensures the out directory exists, and has the proper files to prevent kati
+// from recursing into it.
+func SetupOutDir(ctx Context, config Config) {
+	ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "Android.mk"))
+	ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "CleanSpec.mk"))
+	ensureEmptyFileExists(ctx, filepath.Join(config.SoongOutDir(), ".soong.in_make"))
+	// The ninja_build file is used by our buildbots to understand that the output
+	// can be parsed as ninja output.
+	ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build"))
+}
+
+var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(`
+builddir = {{.OutDir}}
+include {{.KatiNinjaFile}}
+include {{.SoongNinjaFile}}
+build {{.CombinedNinjaFile}}: phony {{.SoongNinjaFile}}
+`))
+
+func createCombinedBuildNinjaFile(ctx Context, config Config) {
+	file, err := os.Create(config.CombinedNinjaFile())
+	if err != nil {
+		ctx.Fatalln("Failed to create combined ninja file:", err)
+	}
+	defer file.Close()
+
+	if err := combinedBuildNinjaTemplate.Execute(file, config); err != nil {
+		ctx.Fatalln("Failed to write combined ninja file:", err)
+	}
+}
+
+const (
+	BuildNone          = iota
+	BuildProductConfig = 1 << iota
+	BuildSoong         = 1 << iota
+	BuildKati          = 1 << iota
+	BuildNinja         = 1 << iota
+	BuildAll           = BuildProductConfig | BuildSoong | BuildKati | BuildNinja
+)
+
+// Build the tree. The 'what' argument can be used to chose which components of
+// the build to run.
+func Build(ctx Context, config Config, what int) {
+	ctx.Verboseln("Starting build with args:", config.Arguments())
+	ctx.Verboseln("Environment:", config.Environment().Environ())
+
+	if inList("help", config.Arguments()) {
+		cmd := exec.CommandContext(ctx.Context, "make", "-f", "build/core/help.mk")
+		cmd.Env = config.Environment().Environ()
+		cmd.Stdout = ctx.Stdout()
+		cmd.Stderr = ctx.Stderr()
+		if err := cmd.Run(); err != nil {
+			ctx.Fatalln("Failed to run make:", err)
+		}
+		return
+	}
+
+	SetupOutDir(ctx, config)
+
+	if what&BuildProductConfig != 0 {
+		// Run make for product config
+		runMakeProductConfig(ctx, config)
+	}
+
+	if what&BuildSoong != 0 {
+		// Run Soong
+		runSoongBootstrap(ctx, config)
+		runSoong(ctx, config)
+	}
+
+	if what&BuildKati != 0 {
+		// Run ckati
+		runKati(ctx, config)
+	}
+
+	if what&BuildNinja != 0 {
+		// Write combined ninja file
+		createCombinedBuildNinjaFile(ctx, config)
+
+		// Run ninja
+		runNinja(ctx, config)
+	}
+}

ui/build/config.go

@@ -0,0 +1,274 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+type Config struct{ *configImpl }
+
+type configImpl struct {
+	// From the environment
+	arguments []string
+	goma      bool
+	environ   *Environment
+
+	// From the arguments
+	parallel  int
+	keepGoing int
+	verbose   bool
+
+	// From the product config
+	katiArgs   []string
+	ninjaArgs  []string
+	katiSuffix string
+}
+
+func NewConfig(ctx Context, args ...string) Config {
+	ret := &configImpl{
+		environ: OsEnvironment(),
+	}
+
+	ret.environ.Unset(
+		// We're already using it
+		"USE_SOONG_UI",
+
+		// We should never use GOROOT/GOPATH from the shell environment
+		"GOROOT",
+		"GOPATH",
+
+		// These should only come from Soong, not the environment.
+		"CLANG",
+		"CLANG_CXX",
+		"CCC_CC",
+		"CCC_CXX",
+
+		// Used by the goma compiler wrapper, but should only be set by
+		// gomacc
+		"GOMACC_PATH",
+	)
+
+	// Tell python not to spam the source tree with .pyc files.
+	ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
+
+	// Sane default matching ninja
+	ret.parallel = runtime.NumCPU() + 2
+	ret.keepGoing = 1
+
+	for _, arg := range args {
+		arg = strings.TrimSpace(arg)
+		if arg == "--make-mode" {
+			continue
+		} else if arg == "showcommands" {
+			ret.verbose = true
+			continue
+		}
+		if arg[0] == '-' {
+			var err error
+			if arg[1] == 'j' {
+				// TODO: handle space between j and number
+				// Unnecessary if used with makeparallel
+				ret.parallel, err = strconv.Atoi(arg[2:])
+			} else if arg[1] == 'k' {
+				// TODO: handle space between k and number
+				// Unnecessary if used with makeparallel
+				ret.keepGoing, err = strconv.Atoi(arg[2:])
+			} else {
+				ctx.Fatalln("Unknown option:", arg)
+			}
+			if err != nil {
+				ctx.Fatalln("Argument error:", err, arg)
+			}
+		} else {
+			ret.arguments = append(ret.arguments, arg)
+		}
+	}
+
+	return Config{ret}
+}
+
+// Lunch configures the environment for a specific product similarly to the
+// `lunch` bash function.
+func (c *configImpl) Lunch(ctx Context, product, variant string) {
+	if variant != "eng" && variant != "userdebug" && variant != "user" {
+		ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
+	}
+
+	c.environ.Set("TARGET_PRODUCT", product)
+	c.environ.Set("TARGET_BUILD_VARIANT", variant)
+	c.environ.Set("TARGET_BUILD_TYPE", "release")
+	c.environ.Unset("TARGET_BUILD_APPS")
+}
+
+// Tapas configures the environment to build one or more unbundled apps,
+// similarly to the `tapas` bash function.
+func (c *configImpl) Tapas(ctx Context, apps []string, arch, variant string) {
+	if len(apps) == 0 {
+		apps = []string{"all"}
+	}
+	if variant == "" {
+		variant = "eng"
+	}
+
+	if variant != "eng" && variant != "userdebug" && variant != "user" {
+		ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
+	}
+
+	var product string
+	switch arch {
+	case "armv5":
+		product = "generic_armv5"
+	case "arm", "":
+		product = "aosp_arm"
+	case "arm64":
+		product = "aosm_arm64"
+	case "mips":
+		product = "aosp_mips"
+	case "mips64":
+		product = "aosp_mips64"
+	case "x86":
+		product = "aosp_x86"
+	case "x86_64":
+		product = "aosp_x86_64"
+	default:
+		ctx.Fatalf("Invalid architecture: %q", arch)
+	}
+
+	c.environ.Set("TARGET_PRODUCT", product)
+	c.environ.Set("TARGET_BUILD_VARIANT", variant)
+	c.environ.Set("TARGET_BUILD_TYPE", "release")
+	c.environ.Set("TARGET_BUILD_APPS", strings.Join(apps, " "))
+}
+
+func (c *configImpl) Environment() *Environment {
+	return c.environ
+}
+
+func (c *configImpl) Arguments() []string {
+	return c.arguments
+}
+
+func (c *configImpl) OutDir() string {
+	if outDir, ok := c.environ.Get("OUT_DIR"); ok {
+		return outDir
+	}
+	return "out"
+}
+
+func (c *configImpl) NinjaArgs() []string {
+	return c.ninjaArgs
+}
+
+func (c *configImpl) SoongOutDir() string {
+	return filepath.Join(c.OutDir(), "soong")
+}
+
+func (c *configImpl) KatiSuffix() string {
+	if c.katiSuffix != "" {
+		return c.katiSuffix
+	}
+	panic("SetKatiSuffix has not been called")
+}
+
+func (c *configImpl) IsVerbose() bool {
+	return c.verbose
+}
+
+func (c *configImpl) TargetProduct() string {
+	if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
+		return v
+	}
+	panic("TARGET_PRODUCT is not defined")
+}
+
+func (c *configImpl) KatiArgs() []string {
+	return c.katiArgs
+}
+
+func (c *configImpl) Parallel() int {
+	return c.parallel
+}
+
+func (c *configImpl) UseGoma() bool {
+	if v, ok := c.environ.Get("USE_GOMA"); ok {
+		v = strings.TrimSpace(v)
+		if v != "" && v != "false" {
+			return true
+		}
+	}
+	return false
+}
+
+// RemoteParallel controls how many remote jobs (i.e., commands which contain
+// gomacc) are run in parallel.  Note the paralleism of all other jobs is
+// still limited by Parallel()
+func (c *configImpl) RemoteParallel() int {
+	if v, ok := c.environ.Get("NINJA_REMOTE_NUM_JOBS"); ok {
+		if i, err := strconv.Atoi(v); err == nil {
+			return i
+		}
+	}
+	return 500
+}
+
+func (c *configImpl) SetKatiArgs(args []string) {
+	c.katiArgs = args
+}
+
+func (c *configImpl) SetNinjaArgs(args []string) {
+	c.ninjaArgs = args
+}
+
+func (c *configImpl) SetKatiSuffix(suffix string) {
+	c.katiSuffix = suffix
+}
+
+func (c *configImpl) KatiEnvFile() string {
+	return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
+}
+
+func (c *configImpl) KatiNinjaFile() string {
+	return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+".ninja")
+}
+
+func (c *configImpl) SoongNinjaFile() string {
+	return filepath.Join(c.SoongOutDir(), "build.ninja")
+}
+
+func (c *configImpl) CombinedNinjaFile() string {
+	return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
+}
+
+func (c *configImpl) SoongAndroidMk() string {
+	return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
+}
+
+func (c *configImpl) SoongMakeVarsMk() string {
+	return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
+}
+
+func (c *configImpl) HostPrebuiltTag() string {
+	if runtime.GOOS == "linux" {
+		return "linux-x86"
+	} else if runtime.GOOS == "darwin" {
+		return "darwin-x86"
+	} else {
+		panic("Unsupported OS")
+	}
+}

ui/build/context.go

@@ -0,0 +1,64 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"android/soong/ui/logger"
+)
+
+type StdioInterface interface {
+	Stdin() io.Reader
+	Stdout() io.Writer
+	Stderr() io.Writer
+}
+
+type StdioImpl struct{}
+
+func (StdioImpl) Stdin() io.Reader  { return os.Stdin }
+func (StdioImpl) Stdout() io.Writer { return os.Stdout }
+func (StdioImpl) Stderr() io.Writer { return os.Stderr }
+
+var _ StdioInterface = StdioImpl{}
+
+type customStdio struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+}
+
+func NewCustomStdio(stdin io.Reader, stdout, stderr io.Writer) StdioInterface {
+	return customStdio{stdin, stdout, stderr}
+}
+
+func (c customStdio) Stdin() io.Reader  { return c.stdin }
+func (c customStdio) Stdout() io.Writer { return c.stdout }
+func (c customStdio) Stderr() io.Writer { return c.stderr }
+
+var _ StdioInterface = customStdio{}
+
+// Context combines a context.Context, logger.Logger, and StdIO redirection.
+// These all are agnostic of the current build, and may be used for multiple
+// builds, while the Config objects contain per-build information.
+type Context *ContextImpl
+type ContextImpl struct {
+	context.Context
+	logger.Logger
+
+	StdioInterface
+}

ui/build/environment.go

@@ -0,0 +1,152 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// Environment adds a number of useful manipulation functions to the list of
+// strings returned by os.Environ() and used in exec.Cmd.Env.
+type Environment []string
+
+// OsEnvironment wraps the current environment returned by os.Environ()
+func OsEnvironment() *Environment {
+	env := Environment(os.Environ())
+	return &env
+}
+
+// Get returns the value associated with the key, and whether it exists.
+// It's equivalent to the os.LookupEnv function, but with this copy of the
+// Environment.
+func (e *Environment) Get(key string) (string, bool) {
+	for _, env := range *e {
+		if k, v, ok := decodeKeyValue(env); ok && k == key {
+			return v, true
+		}
+	}
+	return "", false
+}
+
+// Set sets the value associated with the key, overwriting the current value
+// if it exists.
+func (e *Environment) Set(key, value string) {
+	e.Unset(key)
+	*e = append(*e, key+"="+value)
+}
+
+// Unset removes the specified keys from the Environment.
+func (e *Environment) Unset(keys ...string) {
+	out := (*e)[:0]
+	for _, env := range *e {
+		if key, _, ok := decodeKeyValue(env); ok && inList(key, keys) {
+			continue
+		}
+		out = append(out, env)
+	}
+	*e = out
+}
+
+// Environ returns the []string required for exec.Cmd.Env
+func (e *Environment) Environ() []string {
+	return []string(*e)
+}
+
+// Copy returns a copy of the Environment so that independent changes may be made.
+func (e *Environment) Copy() *Environment {
+	ret := Environment(make([]string, len(*e)))
+	for i, v := range *e {
+		ret[i] = v
+	}
+	return &ret
+}
+
+// IsTrue returns whether an environment variable is set to a positive value (1,y,yes,on,true)
+func (e *Environment) IsEnvTrue(key string) bool {
+	if value, ok := e.Get(key); ok {
+		return value == "1" || value == "y" || value == "yes" || value == "on" || value == "true"
+	}
+	return false
+}
+
+// IsFalse returns whether an environment variable is set to a negative value (0,n,no,off,false)
+func (e *Environment) IsFalse(key string) bool {
+	if value, ok := e.Get(key); ok {
+		return value == "0" || value == "n" || value == "no" || value == "off" || value == "false"
+	}
+	return false
+}
+
+// AppendFromKati reads a shell script written by Kati that exports or unsets
+// environment variables, and applies those to the local Environment.
+func (e *Environment) AppendFromKati(filename string) error {
+	file, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return e.appendFromKati(file)
+}
+
+func (e *Environment) appendFromKati(reader io.Reader) error {
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+
+		if len(text) == 0 || text[0] == '#' {
+			continue
+		}
+
+		cmd := strings.SplitN(text, " ", 2)
+		if len(cmd) != 2 {
+			return fmt.Errorf("Unknown kati environment line: %q", text)
+		}
+
+		if cmd[0] == "unset" {
+			str, ok := singleUnquote(cmd[1])
+			if !ok {
+				fmt.Errorf("Failed to unquote kati line: %q", text)
+			}
+			e.Unset(str)
+		} else if cmd[0] == "export" {
+			key, value, ok := decodeKeyValue(cmd[1])
+			if !ok {
+				return fmt.Errorf("Failed to parse export: %v", cmd)
+			}
+
+			key, ok = singleUnquote(key)
+			if !ok {
+				return fmt.Errorf("Failed to unquote kati line: %q", text)
+			}
+			value, ok = singleUnquote(value)
+			if !ok {
+				return fmt.Errorf("Failed to unquote kati line: %q", text)
+			}
+
+			e.Set(key, value)
+		} else {
+			return fmt.Errorf("Unknown kati environment command: %q", text)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+	return nil
+}

ui/build/environment_test.go

@@ -0,0 +1,80 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestEnvUnset(t *testing.T) {
+	initial := &Environment{"TEST=1", "TEST2=0"}
+	initial.Unset("TEST")
+	got := initial.Environ()
+	if len(got) != 1 || got[0] != "TEST2=0" {
+		t.Errorf("Expected [TEST2=0], got: %v", got)
+	}
+}
+
+func TestEnvUnsetMissing(t *testing.T) {
+	initial := &Environment{"TEST2=0"}
+	initial.Unset("TEST")
+	got := initial.Environ()
+	if len(got) != 1 || got[0] != "TEST2=0" {
+		t.Errorf("Expected [TEST2=0], got: %v", got)
+	}
+}
+
+func TestEnvSet(t *testing.T) {
+	initial := &Environment{}
+	initial.Set("TEST", "0")
+	got := initial.Environ()
+	if len(got) != 1 || got[0] != "TEST=0" {
+		t.Errorf("Expected [TEST=0], got: %v", got)
+	}
+}
+
+func TestEnvSetDup(t *testing.T) {
+	initial := &Environment{"TEST=1"}
+	initial.Set("TEST", "0")
+	got := initial.Environ()
+	if len(got) != 1 || got[0] != "TEST=0" {
+		t.Errorf("Expected [TEST=0], got: %v", got)
+	}
+}
+
+const testKatiEnvFileContents = `#!/bin/sh
+# Generated by kati unknown
+
+unset 'CLANG'
+export 'BUILD_ID'='NYC'
+`
+
+func TestEnvAppendFromKati(t *testing.T) {
+	initial := &Environment{"CLANG=/usr/bin/clang", "TEST=0"}
+	err := initial.appendFromKati(strings.NewReader(testKatiEnvFileContents))
+	if err != nil {
+		t.Fatalf("Unexpected error from %v", err)
+	}
+
+	got := initial.Environ()
+	expected := []string{"TEST=0", "BUILD_ID=NYC"}
+	if !reflect.DeepEqual(got, expected) {
+		t.Errorf("Environment list does not match")
+		t.Errorf("expected: %v", expected)
+		t.Errorf("     got: %v", got)
+	}
+}

ui/build/kati.go

@@ -0,0 +1,104 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+var spaceSlashReplacer = strings.NewReplacer("/", "_", " ", "_")
+
+// genKatiSuffix creates a suffix for kati-generated files so that we can cache
+// them based on their inputs. So this should encode all common changes to Kati
+// inputs. Currently that includes the TARGET_PRODUCT, kati-processed command
+// line arguments, and the directories specified by mm/mmm.
+func genKatiSuffix(ctx Context, config Config) {
+	katiSuffix := "-" + config.TargetProduct()
+	if args := config.KatiArgs(); len(args) > 0 {
+		katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_"))
+	}
+	if oneShot, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok {
+		katiSuffix += "-" + spaceSlashReplacer.Replace(oneShot)
+	}
+
+	// If the suffix is too long, replace it with a md5 hash and write a
+	// file that contains the original suffix.
+	if len(katiSuffix) > 64 {
+		shortSuffix := "-" + fmt.Sprintf("%x", md5.Sum([]byte(katiSuffix)))
+		config.SetKatiSuffix(shortSuffix)
+
+		ctx.Verbosef("Kati ninja suffix too long: %q", katiSuffix)
+		ctx.Verbosef("Replacing with: %q", shortSuffix)
+
+		if err := ioutil.WriteFile(strings.TrimSuffix(config.KatiNinjaFile(), "ninja")+"suf", []byte(katiSuffix), 0777); err != nil {
+			ctx.Println("Error writing suffix file:", err)
+		}
+	} else {
+		config.SetKatiSuffix(katiSuffix)
+	}
+}
+
+func runKati(ctx Context, config Config) {
+	genKatiSuffix(ctx, config)
+
+	executable := "prebuilts/build-tools/" + config.HostPrebuiltTag() + "/bin/ckati"
+	args := []string{
+		"--ninja",
+		"--ninja_dir=" + config.OutDir(),
+		"--ninja_suffix=" + config.KatiSuffix(),
+		"--regen",
+		"--ignore_optional_include=" + filepath.Join(config.OutDir(), "%.P"),
+		"--detect_android_echo",
+	}
+
+	if !config.Environment().IsFalse("KATI_EMULATE_FIND") {
+		args = append(args, "--use_find_emulator")
+	}
+
+	// The argument order could be simplified, but currently this matches
+	// the ordering in Make
+	args = append(args, "-f", "build/core/main.mk")
+
+	args = append(args, config.KatiArgs()...)
+
+	args = append(args,
+		"--gen_all_targets",
+		"BUILDING_WITH_NINJA=true",
+		"SOONG_ANDROID_MK="+config.SoongAndroidMk(),
+		"SOONG_MAKEVARS_MK="+config.SoongMakeVarsMk())
+
+	if config.UseGoma() {
+		args = append(args, "-j"+strconv.Itoa(config.Parallel()))
+	}
+
+	cmd := exec.CommandContext(ctx.Context, executable, args...)
+	cmd.Env = config.Environment().Environ()
+	cmd.Stdout = ctx.Stdout()
+	cmd.Stderr = ctx.Stderr()
+	ctx.Verboseln(cmd.Path, cmd.Args)
+	if err := cmd.Run(); err != nil {
+		if e, ok := err.(*exec.ExitError); ok {
+			ctx.Fatalln("ckati failed with:", e.ProcessState.String())
+		} else {
+			ctx.Fatalln("Failed to run ckati:", err)
+		}
+	}
+}

ui/build/make.go

@@ -0,0 +1,160 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// DumpMakeVars can be used to extract the values of Make variables after the
+// product configurations are loaded. This is roughly equivalent to the
+// `get_build_var` bash function.
+//
+// goals can be used to set MAKECMDGOALS, which emulates passing arguments to
+// Make without actually building them. So all the variables based on
+// MAKECMDGOALS can be read.
+//
+// extra_targets adds real arguments to the make command, in case other targets
+// actually need to be run (like the Soong config generator).
+//
+// vars is the list of variables to read. The values will be put in the
+// returned map.
+func DumpMakeVars(ctx Context, config Config, goals, extra_targets, vars []string) (map[string]string, error) {
+	cmd := exec.CommandContext(ctx.Context,
+		"make",
+		"--no-print-directory",
+		"-f", "build/core/config.mk",
+		"dump-many-vars",
+		"CALLED_FROM_SETUP=true",
+		"BUILD_SYSTEM=build/core",
+		"MAKECMDGOALS="+strings.Join(goals, " "),
+		"DUMP_MANY_VARS="+strings.Join(vars, " "),
+		"OUT_DIR="+config.OutDir())
+	cmd.Env = config.Environment().Environ()
+	cmd.Args = append(cmd.Args, extra_targets...)
+	// TODO: error out when Stderr contains any content
+	cmd.Stderr = ctx.Stderr()
+	ctx.Verboseln(cmd.Path, cmd.Args)
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make(map[string]string, len(vars))
+	for _, line := range strings.Split(string(output), "\n") {
+		if len(line) == 0 {
+			continue
+		}
+
+		if key, value, ok := decodeKeyValue(line); ok {
+			if value, ok = singleUnquote(value); ok {
+				ret[key] = value
+				ctx.Verboseln(key, value)
+			} else {
+				return nil, fmt.Errorf("Failed to parse make line: %q", line)
+			}
+		} else {
+			return nil, fmt.Errorf("Failed to parse make line: %q", line)
+		}
+	}
+
+	return ret, nil
+}
+
+func runMakeProductConfig(ctx Context, config Config) {
+	// Variables to export into the environment of Kati/Ninja
+	exportEnvVars := []string{
+		// So that we can use the correct TARGET_PRODUCT if it's been
+		// modified by PRODUCT-* arguments
+		"TARGET_PRODUCT",
+
+		// compiler wrappers set up by make
+		"CC_WRAPPER",
+		"CXX_WRAPPER",
+
+		// ccache settings
+		"CCACHE_COMPILERCHECK",
+		"CCACHE_SLOPPINESS",
+		"CCACHE_BASEDIR",
+		"CCACHE_CPP2",
+	}
+
+	// Variables to print out in the top banner
+	bannerVars := []string{
+		"PLATFORM_VERSION_CODENAME",
+		"PLATFORM_VERSION",
+		"TARGET_PRODUCT",
+		"TARGET_BUILD_VARIANT",
+		"TARGET_BUILD_TYPE",
+		"TARGET_BUILD_APPS",
+		"TARGET_ARCH",
+		"TARGET_ARCH_VARIANT",
+		"TARGET_CPU_VARIANT",
+		"TARGET_2ND_ARCH",
+		"TARGET_2ND_ARCH_VARIANT",
+		"TARGET_2ND_CPU_VARIANT",
+		"HOST_ARCH",
+		"HOST_2ND_ARCH",
+		"HOST_OS",
+		"HOST_OS_EXTRA",
+		"HOST_CROSS_OS",
+		"HOST_CROSS_ARCH",
+		"HOST_CROSS_2ND_ARCH",
+		"HOST_BUILD_TYPE",
+		"BUILD_ID",
+		"OUT_DIR",
+		"AUX_OS_VARIANT_LIST",
+		"TARGET_BUILD_PDK",
+		"PDK_FUSION_PLATFORM_ZIP",
+	}
+
+	allVars := append(append([]string{
+		// Used to execute Kati and Ninja
+		"NINJA_GOALS",
+		"KATI_GOALS",
+	}, exportEnvVars...), bannerVars...)
+
+	make_vars, err := DumpMakeVars(ctx, config, config.Arguments(), []string{
+		filepath.Join(config.SoongOutDir(), "soong.variables"),
+	}, allVars)
+	if err != nil {
+		ctx.Fatalln("Error dumping make vars:", err)
+	}
+
+	// Print the banner like make does
+	fmt.Fprintln(ctx.Stdout(), "============================================")
+	for _, name := range bannerVars {
+		if make_vars[name] != "" {
+			fmt.Fprintf(ctx.Stdout(), "%s=%s\n", name, make_vars[name])
+		}
+	}
+	fmt.Fprintln(ctx.Stdout(), "============================================")
+
+	// Populate the environment
+	env := config.Environment()
+	for _, name := range exportEnvVars {
+		if make_vars[name] == "" {
+			env.Unset(name)
+		} else {
+			env.Set(name, make_vars[name])
+		}
+	}
+
+	config.SetKatiArgs(strings.Fields(make_vars["KATI_GOALS"]))
+	config.SetNinjaArgs(strings.Fields(make_vars["NINJA_GOALS"]))
+}

ui/build/ninja.go

@@ -0,0 +1,78 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+func runNinja(ctx Context, config Config) {
+	executable := "prebuilts/build-tools/" + config.HostPrebuiltTag() + "/bin/ninja"
+	args := []string{
+		"-d", "keepdepfile",
+	}
+
+	args = append(args, config.NinjaArgs()...)
+
+	var parallel int
+	if config.UseGoma() {
+		parallel = config.RemoteParallel()
+	} else {
+		parallel = config.Parallel()
+	}
+	args = append(args, "-j", strconv.Itoa(parallel))
+	if config.keepGoing != 1 {
+		args = append(args, "-k", strconv.Itoa(config.keepGoing))
+	}
+
+	args = append(args, "-f", config.CombinedNinjaFile())
+
+	if config.IsVerbose() {
+		args = append(args, "-v")
+	}
+	args = append(args, "-w", "dupbuild=err")
+
+	env := config.Environment().Copy()
+	env.AppendFromKati(config.KatiEnvFile())
+
+	// Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been
+	// used in the past to specify extra ninja arguments.
+	if extra, ok := env.Get("NINJA_ARGS"); ok {
+		args = append(args, strings.Fields(extra)...)
+	}
+	if extra, ok := env.Get("NINJA_EXTRA_ARGS"); ok {
+		args = append(args, strings.Fields(extra)...)
+	}
+
+	if _, ok := env.Get("NINJA_STATUS"); !ok {
+		env.Set("NINJA_STATUS", "[%p %f/%t] ")
+	}
+
+	cmd := exec.CommandContext(ctx.Context, executable, args...)
+	cmd.Env = env.Environ()
+	cmd.Stdin = ctx.Stdin()
+	cmd.Stdout = ctx.Stdout()
+	cmd.Stderr = ctx.Stderr()
+	ctx.Verboseln(cmd.Path, cmd.Args)
+	if err := cmd.Run(); err != nil {
+		if e, ok := err.(*exec.ExitError); ok {
+			ctx.Fatalln("ninja failed with:", e.ProcessState.String())
+		} else {
+			ctx.Fatalln("Failed to run ninja:", err)
+		}
+	}
+}

ui/build/signal.go

@@ -0,0 +1,60 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"os"
+	"os/signal"
+	"runtime/debug"
+	"syscall"
+
+	"android/soong/ui/logger"
+)
+
+// SetupSignals sets up signal handling to kill our children and allow us to cleanly finish
+// writing our log/trace files.
+//
+// Currently, on the first SIGINT|SIGALARM we call the cancel() function, which is usually
+// the CancelFunc returned by context.WithCancel, which will kill all the commands running
+// within that Context. Usually that's enough, and you'll run through your normal error paths.
+//
+// If another signal comes in after the first one, we'll trigger a panic with full stacktraces
+// from every goroutine so that it's possible to debug what is stuck. Just before the process
+// exits, we'll call the cleanup() function so that you can flush your log files.
+func SetupSignals(log logger.Logger, cancel, cleanup func()) {
+	signals := make(chan os.Signal, 5)
+	// TODO: Handle other signals
+	signal.Notify(signals, os.Interrupt, syscall.SIGALRM)
+	go handleSignals(signals, log, cancel, cleanup)
+}
+
+func handleSignals(signals chan os.Signal, log logger.Logger, cancel, cleanup func()) {
+	defer cleanup()
+
+	var force bool
+
+	for {
+		s := <-signals
+		if force {
+			// So that we can better see what was stuck
+			debug.SetTraceback("all")
+			log.Panicln("Second signal received:", s)
+		} else {
+			log.Println("Got signal:", s)
+			cancel()
+			force = true
+		}
+	}
+}

ui/build/soong.go

@@ -0,0 +1,57 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"os/exec"
+	"path/filepath"
+)
+
+func runSoongBootstrap(ctx Context, config Config) {
+	cmd := exec.CommandContext(ctx.Context, "./bootstrap.bash")
+	env := config.Environment().Copy()
+	env.Set("BUILDDIR", config.SoongOutDir())
+	cmd.Env = env.Environ()
+	cmd.Stdout = ctx.Stdout()
+	cmd.Stderr = ctx.Stderr()
+	ctx.Verboseln(cmd.Path, cmd.Args)
+	if err := cmd.Run(); err != nil {
+		if e, ok := err.(*exec.ExitError); ok {
+			ctx.Fatalln("soong bootstrap failed with:", e.ProcessState.String())
+		} else {
+			ctx.Fatalln("Failed to run soong bootstrap:", err)
+		}
+	}
+}
+
+func runSoong(ctx Context, config Config) {
+	cmd := exec.CommandContext(ctx.Context, filepath.Join(config.SoongOutDir(), "soong"), "-w", "dupbuild=err")
+	if config.IsVerbose() {
+		cmd.Args = append(cmd.Args, "-v")
+	}
+	env := config.Environment().Copy()
+	env.Set("SKIP_NINJA", "true")
+	cmd.Env = env.Environ()
+	cmd.Stdout = ctx.Stdout()
+	cmd.Stderr = ctx.Stderr()
+	ctx.Verboseln(cmd.Path, cmd.Args)
+	if err := cmd.Run(); err != nil {
+		if e, ok := err.(*exec.ExitError); ok {
+			ctx.Fatalln("soong bootstrap failed with:", e.ProcessState.String())
+		} else {
+			ctx.Fatalln("Failed to run soong bootstrap:", err)
+		}
+	}
+}

ui/build/util.go

@@ -0,0 +1,79 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// indexList finds the index of a string in a []string
+func indexList(s string, list []string) int {
+	for i, l := range list {
+		if l == s {
+			return i
+		}
+	}
+
+	return -1
+}
+
+// inList determines whether a string is in a []string
+func inList(s string, list []string) bool {
+	return indexList(s, list) != -1
+}
+
+// ensureDirectoriesExist is a shortcut to os.MkdirAll, sending errors to the ctx logger.
+func ensureDirectoriesExist(ctx Context, dirs ...string) {
+	for _, dir := range dirs {
+		err := os.MkdirAll(dir, 0777)
+		if err != nil {
+			ctx.Fatalf("Error creating %s: %q\n", dir, err)
+		}
+	}
+}
+
+// ensureEmptyFileExists ensures that the containing directory exists, and the
+// specified file exists. If it doesn't exist, it will write an empty file.
+func ensureEmptyFileExists(ctx Context, file string) {
+	ensureDirectoriesExist(ctx, filepath.Dir(file))
+	if _, err := os.Stat(file); os.IsNotExist(err) {
+		f, err := os.Create(file)
+		if err != nil {
+			ctx.Fatalf("Error creating %s: %q\n", file, err)
+		}
+		f.Close()
+	} else if err != nil {
+		ctx.Fatalf("Error checking %s: %q\n", file, err)
+	}
+}
+
+// singleUnquote is similar to strconv.Unquote, but can handle multi-character strings inside single quotes.
+func singleUnquote(str string) (string, bool) {
+	if len(str) < 2 || str[0] != '\'' || str[len(str)-1] != '\'' {
+		return "", false
+	}
+	return str[1 : len(str)-1], true
+}
+
+// decodeKeyValue decodes a key=value string
+func decodeKeyValue(str string) (string, string, bool) {
+	idx := strings.IndexRune(str, '=')
+	if idx == -1 {
+		return "", "", false
+	}
+	return str[:idx], str[idx+1:], true
+}

ui/logger/Android.bp

@@ -0,0 +1,24 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+bootstrap_go_package {
+    name: "soong-ui-logger",
+    pkgPath: "android/soong/ui/logger",
+    srcs: [
+        "logger.go",
+    ],
+    testSrcs: [
+        "logger_test.go",
+    ],
+}

ui/logger/logger.go

@@ -0,0 +1,302 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package logger implements a logging package designed for command line
+// utilities.  It uses the standard 'log' package and function, but splits
+// output between stderr and a rotating log file.
+//
+// In addition to the standard logger functions, Verbose[f|ln] calls only go to
+// the log file by default, unless SetVerbose(true) has been called.
+//
+// The log file also includes extended date/time/source information, which are
+// omitted from the stderr output for better readability.
+//
+// In order to better handle resource cleanup after a Fatal error, the Fatal
+// functions panic instead of calling os.Exit(). To actually do the cleanup,
+// and prevent the printing of the panic, call defer logger.Cleanup() at the
+// beginning of your main function.
+package logger
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+)
+
+type Logger interface {
+	// Print* prints to both stderr and the file log.
+	// Arguments to Print are handled in the manner of fmt.Print.
+	Print(v ...interface{})
+	// Arguments to Printf are handled in the manner of fmt.Printf
+	Printf(format string, v ...interface{})
+	// Arguments to Println are handled in the manner of fmt.Println
+	Println(v ...interface{})
+
+	// Verbose* is equivalent to Print*, but skips stderr unless the
+	// logger has been configured in verbose mode.
+	Verbose(v ...interface{})
+	Verbosef(format string, v ...interface{})
+	Verboseln(v ...interface{})
+
+	// Fatal* is equivalent to Print* followed by a call to panic that
+	// can be converted to an error using Recover, or will be converted
+	// to a call to os.Exit(1) with a deferred call to Cleanup()
+	Fatal(v ...interface{})
+	Fatalf(format string, v ...interface{})
+	Fatalln(v ...interface{})
+
+	// Panic is equivalent to Print* followed by a call to panic.
+	Panic(v ...interface{})
+	Panicf(format string, v ...interface{})
+	Panicln(v ...interface{})
+
+	// Output writes the string to both stderr and the file log.
+	Output(calldepth int, str string) error
+}
+
+// fatalLog is the type used when Fatal[f|ln]
+type fatalLog error
+
+func fileRotation(from, baseName, ext string, cur, max int) error {
+	newName := baseName + "." + strconv.Itoa(cur) + ext
+
+	if _, err := os.Lstat(newName); err == nil {
+		if cur+1 <= max {
+			fileRotation(newName, baseName, ext, cur+1, max)
+		}
+	}
+
+	if err := os.Rename(from, newName); err != nil {
+		return fmt.Errorf("Failed to rotate", from, "to", newName, ".", err)
+	}
+	return nil
+}
+
+// CreateFileWithRotation returns a new os.File using os.Create, renaming any
+// existing files to <filename>.#.<ext>, keeping up to maxCount files.
+// <filename>.1.<ext> is the most recent backup, <filename>.2.<ext> is the
+// second most recent backup, etc.
+//
+// TODO: This function is not guaranteed to be atomic, if there are multiple
+// users attempting to do the same operation, the result is undefined.
+func CreateFileWithRotation(filename string, maxCount int) (*os.File, error) {
+	if _, err := os.Lstat(filename); err == nil {
+		ext := filepath.Ext(filename)
+		basename := filename[:len(filename)-len(ext)]
+		if err = fileRotation(filename, basename, ext, 1, maxCount); err != nil {
+			return nil, err
+		}
+	}
+
+	return os.Create(filename)
+}
+
+// Recover can be used with defer in a GoRoutine to convert a Fatal panics to
+// an error that can be handled.
+func Recover(fn func(err error)) {
+	p := recover()
+
+	if p == nil {
+		return
+	} else if log, ok := p.(fatalLog); ok {
+		fn(error(log))
+	} else {
+		panic(p)
+	}
+}
+
+type stdLogger struct {
+	stderr  *log.Logger
+	verbose bool
+
+	fileLogger *log.Logger
+	mutex      sync.Mutex
+	file       *os.File
+}
+
+var _ Logger = &stdLogger{}
+
+// New creates a new Logger. The out variable sets the destination, commonly
+// os.Stderr, but it may be a buffer for tests, or a separate log file if
+// the user doesn't need to see the output.
+func New(out io.Writer) *stdLogger {
+	return &stdLogger{
+		stderr:     log.New(out, "", log.Ltime),
+		fileLogger: log.New(ioutil.Discard, "", log.Ldate|log.Lmicroseconds|log.Llongfile),
+	}
+}
+
+// SetVerbose controls whether Verbose[f|ln] logs to stderr as well as the
+// file-backed log.
+func (s *stdLogger) SetVerbose(v bool) {
+	s.verbose = v
+}
+
+// SetOutput controls where the file-backed log will be saved. It will keep
+// some number of backups of old log files.
+func (s *stdLogger) SetOutput(path string) {
+	if f, err := CreateFileWithRotation(path, 5); err == nil {
+		s.mutex.Lock()
+		defer s.mutex.Unlock()
+
+		if s.file != nil {
+			s.file.Close()
+		}
+		s.file = f
+		s.fileLogger.SetOutput(f)
+	} else {
+		s.Fatal(err.Error())
+	}
+}
+
+// Close disables logging to the file and closes the file handle.
+func (s *stdLogger) Close() {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if s.file != nil {
+		s.fileLogger.SetOutput(ioutil.Discard)
+		s.file.Close()
+		s.file = nil
+	}
+}
+
+// Cleanup should be used with defer in your main function. It will close the
+// log file and convert any Fatal panics back to os.Exit(1)
+func (s *stdLogger) Cleanup() {
+	fatal := false
+	p := recover()
+
+	if _, ok := p.(fatalLog); ok {
+		fatal = true
+		p = nil
+	} else if p != nil {
+		s.Println(p)
+	}
+
+	s.Close()
+
+	if p != nil {
+		panic(p)
+	} else if fatal {
+		os.Exit(1)
+	}
+}
+
+// Output writes string to both stderr and the file log.
+func (s *stdLogger) Output(calldepth int, str string) error {
+	s.stderr.Output(calldepth+1, str)
+	return s.fileLogger.Output(calldepth+1, str)
+}
+
+// VerboseOutput is equivalent to Output, but only goes to the file log
+// unless SetVerbose(true) has been called.
+func (s *stdLogger) VerboseOutput(calldepth int, str string) error {
+	if s.verbose {
+		s.stderr.Output(calldepth+1, str)
+	}
+	return s.fileLogger.Output(calldepth+1, str)
+}
+
+// Print prints to both stderr and the file log.
+// Arguments are handled in the manner of fmt.Print.
+func (s *stdLogger) Print(v ...interface{}) {
+	output := fmt.Sprint(v...)
+	s.Output(2, output)
+}
+
+// Printf prints to both stderr and the file log.
+// Arguments are handled in the manner of fmt.Printf.
+func (s *stdLogger) Printf(format string, v ...interface{}) {
+	output := fmt.Sprintf(format, v...)
+	s.Output(2, output)
+}
+
+// Println prints to both stderr and the file log.
+// Arguments are handled in the manner of fmt.Println.
+func (s *stdLogger) Println(v ...interface{}) {
+	output := fmt.Sprintln(v...)
+	s.Output(2, output)
+}
+
+// Verbose is equivalent to Print, but only goes to the file log unless
+// SetVerbose(true) has been called.
+func (s *stdLogger) Verbose(v ...interface{}) {
+	output := fmt.Sprint(v...)
+	s.VerboseOutput(2, output)
+}
+
+// Verbosef is equivalent to Printf, but only goes to the file log unless
+// SetVerbose(true) has been called.
+func (s *stdLogger) Verbosef(format string, v ...interface{}) {
+	output := fmt.Sprintf(format, v...)
+	s.VerboseOutput(2, output)
+}
+
+// Verboseln is equivalent to Println, but only goes to the file log unless
+// SetVerbose(true) has been called.
+func (s *stdLogger) Verboseln(v ...interface{}) {
+	output := fmt.Sprintln(v...)
+	s.VerboseOutput(2, output)
+}
+
+// Fatal is equivalent to Print() followed by a call to panic() that
+// Cleanup will convert to a os.Exit(1).
+func (s *stdLogger) Fatal(v ...interface{}) {
+	output := fmt.Sprint(v...)
+	s.Output(2, output)
+	panic(fatalLog(errors.New(output)))
+}
+
+// Fatalf is equivalent to Printf() followed by a call to panic() that
+// Cleanup will convert to a os.Exit(1).
+func (s *stdLogger) Fatalf(format string, v ...interface{}) {
+	output := fmt.Sprintf(format, v...)
+	s.Output(2, output)
+	panic(fatalLog(errors.New(output)))
+}
+
+// Fatalln is equivalent to Println() followed by a call to panic() that
+// Cleanup will convert to a os.Exit(1).
+func (s *stdLogger) Fatalln(v ...interface{}) {
+	output := fmt.Sprintln(v...)
+	s.Output(2, output)
+	panic(fatalLog(errors.New(output)))
+}
+
+// Panic is equivalent to Print() followed by a call to panic().
+func (s *stdLogger) Panic(v ...interface{}) {
+	output := fmt.Sprint(v...)
+	s.Output(2, output)
+	panic(output)
+}
+
+// Panicf is equivalent to Printf() followed by a call to panic().
+func (s *stdLogger) Panicf(format string, v ...interface{}) {
+	output := fmt.Sprintf(format, v...)
+	s.Output(2, output)
+	panic(output)
+}
+
+// Panicln is equivalent to Println() followed by a call to panic().
+func (s *stdLogger) Panicln(v ...interface{}) {
+	output := fmt.Sprintln(v...)
+	s.Output(2, output)
+	panic(output)
+}

ui/logger/logger_test.go

@@ -0,0 +1,198 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logger
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"syscall"
+	"testing"
+)
+
+func TestCreateFileWithRotation(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test-rotation")
+	if err != nil {
+		t.Fatalf("Failed to get TempDir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "build.log")
+
+	writeFile := func(name string, data string) {
+		f, err := CreateFileWithRotation(name, 3)
+		if err != nil {
+			t.Fatalf("Failed to create file: %v", err)
+		}
+		if n, err := io.WriteString(f, data); err == nil && n < len(data) {
+			t.Fatalf("Short write")
+		} else if err != nil {
+			t.Fatalf("Failed to write: %v", err)
+		}
+		if err := f.Close(); err != nil {
+			t.Fatalf("Failed to close: %v", err)
+		}
+	}
+
+	writeFile(file, "a")
+	writeFile(file, "b")
+	writeFile(file, "c")
+	writeFile(file, "d")
+	writeFile(file, "e")
+
+	d, err := os.Open(dir)
+	if err != nil {
+		t.Fatalf("Failed to open dir: %v", err)
+	}
+	names, err := d.Readdirnames(0)
+	if err != nil {
+		t.Fatalf("Failed to read dir: %v", err)
+	}
+	sort.Strings(names)
+	expected := []string{"build.1.log", "build.2.log", "build.3.log", "build.log"}
+	if !reflect.DeepEqual(names, expected) {
+		t.Errorf("File list does not match.")
+		t.Errorf("     got: %v", names)
+		t.Errorf("expected: %v", expected)
+		t.FailNow()
+	}
+
+	expectFileContents := func(name, expected string) {
+		data, err := ioutil.ReadFile(filepath.Join(dir, name))
+		if err != nil {
+			t.Errorf("Error reading file: %v", err)
+			return
+		}
+		str := string(data)
+		if str != expected {
+			t.Errorf("Contents of %v does not match.", name)
+			t.Errorf("     got: %v", data)
+			t.Errorf("expected: %v", expected)
+		}
+	}
+
+	expectFileContents("build.log", "e")
+	expectFileContents("build.1.log", "d")
+	expectFileContents("build.2.log", "c")
+	expectFileContents("build.3.log", "b")
+}
+
+func TestPanic(t *testing.T) {
+	if os.Getenv("ACTUALLY_PANIC") == "1" {
+		panicValue := "foo"
+		log := New(&bytes.Buffer{})
+
+		defer func() {
+			p := recover()
+
+			if p == panicValue {
+				os.Exit(42)
+			} else {
+				fmt.Fprintln(os.Stderr, "Expected %q, got %v", panicValue, p)
+				os.Exit(3)
+			}
+		}()
+		defer log.Cleanup()
+
+		log.Panic(panicValue)
+		os.Exit(2)
+		return
+	}
+
+	// Run this in an external process so that we don't pollute stderr
+	cmd := exec.Command(os.Args[0], "-test.run=TestPanic")
+	cmd.Env = append(os.Environ(), "ACTUALLY_PANIC=1")
+	err := cmd.Run()
+	if e, ok := err.(*exec.ExitError); ok && e.Sys().(syscall.WaitStatus).ExitStatus() == 42 {
+		return
+	}
+	t.Errorf("Expected process to exit with status 42, got %v", err)
+}
+
+func TestFatal(t *testing.T) {
+	if os.Getenv("ACTUALLY_FATAL") == "1" {
+		log := New(&bytes.Buffer{})
+		defer func() {
+			// Shouldn't get here
+			os.Exit(3)
+		}()
+		defer log.Cleanup()
+		log.Fatal("Test")
+		os.Exit(0)
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run=TestFatal")
+	cmd.Env = append(os.Environ(), "ACTUALLY_FATAL=1")
+	err := cmd.Run()
+	if e, ok := err.(*exec.ExitError); ok && e.Sys().(syscall.WaitStatus).ExitStatus() == 1 {
+		return
+	}
+	t.Errorf("Expected process to exit with status 1, got %v", err)
+}
+
+func TestNonFatal(t *testing.T) {
+	if os.Getenv("ACTUAL_TEST") == "1" {
+		log := New(&bytes.Buffer{})
+		defer log.Cleanup()
+		log.Println("Test")
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run=TestNonFatal")
+	cmd.Env = append(os.Environ(), "ACTUAL_TEST=1")
+	err := cmd.Run()
+	if e, ok := err.(*exec.ExitError); ok || (ok && !e.Success()) {
+		t.Errorf("Expected process to exit cleanly, got %v", err)
+	}
+}
+
+func TestRecoverFatal(t *testing.T) {
+	log := New(&bytes.Buffer{})
+	defer func() {
+		if p := recover(); p != nil {
+			t.Errorf("Unexpected panic: %#v", p)
+		}
+	}()
+	defer Recover(func(err error) {
+		if err.Error() != "Test" {
+			t.Errorf("Expected %q, but got %q", "Test", err.Error())
+		}
+	})
+	log.Fatal("Test")
+	t.Errorf("Should not get here")
+}
+
+func TestRecoverNonFatal(t *testing.T) {
+	log := New(&bytes.Buffer{})
+	defer func() {
+		if p := recover(); p == nil {
+			t.Errorf("Panic not thrown")
+		} else if p != "Test" {
+			t.Errorf("Expected %q, but got %#v", "Test", p)
+		}
+	}()
+	defer Recover(func(err error) {
+		t.Errorf("Recover function should not be called")
+	})
+	log.Panic("Test")
+	t.Errorf("Should not get here")
+}