Add multiproduct_kati

This is a replacement for build/tools/kati_all_products.sh using the new
Soong ui/build package. It doesn't even attempt to run ninja, and it can
be configured to run only the product config, or only the product config
and Soong.

For AOSP on my machine:
 -only-config  1.4s
 -only-soong   1m20s
 <none>        13m

Test: multiproduct_kati
Change-Id: Ie3e6e7bdf692e46a8b8eb828f437190f8003500b

cmd/multiproduct_kati/Android.bp

@@ -0,0 +1,24 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+blueprint_go_binary {
+    name: "multiproduct_kati",
+    deps: [
+        "soong-ui-build",
+        "soong-ui-logger",
+    ],
+    srcs: [
+        "main.go",
+    ],
+}

cmd/multiproduct_kati/main.go

@@ -0,0 +1,194 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"android/soong/ui/build"
+	"android/soong/ui/logger"
+)
+
+// We default to number of cpus / 4, which seems to be the sweet spot for my
+// system. I suspect this is mostly due to memory or disk bandwidth though, and
+// may depend on the size ofthe source tree, so this probably isn't a great
+// default.
+func detectNumJobs() int {
+	if runtime.NumCPU() < 4 {
+		return 1
+	}
+	return runtime.NumCPU() / 4
+}
+
+var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs")
+
+var keep = flag.Bool("keep", false, "keep successful output files")
+
+var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
+
+var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
+var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
+
+type Product struct {
+	ctx    build.Context
+	config build.Config
+}
+
+func main() {
+	log := logger.New(os.Stderr)
+	defer log.Cleanup()
+
+	flag.Parse()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	build.SetupSignals(log, cancel, log.Cleanup)
+
+	buildCtx := &build.ContextImpl{
+		Context:        ctx,
+		Logger:         log,
+		StdioInterface: build.StdioImpl{},
+	}
+
+	failed := false
+
+	config := build.NewConfig(buildCtx)
+	if *outDir == "" {
+		var err error
+		*outDir, err = ioutil.TempDir(config.OutDir(), "multiproduct")
+		if err != nil {
+			log.Fatalf("Failed to create tempdir: %v", err)
+		}
+
+		if !*keep {
+			defer func() {
+				if !failed {
+					os.RemoveAll(*outDir)
+				}
+			}()
+		}
+	}
+	config.Environment().Set("OUT_DIR", *outDir)
+	log.Println("Output directory:", *outDir)
+
+	build.SetupOutDir(buildCtx, config)
+	log.SetOutput(filepath.Join(config.OutDir(), "build.log"))
+
+	vars, err := build.DumpMakeVars(buildCtx, config, nil, nil, []string{"all_named_products"})
+	if err != nil {
+		log.Fatal(err)
+	}
+	products := strings.Fields(vars["all_named_products"])
+	log.Verbose("Got product list:", products)
+
+	var wg sync.WaitGroup
+	errs := make(chan error, len(products))
+	productConfigs := make(chan Product, len(products))
+
+	// Run the product config for every product in parallel
+	for _, product := range products {
+		wg.Add(1)
+		go func(product string) {
+			defer wg.Done()
+			defer logger.Recover(func(err error) {
+				errs <- fmt.Errorf("Error building %s: %v", product, err)
+			})
+
+			productOutDir := filepath.Join(config.OutDir(), product)
+
+			if err := os.MkdirAll(productOutDir, 0777); err != nil {
+				log.Fatalf("Error creating out directory: %v", err)
+			}
+
+			f, err := os.Create(filepath.Join(productOutDir, "std.log"))
+			if err != nil {
+				log.Fatalf("Error creating std.log: %v", err)
+			}
+
+			productLog := logger.New(&bytes.Buffer{})
+			productLog.SetOutput(filepath.Join(productOutDir, "build.log"))
+
+			productCtx := &build.ContextImpl{
+				Context:        ctx,
+				Logger:         productLog,
+				StdioInterface: build.NewCustomStdio(nil, f, f),
+			}
+
+			productConfig := build.NewConfig(productCtx)
+			productConfig.Environment().Set("OUT_DIR", productOutDir)
+			productConfig.Lunch(productCtx, product, "eng")
+
+			build.Build(productCtx, productConfig, build.BuildProductConfig)
+			productConfigs <- Product{productCtx, productConfig}
+		}(product)
+	}
+	go func() {
+		defer close(productConfigs)
+		wg.Wait()
+	}()
+
+	var wg2 sync.WaitGroup
+	// Then run up to numJobs worth of Soong and Kati
+	for i := 0; i < *numJobs; i++ {
+		wg2.Add(1)
+		go func() {
+			defer wg2.Done()
+			for product := range productConfigs {
+				func() {
+					defer logger.Recover(func(err error) {
+						errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err)
+					})
+
+					buildWhat := 0
+					if !*onlyConfig {
+						buildWhat |= build.BuildSoong
+						if !*onlySoong {
+							buildWhat |= build.BuildKati
+						}
+					}
+					build.Build(product.ctx, product.config, buildWhat)
+					if !*keep {
+						// TODO: kati aborts from opendir while setting up the find emulator
+						//os.RemoveAll(product.config.OutDir())
+					}
+					log.Println("Finished running for", product.config.TargetProduct())
+				}()
+			}
+		}()
+	}
+	go func() {
+		wg2.Wait()
+		close(errs)
+	}()
+
+	for err := range errs {
+		failed = true
+		log.Print(err)
+	}
+
+	if failed {
+		log.Fatalln("Failed")
+	}
+}

cmd/soong_ui/main.go

@@ -48,14 +48,6 @@ func main() {
 		log.Fatalln("The `soong` native UI is not yet available.")
 	}
 
-	// Precondition: the current directory is the top of the source tree
-	if _, err := os.Stat("build/soong/root.bp"); err != nil {
-		if os.IsNotExist(err) {
-			log.Fatalln("soong_ui should run from the root of the source directory: build/soong/root.bp not found")
-		}
-		log.Fatalln("Error verifying tree state:", err)
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 

ui/build/config.go

@@ -15,6 +15,8 @@
 package build
 
 import (
+	"log"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -40,6 +42,8 @@ type configImpl struct {
 	katiSuffix string
 }
 
+const srcDirFileCheck = "build/soong/root.bp"
+
 func NewConfig(ctx Context, args ...string) Config {
 	ret := &configImpl{
 		environ: OsEnvironment(),
@@ -71,6 +75,14 @@ func NewConfig(ctx Context, args ...string) Config {
 	ret.parallel = runtime.NumCPU() + 2
 	ret.keepGoing = 1
 
+	// Precondition: the current directory is the top of the source tree
+	if _, err := os.Stat(srcDirFileCheck); err != nil {
+		if os.IsNotExist(err) {
+			log.Fatalf("Current working directory must be the source tree. %q not found", srcDirFileCheck)
+		}
+		log.Fatalln("Error verifying tree state:", err)
+	}
+
 	for _, arg := range args {
 		arg = strings.TrimSpace(arg)
 		if arg == "--make-mode" {