Add multiproduct_kati

am: c2af0bedc1

Change-Id: Ic89e4380edb4562bf3e51850ab012274c86e0b26

cmd/multiproduct_kati/Android.bp

@@ -0,0 +1,24 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+blueprint_go_binary {
+    name: "multiproduct_kati",
+    deps: [
+        "soong-ui-build",
+        "soong-ui-logger",
+    ],
+    srcs: [
+        "main.go",
+    ],
+}

cmd/multiproduct_kati/main.go

@@ -0,0 +1,194 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"android/soong/ui/build"
+	"android/soong/ui/logger"
+)
+
+// We default to number of cpus / 4, which seems to be the sweet spot for my
+// system. I suspect this is mostly due to memory or disk bandwidth though, and
+// may depend on the size ofthe source tree, so this probably isn't a great
+// default.
+func detectNumJobs() int {
+	if runtime.NumCPU() < 4 {
+		return 1
+	}
+	return runtime.NumCPU() / 4
+}
+
+var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs")
+
+var keep = flag.Bool("keep", false, "keep successful output files")
+
+var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
+
+var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
+var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
+
+type Product struct {
+	ctx    build.Context
+	config build.Config
+}
+
+func main() {
+	log := logger.New(os.Stderr)
+	defer log.Cleanup()
+
+	flag.Parse()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	build.SetupSignals(log, cancel, log.Cleanup)
+
+	buildCtx := &build.ContextImpl{
+		Context:        ctx,
+		Logger:         log,
+		StdioInterface: build.StdioImpl{},
+	}
+
+	failed := false
+
+	config := build.NewConfig(buildCtx)
+	if *outDir == "" {
+		var err error
+		*outDir, err = ioutil.TempDir(config.OutDir(), "multiproduct")
+		if err != nil {
+			log.Fatalf("Failed to create tempdir: %v", err)
+		}
+
+		if !*keep {
+			defer func() {
+				if !failed {
+					os.RemoveAll(*outDir)
+				}
+			}()
+		}
+	}
+	config.Environment().Set("OUT_DIR", *outDir)
+	log.Println("Output directory:", *outDir)
+
+	build.SetupOutDir(buildCtx, config)
+	log.SetOutput(filepath.Join(config.OutDir(), "build.log"))
+
+	vars, err := build.DumpMakeVars(buildCtx, config, nil, nil, []string{"all_named_products"})
+	if err != nil {
+		log.Fatal(err)
+	}
+	products := strings.Fields(vars["all_named_products"])
+	log.Verbose("Got product list:", products)
+
+	var wg sync.WaitGroup
+	errs := make(chan error, len(products))
+	productConfigs := make(chan Product, len(products))
+
+	// Run the product config for every product in parallel
+	for _, product := range products {
+		wg.Add(1)
+		go func(product string) {
+			defer wg.Done()
+			defer logger.Recover(func(err error) {
+				errs <- fmt.Errorf("Error building %s: %v", product, err)
+			})
+
+			productOutDir := filepath.Join(config.OutDir(), product)
+
+			if err := os.MkdirAll(productOutDir, 0777); err != nil {
+				log.Fatalf("Error creating out directory: %v", err)
+			}
+
+			f, err := os.Create(filepath.Join(productOutDir, "std.log"))
+			if err != nil {
+				log.Fatalf("Error creating std.log: %v", err)
+			}
+
+			productLog := logger.New(&bytes.Buffer{})
+			productLog.SetOutput(filepath.Join(productOutDir, "build.log"))
+
+			productCtx := &build.ContextImpl{
+				Context:        ctx,
+				Logger:         productLog,
+				StdioInterface: build.NewCustomStdio(nil, f, f),
+			}
+
+			productConfig := build.NewConfig(productCtx)
+			productConfig.Environment().Set("OUT_DIR", productOutDir)
+			productConfig.Lunch(productCtx, product, "eng")
+
+			build.Build(productCtx, productConfig, build.BuildProductConfig)
+			productConfigs <- Product{productCtx, productConfig}
+		}(product)
+	}
+	go func() {
+		defer close(productConfigs)
+		wg.Wait()
+	}()
+
+	var wg2 sync.WaitGroup
+	// Then run up to numJobs worth of Soong and Kati
+	for i := 0; i < *numJobs; i++ {
+		wg2.Add(1)
+		go func() {
+			defer wg2.Done()
+			for product := range productConfigs {
+				func() {
+					defer logger.Recover(func(err error) {
+						errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err)
+					})
+
+					buildWhat := 0
+					if !*onlyConfig {
+						buildWhat |= build.BuildSoong
+						if !*onlySoong {
+							buildWhat |= build.BuildKati
+						}
+					}
+					build.Build(product.ctx, product.config, buildWhat)
+					if !*keep {
+						// TODO: kati aborts from opendir while setting up the find emulator
+						//os.RemoveAll(product.config.OutDir())
+					}
+					log.Println("Finished running for", product.config.TargetProduct())
+				}()
+			}
+		}()
+	}
+	go func() {
+		wg2.Wait()
+		close(errs)
+	}()
+
+	for err := range errs {
+		failed = true
+		log.Print(err)
+	}
+
+	if failed {
+		log.Fatalln("Failed")
+	}
+}

cmd/soong_ui/main.go

@@ -48,14 +48,6 @@ func main() {
 		log.Fatalln("The `soong` native UI is not yet available.")
 	}
 
-	// Precondition: the current directory is the top of the source tree
-	if _, err := os.Stat("build/soong/root.bp"); err != nil {
-		if os.IsNotExist(err) {
-			log.Fatalln("soong_ui should run from the root of the source directory: build/soong/root.bp not found")
-		}
-		log.Fatalln("Error verifying tree state:", err)
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 

ui/build/config.go

@@ -15,6 +15,8 @@
 package build
 
 import (
+	"log"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -40,6 +42,8 @@ type configImpl struct {
 	katiSuffix string
 }
 
+const srcDirFileCheck = "build/soong/root.bp"
+
 func NewConfig(ctx Context, args ...string) Config {
 	ret := &configImpl{
 		environ: OsEnvironment(),
@@ -71,6 +75,14 @@ func NewConfig(ctx Context, args ...string) Config {
 	ret.parallel = runtime.NumCPU() + 2
 	ret.keepGoing = 1
 
+	// Precondition: the current directory is the top of the source tree
+	if _, err := os.Stat(srcDirFileCheck); err != nil {
+		if os.IsNotExist(err) {
+			log.Fatalf("Current working directory must be the source tree. %q not found", srcDirFileCheck)
+		}
+		log.Fatalln("Error verifying tree state:", err)
+	}
+
 	for _, arg := range args {
 		arg = strings.TrimSpace(arg)
 		if arg == "--make-mode" {