StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes I07b0ca98,I1c2901e1

Browse Source 
* changes:
  Cut the multiproduct_kati -> soong-ui-build dep.
  Do not create a build.Config in multiproduct_kati.
This commit is contained in:
Lukács T. Berki
2021-08-11 13:17:47 +00:00
committed by Gerrit Code Review
parent 726d44a86d f656b8434b
commit 4ddcd8152b
9 changed files with 125 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/multiproduct_kati/Android.bp
View File

@@ -19,8 +19,8 @@ package {
blueprint_go_binary {
name: "multiproduct_kati",
deps: [
"soong-ui-build",
"soong-ui-logger",
"soong-ui-signal",
"soong-ui-terminal",
"soong-ui-tracer",
"soong-zip",

201
cmd/multiproduct_kati/main.go
View File

@@ -20,20 +20,19 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"syscall"
"time"
"android/soong/finder"
"android/soong/ui/build"
"android/soong/ui/logger"
"android/soong/ui/signal"
"android/soong/ui/status"
"android/soong/ui/terminal"
"android/soong/ui/tracer"
@@ -77,36 +76,6 @@ func (m *multipleStringArg) Set(s string) error {
return nil
}
const errorLeadingLines = 20
const errorTrailingLines = 20
func errMsgFromLog(filename string) string {
if filename == "" {
return ""
}
data, err := ioutil.ReadFile(filename)
if err != nil {
return ""
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) > errorLeadingLines+errorTrailingLines+1 {
lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...",
len(lines)-errorLeadingLines-errorTrailingLines)
lines = append(lines[:errorLeadingLines+1],
lines[len(lines)-errorTrailingLines:]...)
}
var buf strings.Builder
for _, line := range lines {
buf.WriteString("> ")
buf.WriteString(line)
buf.WriteString("\n")
}
return buf.String()
}
// TODO(b/70370883): This tool uses a lot of open files -- over the default
// soft limit of 1024 on some systems. So bump up to the hard limit until I fix
// the algorithm.
@@ -158,28 +127,59 @@ func copyFile(from, to string) error {
}
type mpContext struct {
Context context.Context
Logger logger.Logger
Status status.ToolStatus
Tracer tracer.Tracer
Finder *finder.Finder
Config build.Config
Logger logger.Logger
Status status.ToolStatus
LogsDir string
SoongUi string
MainOutDir string
MainLogsDir string
}
func detectTotalRAM() uint64 {
var info syscall.Sysinfo_t
err := syscall.Sysinfo(&info)
if err != nil {
panic(err)
}
return info.Totalram * uint64(info.Unit)
}
func findNamedProducts(soongUi string, log logger.Logger) []string {
cmd := exec.Command(soongUi, "--dumpvars-mode", "--vars=all_named_products")
output, err := cmd.Output()
if err != nil {
log.Fatalf("Cannot determine named products: %v", err)
}
rx := regexp.MustCompile(`^all_named_products='(.*)'$`)
match := rx.FindStringSubmatch(strings.TrimSpace(string(output)))
return strings.Fields(match[1])
}
// ensureEmptyFileExists ensures that the containing directory exists, and the
// specified file exists. If it doesn't exist, it will write an empty file.
func ensureEmptyFileExists(file string, log logger.Logger) {
if _, err := os.Stat(file); os.IsNotExist(err) {
f, err := os.Create(file)
if err != nil {
log.Fatalf("Error creating %s: %q\n", file, err)
}
f.Close()
} else if err != nil {
log.Fatalf("Error checking %s: %q\n", file, err)
}
}
func main() {
stdio := terminal.StdioImpl{}
output := terminal.NewStatusOutput(stdio.Stdout(), "", false,
build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))
output := terminal.NewStatusOutput(stdio.Stdout(), "", false, false)
log := logger.New(output)
defer log.Cleanup()
flag.Parse()
ctx, cancel := context.WithCancel(context.Background())
_, cancel := context.WithCancel(context.Background())
defer cancel()
trace := tracer.New(log)
@@ -192,69 +192,59 @@ func main() {
var failures failureCount
stat.AddOutput(&failures)
build.SetupSignals(log, cancel, func() {
signal.SetupSignals(log, cancel, func() {
trace.Close()
log.Cleanup()
stat.Finish()
})
buildCtx := build.Context{ContextImpl: &build.ContextImpl{
Context: ctx,
Logger: log,
Tracer: trace,
Writer: output,
Status: stat,
}}
args := ""
if *alternateResultDir {
args = "dist"
}
originalOutDir := os.Getenv("OUT_DIR")
if originalOutDir == "" {
originalOutDir = "out"
}
soongUi := "build/soong/soong_ui.bash"
config := build.NewConfig(buildCtx, args)
if *outDir == "" {
var outputDir string
if *outDir != "" {
outputDir = *outDir
} else {
name := "multiproduct"
if !*incremental {
name += "-" + time.Now().Format("20060102150405")
}
*outDir = filepath.Join(originalOutDir, name)
// Ensure the empty files exist in the output directory
// containing our output directory too. This is mostly for
// safety, but also triggers the ninja_build file so that our
// build servers know that they can parse the output as if it
// was ninja output.
build.SetupOutDir(buildCtx, config)
if err := os.MkdirAll(*outDir, 0777); err != nil {
log.Fatalf("Failed to create tempdir: %v", err)
outDirBase := os.Getenv("OUT_DIR")
if outDirBase == "" {
outDirBase = "out"
}
}
config.Environment().Set("OUT_DIR", *outDir)
log.Println("Output directory:", *outDir)
logsDir := filepath.Join(config.OutDir(), "logs")
outputDir = filepath.Join(outDirBase, name)
}
log.Println("Output directory:", outputDir)
// The ninja_build file is used by our buildbots to understand that the output
// can be parsed as ninja output.
if err := os.MkdirAll(outputDir, 0777); err != nil {
log.Fatalf("Failed to create output directory: %v", err)
}
ensureEmptyFileExists(filepath.Join(outputDir, "ninja_build"), log)
logsDir := filepath.Join(outputDir, "logs")
os.MkdirAll(logsDir, 0777)
build.SetupOutDir(buildCtx, config)
var configLogsDir string
if *alternateResultDir {
configLogsDir = filepath.Join(outputDir, "dist/logs")
} else {
configLogsDir = outputDir
}
os.MkdirAll(config.LogsDir(), 0777)
log.SetOutput(filepath.Join(config.LogsDir(), "soong.log"))
trace.SetOutput(filepath.Join(config.LogsDir(), "build.trace"))
os.MkdirAll(configLogsDir, 0777)
log.SetOutput(filepath.Join(configLogsDir, "soong.log"))
trace.SetOutput(filepath.Join(configLogsDir, "build.trace"))
var jobs = *numJobs
if jobs < 1 {
jobs = runtime.NumCPU() / 4
ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024)
ramGb := int(detectTotalRAM() / (1024 * 1024 * 1024))
if ramJobs := ramGb / 25; ramGb > 0 && jobs > ramJobs {
jobs = ramJobs
}
@@ -267,17 +257,8 @@ func main() {
setMaxFiles(log)
finder := build.NewSourceFinder(buildCtx, config)
defer finder.Shutdown()
build.FindSources(buildCtx, config, finder)
vars, err := build.DumpMakeVars(buildCtx, config, nil, []string{"all_named_products"})
if err != nil {
log.Fatal(err)
}
allProducts := findNamedProducts(soongUi, log)
var productsList []string
allProducts := strings.Fields(vars["all_named_products"])
if len(includeProducts) > 0 {
var missingProducts []string
@@ -325,19 +306,15 @@ func main() {
log.Verbose("Got product list: ", finalProductsList)
s := buildCtx.Status.StartTool()
s := stat.StartTool()
s.SetTotalActions(len(finalProductsList))
mpCtx := &mpContext{
Context: ctx,
Logger: log,
Status: s,
Tracer: trace,
Finder: finder,
Config: config,
LogsDir: logsDir,
Logger: log,
Status: s,
SoongUi: soongUi,
MainOutDir: outputDir,
MainLogsDir: logsDir,
}
products := make(chan string, len(productsList))
@@ -359,7 +336,7 @@ func main() {
if product == "" {
return
}
runSoongUiForProduct(mpCtx, product, soongUi)
runSoongUiForProduct(mpCtx, product)
}
}
}()
@@ -371,7 +348,7 @@ func main() {
FileArgs: []zip.FileArg{
{GlobDir: logsDir, SourcePrefixToStrip: logsDir},
},
OutputFilePath: filepath.Join(config.RealDistDir(), "logs.zip"),
OutputFilePath: filepath.Join(outputDir, "dist/logs.zip"),
NumParallelJobs: runtime.NumCPU(),
CompressionLevel: 5,
}
@@ -413,10 +390,10 @@ func cleanupAfterProduct(outDir, productZip string) {
}
}
func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) {
outDir := filepath.Join(mpctx.Config.OutDir(), product)
logsDir := filepath.Join(mpctx.LogsDir, product)
productZip := filepath.Join(mpctx.Config.OutDir(), product+".zip")
func runSoongUiForProduct(mpctx *mpContext, product string) {
outDir := filepath.Join(mpctx.MainOutDir, product)
logsDir := filepath.Join(mpctx.MainLogsDir, product)
productZip := filepath.Join(mpctx.MainOutDir, product+".zip")
consoleLogPath := filepath.Join(logsDir, "std.log")
if err := os.MkdirAll(outDir, 0777); err != nil {
@@ -451,7 +428,7 @@ func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) {
args = append(args, "dist")
}
cmd := exec.Command(soongUi, args...)
cmd := exec.Command(mpctx.SoongUi, args...)
cmd.Stdout = consoleLogWriter
cmd.Stderr = consoleLogWriter
cmd.Env = append(os.Environ(),

1
cmd/soong_ui/Android.bp
View File

@@ -20,6 +20,7 @@ blueprint_go_binary {
name: "soong_ui",
deps: [
"soong-ui-build",
"soong-ui-signal",
"soong-ui-logger",
"soong-ui-terminal",
"soong-ui-tracer",

3
cmd/soong_ui/main.go
View File

@@ -30,6 +30,7 @@ import (
"android/soong/ui/build"
"android/soong/ui/logger"
"android/soong/ui/metrics"
"android/soong/ui/signal"
"android/soong/ui/status"
"android/soong/ui/terminal"
"android/soong/ui/tracer"
@@ -190,7 +191,7 @@ func main() {
stat.AddOutput(trace.StatusTracer())
// Set up a cleanup procedure in case the normal termination process doesn't work.
build.SetupSignals(log, cancel, func() {
signal.SetupSignals(log, cancel, func() {
trace.Close()
log.Cleanup()
stat.Finish()

1
cuj/Android.bp
View File

@@ -7,6 +7,7 @@ blueprint_go_binary {
deps: [
"soong-ui-build",
"soong-ui-logger",
"soong-ui-signal",
"soong-ui-terminal",
"soong-ui-tracer",
],

3
cuj/cuj.go
View File

@@ -27,6 +27,7 @@ import (
"android/soong/ui/build"
"android/soong/ui/logger"
"android/soong/ui/metrics"
"android/soong/ui/signal"
"android/soong/ui/status"
"android/soong/ui/terminal"
"android/soong/ui/tracer"
@@ -65,7 +66,7 @@ func (t *Test) Run(logsDir string) {
stat.AddOutput(output)
stat.AddOutput(trace.StatusTracer())
build.SetupSignals(log, cancel, func() {
signal.SetupSignals(log, cancel, func() {
trace.Close()
log.Cleanup()
stat.Finish()

1
ui/build/Android.bp
View File

@@ -61,7 +61,6 @@ bootstrap_go_package {
"proc_sync.go",
"rbe.go",
"sandbox_config.go",
"signal.go",
"soong.go",
"test_build.go",
"upload.go",

28
ui/signal/Android.bp Normal file
View File

@@ -0,0 +1,28 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
bootstrap_go_package {
name: "soong-ui-signal",
pkgPath: "android/soong/ui/signal",
srcs: [
"signal.go",
],
deps: [
"soong-ui-logger",
],
}

2
ui/build/signal.go → ui/signal/signal.go
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package build
package signal
import (
"os"