StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Make globs compatible with hash-based ninja semantics" into main

Browse Source
This commit is contained in:
Cole Faust
2024-09-10 23:20:10 +00:00
committed by Gerrit Code Review
parent e97ec10a98 2fec4128e0
commit 8ca2a80d27
5 changed files with 214 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
cmd/soong_build/main.go
View File

@@ -15,7 +15,6 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"flag"
@@ -29,10 +28,12 @@ import (
"android/soong/android/allowlists"
"android/soong/bp2build"
"android/soong/shared"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools"
"github.com/google/blueprint/metrics"
"github.com/google/blueprint/pathtools"
"github.com/google/blueprint/proptools"
androidProtobuf "google.golang.org/protobuf/android"
)
@@ -42,8 +43,6 @@ var (
availableEnvFile string
usedEnvFile string
globFile string
globListDir string
delveListen string
delvePath string
@@ -64,8 +63,6 @@ func init() {
flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
@@ -206,20 +203,6 @@ func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArg
ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
}
func writeBuildGlobsNinjaFile(ctx *android.Context) {
ctx.EventHandler.Begin("globs_ninja_file")
defer ctx.EventHandler.End("globs_ninja_file")
globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
GlobLister: ctx.Globs,
GlobFile: globFile,
GlobDir: globDir,
SrcDir: ctx.SrcDir(),
}, ctx.Config())
maybeQuit(err, "")
}
func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {
eventHandler.Begin("ninja_deps")
defer eventHandler.End("ninja_deps")
@@ -283,7 +266,9 @@ func writeConfigCache(configCache *ConfigCache, configCacheFile string) {
}
// runSoongOnlyBuild runs the standard Soong build in a number of different modes.
func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
// It returns the path to the output file (usually the ninja file) and the deps that need
// to trigger a soong rerun.
func runSoongOnlyBuild(ctx *android.Context) (string, []string) {
ctx.EventHandler.Begin("soong_build")
defer ctx.EventHandler.End("soong_build")
@@ -299,37 +284,30 @@ func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
maybeQuit(err, "")
ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
writeBuildGlobsNinjaFile(ctx)
// Convert the Soong module graph into Bazel BUILD files.
switch ctx.Config().BuildMode {
case android.GenerateQueryView:
queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
return queryviewMarkerFile
return queryviewMarkerFile, ninjaDeps
case android.GenerateModuleGraph:
writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.ModuleGraphFile
return cmdlineArgs.ModuleGraphFile, ninjaDeps
case android.GenerateDocFile:
// TODO: we could make writeDocs() return the list of documentation files
// written and add them to the .d file. Then soong_docs would be re-run
// whenever one is deleted.
err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
maybeQuit(err, "error building Soong documentation")
writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.DocFile
return cmdlineArgs.DocFile, ninjaDeps
default:
// The actual output (build.ninja) was written in the RunBlueprint() call
// above
writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
if needToWriteNinjaHint(ctx) {
writeNinjaHint(ctx)
}
return cmdlineArgs.OutFile
return cmdlineArgs.OutFile, ninjaDeps
}
}
@@ -359,6 +337,8 @@ func parseAvailableEnv() map[string]string {
func main() {
flag.Parse()
soongStartTime := time.Now()
shared.ReexecWithDelveMaybe(delveListen, delvePath)
android.InitSandbox(topDir)
@@ -369,13 +349,6 @@ func main() {
configuration.SetAllowMissingDependencies()
}
extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
if shared.IsDebugging() {
// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
// enabled even if it completed successfully.
extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
}
// Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
// change between every CI build, so tracking it would require re-running Soong for every build.
metricsDir := availableEnv["LOG_DIR"]
@@ -393,7 +366,16 @@ func main() {
ctx.SetIncrementalAnalysis(incremental)
ctx.Register()
finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)
finalOutputFile, ninjaDeps := runSoongOnlyBuild(ctx)
ninjaDeps = append(ninjaDeps, usedEnvFile)
if shared.IsDebugging() {
// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
// enabled even if it completed successfully.
ninjaDeps = append(ninjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
}
writeDepFile(finalOutputFile, ctx.EventHandler, ninjaDeps)
if ctx.GetIncrementalEnabled() {
data, err := shared.EnvFileContents(configuration.EnvDeps())
@@ -407,6 +389,9 @@ func main() {
writeUsedEnvironmentFile(configuration)
err = writeGlobFile(ctx.EventHandler, finalOutputFile, ctx.Globs(), soongStartTime)
maybeQuit(err, "")
// Touch the output file so that it's the newest file created by soong_build.
// This is necessary because, if soong_build generated any files which
// are ninja inputs to the main output file, then ninja would superfluously
@@ -423,18 +408,33 @@ func writeUsedEnvironmentFile(configuration android.Config) {
data, err := shared.EnvFileContents(configuration.EnvDeps())
maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
if preexistingData, err := os.ReadFile(path); err != nil {
if !os.IsNotExist(err) {
maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
}
} else if bytes.Equal(preexistingData, data) {
// used environment file is unchanged
return
}
err = os.WriteFile(path, data, 0666)
err = pathtools.WriteFileIfChanged(path, data, 0666)
maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
}
func writeGlobFile(eventHandler *metrics.EventHandler, finalOutFile string, globs pathtools.MultipleGlobResults, soongStartTime time.Time) error {
eventHandler.Begin("writeGlobFile")
defer eventHandler.End("writeGlobFile")
globsFile, err := os.Create(shared.JoinPath(topDir, finalOutFile+".globs"))
if err != nil {
return err
}
defer globsFile.Close()
globsFileEncoder := json.NewEncoder(globsFile)
for _, glob := range globs {
if err := globsFileEncoder.Encode(glob); err != nil {
return err
}
}
return os.WriteFile(
shared.JoinPath(topDir, finalOutFile+".globs_time"),
[]byte(fmt.Sprintf("%d\n", soongStartTime.UnixMicro())),
0666,
)
}
func touch(path string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
maybeQuit(err, "Error touching '%s'", path)

17
tests/bootstrap_test.sh
View File

@@ -145,36 +145,19 @@ EOF
run_soong
local -r ninja_mtime1=$(stat -c "%y" out/soong/build."${target_product}".ninja)
local glob_deps_file=out/soong/globs/"${target_product}"/0.d
run_soong
local -r ninja_mtime2=$(stat -c "%y" out/soong/build."${target_product}".ninja)
# There is an ineffiencency in glob that requires bpglob to rerun once for each glob to update
# the entry in the .ninja_log. It doesn't update the output file, but we can detect the rerun
# by checking if the deps file was created.
if [ ! -e "$glob_deps_file" ]; then
fail "Glob deps file missing after second build"
fi
local -r glob_deps_mtime2=$(stat -c "%y" "$glob_deps_file")
if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then
fail "Ninja file rewritten on null incremental build"
fi
run_soong
local -r ninja_mtime3=$(stat -c "%y" out/soong/build."${target_product}".ninja)
local -r glob_deps_mtime3=$(stat -c "%y" "$glob_deps_file")
if [[ "$ninja_mtime2" != "$ninja_mtime3" ]]; then
fail "Ninja file rewritten on null incremental build"
fi
# The bpglob commands should not rerun after the first incremental build.
if [[ "$glob_deps_mtime2" != "$glob_deps_mtime3" ]]; then
fail "Glob deps file rewritten on second null incremental build"
fi
}
function test_add_file_to_glob() {

4
ui/build/config.go
View File

@@ -1037,10 +1037,6 @@ func (c *configImpl) HostToolDir() string {
}
}
func (c *configImpl) NamedGlobFile(name string) string {
return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
}
func (c *configImpl) UsedEnvFile(tag string) string {
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag)

212
ui/build/soong.go
View File

@@ -15,10 +15,14 @@
package build
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
"sync"
@@ -52,7 +56,7 @@ const (
// bootstrapEpoch is used to determine if an incremental build is incompatible with the current
// version of bootstrap and needs cleaning before continuing the build. Increment this for
// incompatible changes, for example when moving the location of the bpglob binary that is
// incompatible changes, for example when moving the location of a microfactory binary that is
// executed during bootstrap before the primary builder has had a chance to update the path.
bootstrapEpoch = 1
)
@@ -226,10 +230,6 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstra
var allArgs []string
allArgs = append(allArgs, pb.specificArgs...)
globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
allArgs = append(allArgs,
"--globListDir", globPathName,
"--globFile", pb.config.NamedGlobFile(globPathName))
allArgs = append(allArgs, commonArgs...)
allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
@@ -241,10 +241,8 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstra
}
allArgs = append(allArgs, "Android.bp")
globfiles := bootstrap.GlobFileListFiles(bootstrap.GlobDirectory(config.SoongOutDir(), globPathName))
return bootstrap.PrimaryBuilderInvocation{
Implicits: globfiles,
Implicits: []string{pb.output + ".glob_results"},
Outputs: []string{pb.output},
Args: allArgs,
Description: pb.description,
@@ -276,24 +274,15 @@ func bootstrapEpochCleanup(ctx Context, config Config) {
os.Remove(file)
}
}
for _, globFile := range bootstrapGlobFileList(config) {
os.Remove(globFile)
}
os.Remove(soongNinjaFile + ".globs")
os.Remove(soongNinjaFile + ".globs_time")
os.Remove(soongNinjaFile + ".glob_results")
// Mark the tree as up to date with the current epoch by writing the epoch marker file.
writeEmptyFile(ctx, epochPath)
}
}
func bootstrapGlobFileList(config Config) []string {
return []string{
config.NamedGlobFile(getGlobPathName(config)),
config.NamedGlobFile(jsonModuleGraphTag),
config.NamedGlobFile(queryviewTag),
config.NamedGlobFile(soongDocsTag),
}
}
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
@@ -411,32 +400,9 @@ func bootstrapBlueprint(ctx Context, config Config) {
runGoTests: !config.skipSoongTests,
// If we want to debug soong_build, we need to compile it for debugging
debugCompilation: delvePort != "",
subninjas: bootstrapGlobFileList(config),
primaryBuilderInvocations: invocations,
}
// The glob ninja files are generated during the main build phase. However, the
// primary buildifer invocation depends on all of its glob files, even before
// it's been run. Generate a "empty" glob ninja file on the first run,
// so that the files can be there to satisfy the dependency.
for _, pb := range pbfs {
globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
globNinjaFile := config.NamedGlobFile(globPathName)
if _, err := os.Stat(globNinjaFile); os.IsNotExist(err) {
err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
GlobLister: func() pathtools.MultipleGlobResults { return nil },
GlobFile: globNinjaFile,
GlobDir: bootstrap.GlobDirectory(config.SoongOutDir(), globPathName),
SrcDir: ".",
}, blueprintConfig)
if err != nil {
ctx.Fatal(err)
}
} else if err != nil {
ctx.Fatal(err)
}
}
// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
// reason to write a `bootstrap.ninja.d` file
_, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
@@ -614,9 +580,6 @@ func runSoong(ctx Context, config Config) {
}
}()
runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
map[string]string{"github.com/google/blueprint": "build/blueprint"})
ninja := func(targets ...string) {
ctx.BeginTrace(metrics.RunSoong, "bootstrap")
defer ctx.EndTrace()
@@ -698,6 +661,12 @@ func runSoong(ctx Context, config Config) {
targets = append(targets, config.SoongNinjaFile())
}
for _, target := range targets {
if err := checkGlobs(ctx, target); err != nil {
ctx.Fatalf("Error checking globs: %s", err.Error())
}
}
beforeSoongTimestamp := time.Now()
ninja(targets...)
@@ -724,6 +693,157 @@ func runSoong(ctx Context, config Config) {
}
}
// checkGlobs manages the globs that cause soong to rerun.
//
// When soong_build runs, it will run globs. It will write all the globs
// it ran into the "{finalOutFile}.globs" file. Then every build,
// soong_ui will check that file, rerun the globs, and if they changed
// from the results that soong_build got, update the ".glob_results"
// file, causing soong_build to rerun. The ".glob_results" file will
// be empty on the first run of soong_build, because we don't know
// what the globs are yet, but also remain empty until the globs change
// so that we don't run soong_build a second time unnecessarily.
// Both soong_build and soong_ui will also update a ".globs_time" file
// with the time that they ran at every build. When soong_ui checks
// globs, it only reruns globs whose dependencies are newer than the
// time in the ".globs_time" file.
func checkGlobs(ctx Context, finalOutFile string) error {
ctx.BeginTrace(metrics.RunSoong, "check_globs")
defer ctx.EndTrace()
st := ctx.Status.StartTool()
st.Status("Running globs...")
defer st.Finish()
globsFile, err := os.Open(finalOutFile + ".globs")
if errors.Is(err, fs.ErrNotExist) {
// if the glob file doesn't exist, make sure the glob_results file exists and is empty.
if err := os.MkdirAll(filepath.Dir(finalOutFile), 0777); err != nil {
return err
}
f, err := os.Create(finalOutFile + ".glob_results")
if err != nil {
return err
}
return f.Close()
} else if err != nil {
return err
}
defer globsFile.Close()
globsFileDecoder := json.NewDecoder(globsFile)
globsTimeBytes, err := os.ReadFile(finalOutFile + ".globs_time")
if err != nil {
return err
}
globsTimeMicros, err := strconv.ParseInt(strings.TrimSpace(string(globsTimeBytes)), 10, 64)
if err != nil {
return err
}
globCheckStartTime := time.Now().UnixMicro()
globsChan := make(chan pathtools.GlobResult)
errorsChan := make(chan error)
wg := sync.WaitGroup{}
hasChangedGlobs := false
for i := 0; i < runtime.NumCPU()*2; i++ {
wg.Add(1)
go func() {
for cachedGlob := range globsChan {
// If we've already determined we have changed globs, just finish consuming
// the channel without doing any more checks.
if hasChangedGlobs {
continue
}
// First, check if any of the deps are newer than the last time globs were checked.
// If not, we don't need to rerun the glob.
hasNewDep := false
for _, dep := range cachedGlob.Deps {
info, err := os.Stat(dep)
if err != nil {
errorsChan <- err
continue
}
if info.ModTime().UnixMicro() > globsTimeMicros {
hasNewDep = true
break
}
}
if !hasNewDep {
continue
}
// Then rerun the glob and check if we got the same result as before.
result, err := pathtools.Glob(cachedGlob.Pattern, cachedGlob.Excludes, pathtools.FollowSymlinks)
if err != nil {
errorsChan <- err
} else {
if !slices.Equal(result.Matches, cachedGlob.Matches) {
hasChangedGlobs = true
}
}
}
wg.Done()
}()
}
go func() {
wg.Wait()
close(errorsChan)
}()
errorsWg := sync.WaitGroup{}
errorsWg.Add(1)
var errFromGoRoutines error
go func() {
for result := range errorsChan {
if errFromGoRoutines == nil {
errFromGoRoutines = result
}
}
errorsWg.Done()
}()
var cachedGlob pathtools.GlobResult
for globsFileDecoder.More() {
if err := globsFileDecoder.Decode(&cachedGlob); err != nil {
return err
}
// Need to clone the GlobResult because the json decoder will
// reuse the same slice allocations.
globsChan <- cachedGlob.Clone()
}
close(globsChan)
errorsWg.Wait()
if errFromGoRoutines != nil {
return errFromGoRoutines
}
// Update the globs_time file whether or not we found changed globs,
// so that we don't rerun globs in the future that we just saw didn't change.
err = os.WriteFile(
finalOutFile+".globs_time",
[]byte(fmt.Sprintf("%d\n", globCheckStartTime)),
0666,
)
if err != nil {
return err
}
if hasChangedGlobs {
fmt.Fprintf(os.Stdout, "Globs changed, rerunning soong...\n")
// Write the current time to the glob_results file. We just need
// some unique value to trigger a rerun, it doesn't matter what it is.
err = os.WriteFile(
finalOutFile+".glob_results",
[]byte(fmt.Sprintf("%d\n", globCheckStartTime)),
0666,
)
if err != nil {
return err
}
}
return nil
}
// loadSoongBuildMetrics reads out/soong_build_metrics.pb if it was generated by soong_build and copies the
// events stored in it into the soong_ui trace to provide introspection into how long the different phases of
// soong_build are taking.

4
ui/build/test_build.go
View File

@@ -79,9 +79,6 @@ func testForDanglingRules(ctx Context, config Config) {
// out/build_date.txt is considered a "source file"
buildDatetimeFilePath := filepath.Join(outDir, "build_date.txt")
// bpglob is built explicitly using Microfactory
bpglob := filepath.Join(config.SoongOutDir(), "bpglob")
// release-config files are generated from the initial lunch or Kati phase
// before running soong and ninja.
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
@@ -105,7 +102,6 @@ func testForDanglingRules(ctx Context, config Config) {
line == extraVariablesFilePath ||
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||
line == bpglob ||
strings.HasPrefix(line, releaseConfigDir) ||
buildFingerPrintFilePattern.MatchString(line) {
// Leaf node is in one of Soong's bootstrap directories, which do not have