StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make globs compatible with hash-based ninja semantics

Browse Source 
Previously, globs worked by having soong_build rewrite a ninja file
that ran the globs, and then dependended on the results of that ninja
file. soong_build also pre-filled their outputs so that it wouldn't
be immediately rerun on the 2nd build.

However, the pre-filling of outputs worked for ninja, because although
it updated their timestamps, the soong ninja file was then touched
by soong_build after that, so the soong_build ninja file was newer
and ninja wouldn't rerun soong. But N2 reruns actions if their inputs'
mtimes change in any way, not just if they're newer. Similarly,
hashed-based ninja implementations could not enforce an order on
file contents, so they would have the same problem.

To fix this, lift the glob checking out of ninja and into soong_ui.
Soong_build will output a globs report file every time it's run, and
every time soong_ui is run it will check the globs file, and if any
globs change, update an input to soong_build. soong_ui is essentially
doing what was done in ninja with bpglob actions before.

Bug: 364749114
Test: m nothing, m nothing again doesn't reanalyze, create a new file under a glob directory, m nothing again reanalyzes
Change-Id: I0dbc5ec58c89b869b59cd0602b82215c4972d799
This commit is contained in:
Cole Faust
2024-09-07 17:28:11 -07:00
parent 219009f606
commit 2fec4128e0
5 changed files with 214 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ui/build/config.go
View File

@@ -1037,10 +1037,6 @@ func (c *configImpl) HostToolDir() string {
}
}
func (c *configImpl) NamedGlobFile(name string) string {
return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
}
func (c *configImpl) UsedEnvFile(tag string) string {
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag)

212
ui/build/soong.go
View File

@@ -15,10 +15,14 @@
package build
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
"sync"
@@ -52,7 +56,7 @@ const (
// bootstrapEpoch is used to determine if an incremental build is incompatible with the current
// version of bootstrap and needs cleaning before continuing the build. Increment this for
// incompatible changes, for example when moving the location of the bpglob binary that is
// incompatible changes, for example when moving the location of a microfactory binary that is
// executed during bootstrap before the primary builder has had a chance to update the path.
bootstrapEpoch = 1
)
@@ -226,10 +230,6 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstra
var allArgs []string
allArgs = append(allArgs, pb.specificArgs...)
globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
allArgs = append(allArgs,
"--globListDir", globPathName,
"--globFile", pb.config.NamedGlobFile(globPathName))
allArgs = append(allArgs, commonArgs...)
allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
@@ -241,10 +241,8 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstra
}
allArgs = append(allArgs, "Android.bp")
globfiles := bootstrap.GlobFileListFiles(bootstrap.GlobDirectory(config.SoongOutDir(), globPathName))
return bootstrap.PrimaryBuilderInvocation{
Implicits: globfiles,
Implicits: []string{pb.output + ".glob_results"},
Outputs: []string{pb.output},
Args: allArgs,
Description: pb.description,
@@ -276,24 +274,15 @@ func bootstrapEpochCleanup(ctx Context, config Config) {
os.Remove(file)
}
}
for _, globFile := range bootstrapGlobFileList(config) {
os.Remove(globFile)
}
os.Remove(soongNinjaFile + ".globs")
os.Remove(soongNinjaFile + ".globs_time")
os.Remove(soongNinjaFile + ".glob_results")
// Mark the tree as up to date with the current epoch by writing the epoch marker file.
writeEmptyFile(ctx, epochPath)
}
}
func bootstrapGlobFileList(config Config) []string {
return []string{
config.NamedGlobFile(getGlobPathName(config)),
config.NamedGlobFile(jsonModuleGraphTag),
config.NamedGlobFile(queryviewTag),
config.NamedGlobFile(soongDocsTag),
}
}
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
@@ -411,32 +400,9 @@ func bootstrapBlueprint(ctx Context, config Config) {
runGoTests: !config.skipSoongTests,
// If we want to debug soong_build, we need to compile it for debugging
debugCompilation: delvePort != "",
subninjas: bootstrapGlobFileList(config),
primaryBuilderInvocations: invocations,
}
// The glob ninja files are generated during the main build phase. However, the
// primary buildifer invocation depends on all of its glob files, even before
// it's been run. Generate a "empty" glob ninja file on the first run,
// so that the files can be there to satisfy the dependency.
for _, pb := range pbfs {
globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
globNinjaFile := config.NamedGlobFile(globPathName)
if _, err := os.Stat(globNinjaFile); os.IsNotExist(err) {
err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
GlobLister: func() pathtools.MultipleGlobResults { return nil },
GlobFile: globNinjaFile,
GlobDir: bootstrap.GlobDirectory(config.SoongOutDir(), globPathName),
SrcDir: ".",
}, blueprintConfig)
if err != nil {
ctx.Fatal(err)
}
} else if err != nil {
ctx.Fatal(err)
}
}
// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
// reason to write a `bootstrap.ninja.d` file
_, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
@@ -614,9 +580,6 @@ func runSoong(ctx Context, config Config) {
}
}()
runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
map[string]string{"github.com/google/blueprint": "build/blueprint"})
ninja := func(targets ...string) {
ctx.BeginTrace(metrics.RunSoong, "bootstrap")
defer ctx.EndTrace()
@@ -698,6 +661,12 @@ func runSoong(ctx Context, config Config) {
targets = append(targets, config.SoongNinjaFile())
}
for _, target := range targets {
if err := checkGlobs(ctx, target); err != nil {
ctx.Fatalf("Error checking globs: %s", err.Error())
}
}
beforeSoongTimestamp := time.Now()
ninja(targets...)
@@ -724,6 +693,157 @@ func runSoong(ctx Context, config Config) {
}
}
// checkGlobs manages the globs that cause soong to rerun.
//
// When soong_build runs, it will run globs. It will write all the globs
// it ran into the "{finalOutFile}.globs" file. Then every build,
// soong_ui will check that file, rerun the globs, and if they changed
// from the results that soong_build got, update the ".glob_results"
// file, causing soong_build to rerun. The ".glob_results" file will
// be empty on the first run of soong_build, because we don't know
// what the globs are yet, but also remain empty until the globs change
// so that we don't run soong_build a second time unnecessarily.
// Both soong_build and soong_ui will also update a ".globs_time" file
// with the time that they ran at every build. When soong_ui checks
// globs, it only reruns globs whose dependencies are newer than the
// time in the ".globs_time" file.
func checkGlobs(ctx Context, finalOutFile string) error {
ctx.BeginTrace(metrics.RunSoong, "check_globs")
defer ctx.EndTrace()
st := ctx.Status.StartTool()
st.Status("Running globs...")
defer st.Finish()
globsFile, err := os.Open(finalOutFile + ".globs")
if errors.Is(err, fs.ErrNotExist) {
// if the glob file doesn't exist, make sure the glob_results file exists and is empty.
if err := os.MkdirAll(filepath.Dir(finalOutFile), 0777); err != nil {
return err
}
f, err := os.Create(finalOutFile + ".glob_results")
if err != nil {
return err
}
return f.Close()
} else if err != nil {
return err
}
defer globsFile.Close()
globsFileDecoder := json.NewDecoder(globsFile)
globsTimeBytes, err := os.ReadFile(finalOutFile + ".globs_time")
if err != nil {
return err
}
globsTimeMicros, err := strconv.ParseInt(strings.TrimSpace(string(globsTimeBytes)), 10, 64)
if err != nil {
return err
}
globCheckStartTime := time.Now().UnixMicro()
globsChan := make(chan pathtools.GlobResult)
errorsChan := make(chan error)
wg := sync.WaitGroup{}
hasChangedGlobs := false
for i := 0; i < runtime.NumCPU()*2; i++ {
wg.Add(1)
go func() {
for cachedGlob := range globsChan {
// If we've already determined we have changed globs, just finish consuming
// the channel without doing any more checks.
if hasChangedGlobs {
continue
}
// First, check if any of the deps are newer than the last time globs were checked.
// If not, we don't need to rerun the glob.
hasNewDep := false
for _, dep := range cachedGlob.Deps {
info, err := os.Stat(dep)
if err != nil {
errorsChan <- err
continue
}
if info.ModTime().UnixMicro() > globsTimeMicros {
hasNewDep = true
break
}
}
if !hasNewDep {
continue
}
// Then rerun the glob and check if we got the same result as before.
result, err := pathtools.Glob(cachedGlob.Pattern, cachedGlob.Excludes, pathtools.FollowSymlinks)
if err != nil {
errorsChan <- err
} else {
if !slices.Equal(result.Matches, cachedGlob.Matches) {
hasChangedGlobs = true
}
}
}
wg.Done()
}()
}
go func() {
wg.Wait()
close(errorsChan)
}()
errorsWg := sync.WaitGroup{}
errorsWg.Add(1)
var errFromGoRoutines error
go func() {
for result := range errorsChan {
if errFromGoRoutines == nil {
errFromGoRoutines = result
}
}
errorsWg.Done()
}()
var cachedGlob pathtools.GlobResult
for globsFileDecoder.More() {
if err := globsFileDecoder.Decode(&cachedGlob); err != nil {
return err
}
// Need to clone the GlobResult because the json decoder will
// reuse the same slice allocations.
globsChan <- cachedGlob.Clone()
}
close(globsChan)
errorsWg.Wait()
if errFromGoRoutines != nil {
return errFromGoRoutines
}
// Update the globs_time file whether or not we found changed globs,
// so that we don't rerun globs in the future that we just saw didn't change.
err = os.WriteFile(
finalOutFile+".globs_time",
[]byte(fmt.Sprintf("%d\n", globCheckStartTime)),
0666,
)
if err != nil {
return err
}
if hasChangedGlobs {
fmt.Fprintf(os.Stdout, "Globs changed, rerunning soong...\n")
// Write the current time to the glob_results file. We just need
// some unique value to trigger a rerun, it doesn't matter what it is.
err = os.WriteFile(
finalOutFile+".glob_results",
[]byte(fmt.Sprintf("%d\n", globCheckStartTime)),
0666,
)
if err != nil {
return err
}
}
return nil
}
// loadSoongBuildMetrics reads out/soong_build_metrics.pb if it was generated by soong_build and copies the
// events stored in it into the soong_ui trace to provide introspection into how long the different phases of
// soong_build are taking.

4
ui/build/test_build.go
View File

@@ -79,9 +79,6 @@ func testForDanglingRules(ctx Context, config Config) {
// out/build_date.txt is considered a "source file"
buildDatetimeFilePath := filepath.Join(outDir, "build_date.txt")
// bpglob is built explicitly using Microfactory
bpglob := filepath.Join(config.SoongOutDir(), "bpglob")
// release-config files are generated from the initial lunch or Kati phase
// before running soong and ninja.
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
@@ -105,7 +102,6 @@ func testForDanglingRules(ctx Context, config Config) {
line == extraVariablesFilePath ||
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||
line == bpglob ||
strings.HasPrefix(line, releaseConfigDir) ||
buildFingerPrintFilePattern.MatchString(line) {
// Leaf node is in one of Soong's bootstrap directories, which do not have