Simplify soong_build top-level code

* `android.Context` holds `android.Config`, so provide an accessor to it
  and avoid passing them separately to a lot of functions.
* factor common code in `readBazelPaths`/`getExistingBazelRelatedFiles`
  into `readFileLines`
* refactor check-error-and-quit into `maybeQuit`
* use switch instead of if-elseif-... where appropriate
* rearrange the code in `runApiBp2build`, `runSymlinkForestCreation`

Test: treehugger
Change-Id: I639929c21ec4999cdfd737c07136f32df2d488bc
This commit is contained in:
Sasha Smundak
2022-12-13 14:18:58 -08:00
parent af5ca926be
commit 1845f42085
2 changed files with 189 additions and 254 deletions

View File

@@ -35,7 +35,7 @@ type sortableComponent interface {
// tests. // tests.
componentName() string componentName() string
// register registers this component in the supplied context. // registers this component in the supplied context.
register(ctx *Context) register(ctx *Context)
} }
@@ -124,7 +124,7 @@ func SingletonFactoryAdaptor(ctx *Context, factory SingletonFactory) blueprint.S
return func() blueprint.Singleton { return func() blueprint.Singleton {
singleton := factory() singleton := factory()
if makevars, ok := singleton.(SingletonMakeVarsProvider); ok { if makevars, ok := singleton.(SingletonMakeVarsProvider); ok {
registerSingletonMakeVarsProvider(ctx.config, makevars) ctx.registerSingletonMakeVarsProvider(makevars)
} }
return &singletonAdaptor{Singleton: singleton} return &singletonAdaptor{Singleton: singleton}
} }
@@ -208,6 +208,14 @@ func (ctx *Context) Register() {
singletons.registerAll(ctx) singletons.registerAll(ctx)
} }
func (ctx *Context) Config() Config {
return ctx.config
}
func (ctx *Context) registerSingletonMakeVarsProvider(makevars SingletonMakeVarsProvider) {
registerSingletonMakeVarsProvider(ctx.config, makevars)
}
func collateGloballyRegisteredSingletons() sortableComponents { func collateGloballyRegisteredSingletons() sortableComponents {
allSingletons := append(sortableComponents(nil), singletons...) allSingletons := append(sortableComponents(nil), singletons...)
allSingletons = append(allSingletons, allSingletons = append(allSingletons,
@@ -335,7 +343,7 @@ func (ctx *initRegistrationContext) PreArchMutators(f RegisterMutatorFunc) {
PreArchMutators(f) PreArchMutators(f)
} }
func (ctx *initRegistrationContext) HardCodedPreArchMutators(f RegisterMutatorFunc) { func (ctx *initRegistrationContext) HardCodedPreArchMutators(_ RegisterMutatorFunc) {
// Nothing to do as the mutators are hard code in preArch in mutator.go // Nothing to do as the mutators are hard code in preArch in mutator.go
} }

View File

@@ -108,47 +108,41 @@ func newContext(configuration android.Config) *android.Context {
// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a // Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata // BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
// for modules that should be handled by Bazel. // for modules that should be handled by Bazel.
func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) string { func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
ctx.EventHandler.Begin("mixed_build") ctx.EventHandler.Begin("mixed_build")
defer ctx.EventHandler.End("mixed_build") defer ctx.EventHandler.End("mixed_build")
bazelHook := func() error { bazelHook := func() error {
return configuration.BazelContext.InvokeBazel(configuration, ctx) return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx)
} }
ctx.SetBeforePrepareBuildActionsHook(bazelHook) ctx.SetBeforePrepareBuildActionsHook(bazelHook)
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, configuration) ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, ctx.Config())
ninjaDeps = append(ninjaDeps, extraNinjaDeps...) ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
bazelPaths, err := readBazelPaths(configuration) bazelPaths, err := readFileLines(ctx.Config().Getenv("BAZEL_DEPS_FILE"))
if err != nil { if err != nil {
panic("Bazel deps file not found: " + err.Error()) panic("Bazel deps file not found: " + err.Error())
} }
ninjaDeps = append(ninjaDeps, bazelPaths...) ninjaDeps = append(ninjaDeps, bazelPaths...)
ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
ninjaDeps = append(ninjaDeps, globListFiles...)
writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.OutFile return cmdlineArgs.OutFile
} }
// Run the code-generation phase to convert BazelTargetModules to BUILD files. // Run the code-generation phase to convert BazelTargetModules to BUILD files.
func runQueryView(queryviewDir, queryviewMarker string, configuration android.Config, ctx *android.Context) { func runQueryView(queryviewDir, queryviewMarker string, ctx *android.Context) {
ctx.EventHandler.Begin("queryview") ctx.EventHandler.Begin("queryview")
defer ctx.EventHandler.End("queryview") defer ctx.EventHandler.End("queryview")
codegenContext := bp2build.NewCodegenContext(configuration, ctx, bp2build.QueryView) codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView)
absoluteQueryViewDir := shared.JoinPath(topDir, queryviewDir) err := createBazelWorkspace(codegenContext, shared.JoinPath(topDir, queryviewDir))
if err := createBazelWorkspace(codegenContext, absoluteQueryViewDir); err != nil { maybeQuit(err, "")
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
touch(shared.JoinPath(topDir, queryviewMarker)) touch(shared.JoinPath(topDir, queryviewMarker))
} }
// Run the code-generation phase to convert API contributions to BUILD files. // Run the code-generation phase to convert API contributions to BUILD files.
// Return marker file for the new synthetic workspace // Return marker file for the new synthetic workspace
func runApiBp2build(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) string { func runApiBp2build(ctx *android.Context, extraNinjaDeps []string) string {
ctx.EventHandler.Begin("api_bp2build") ctx.EventHandler.Begin("api_bp2build")
defer ctx.EventHandler.End("api_bp2build") defer ctx.EventHandler.End("api_bp2build")
// Do not allow missing dependencies. // Do not allow missing dependencies.
@@ -168,25 +162,22 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args,
bootstrap.StopBeforePrepareBuildActions, bootstrap.StopBeforePrepareBuildActions,
ctx.Context, ctx.Context,
configuration) ctx.Config())
ninjaDeps = append(ninjaDeps, extraNinjaDeps...) ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
// Add the globbed dependencies // Add the globbed dependencies
globs := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
ninjaDeps = append(ninjaDeps, globs...)
// Run codegen to generate BUILD files // Run codegen to generate BUILD files
codegenContext := bp2build.NewCodegenContext(configuration, ctx, bp2build.ApiBp2build) codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build)
absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir) absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir)
if err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir); err != nil { err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir)
fmt.Fprintf(os.Stderr, "%s", err) maybeQuit(err, "")
os.Exit(1)
}
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
// Create soong_injection repository // Create soong_injection repository
soongInjectionFiles := bp2build.CreateSoongInjectionFiles(configuration, bp2build.CreateCodegenMetrics()) soongInjectionFiles := bp2build.CreateSoongInjectionFiles(ctx.Config(), bp2build.CreateCodegenMetrics())
absoluteSoongInjectionDir := shared.JoinPath(topDir, configuration.SoongOutDir(), bazel.SoongInjectionDirName) absoluteSoongInjectionDir := shared.JoinPath(topDir, ctx.Config().SoongOutDir(), bazel.SoongInjectionDirName)
for _, file := range soongInjectionFiles { for _, file := range soongInjectionFiles {
// The API targets in api_bp2build workspace do not have any dependency on api_bp2build. // The API targets in api_bp2build workspace do not have any dependency on api_bp2build.
// But we need to create these files to prevent errors during Bazel analysis. // But we need to create these files to prevent errors during Bazel analysis.
@@ -196,24 +187,14 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
writeReadWriteFile(absoluteSoongInjectionDir, file) writeReadWriteFile(absoluteSoongInjectionDir, file)
} }
workspace := shared.JoinPath(configuration.SoongOutDir(), "api_bp2build") workspace := shared.JoinPath(ctx.Config().SoongOutDir(), "api_bp2build")
excludes := bazelArtifacts()
// Exclude all src BUILD files
excludes = append(excludes, apiBuildFileExcludes()...)
// Android.bp files for api surfaces are mounted to out/, but out/ should not be a
// dep for api_bp2build.
// Otherwise, api_bp2build will be run every single time
excludes = append(excludes, configuration.OutDir())
// Create the symlink forest // Create the symlink forest
symlinkDeps := bp2build.PlantSymlinkForest( symlinkDeps := bp2build.PlantSymlinkForest(
configuration.IsEnvTrue("BP2BUILD_VERBOSE"), ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE"),
topDir, topDir,
workspace, workspace,
cmdlineArgs.BazelApiBp2buildDir, cmdlineArgs.BazelApiBp2buildDir,
excludes) apiBuildFileExcludes(ctx))
ninjaDeps = append(ninjaDeps, symlinkDeps...) ninjaDeps = append(ninjaDeps, symlinkDeps...)
workspaceMarkerFile := workspace + ".marker" workspaceMarkerFile := workspace + ".marker"
@@ -224,15 +205,12 @@ func runApiBp2build(configuration android.Config, ctx *android.Context, extraNin
// With some exceptions, api_bp2build does not have any dependencies on the checked-in BUILD files // With some exceptions, api_bp2build does not have any dependencies on the checked-in BUILD files
// Exclude them from the generated workspace to prevent unrelated errors during the loading phase // Exclude them from the generated workspace to prevent unrelated errors during the loading phase
func apiBuildFileExcludes() []string { func apiBuildFileExcludes(ctx *android.Context) []string {
ret := make([]string, 0) ret := bazelArtifacts()
srcs, err := getExistingBazelRelatedFiles(topDir) srcs, err := getExistingBazelRelatedFiles(topDir)
if err != nil { maybeQuit(err, "Error determining existing Bazel-related files")
fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err)
os.Exit(1)
}
for _, src := range srcs { for _, src := range srcs {
// Exclude all src BUILD files
if src != "WORKSPACE" && if src != "WORKSPACE" &&
src != "BUILD" && src != "BUILD" &&
src != "BUILD.bazel" && src != "BUILD.bazel" &&
@@ -242,6 +220,9 @@ func apiBuildFileExcludes() []string {
ret = append(ret, src) ret = append(ret, src)
} }
} }
// Android.bp files for api surfaces are mounted to out/, but out/ should not be a
// dep for api_bp2build. Otherwise, api_bp2build will be run every single time
ret = append(ret, ctx.Config().OutDir())
return ret return ret
} }
@@ -252,36 +233,30 @@ func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandl
} }
metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb") metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb")
err := android.WriteMetrics(configuration, eventHandler, metricsFile) err := android.WriteMetrics(configuration, eventHandler, metricsFile)
if err != nil { maybeQuit(err, "error writing soong_build metrics %s", metricsFile)
fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err)
os.Exit(1)
}
} }
func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) { func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) {
graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile)) graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile))
actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleActionsFile)) maybeQuit(graphErr, "graph err")
if graphErr != nil || actionsErr != nil {
fmt.Fprintf(os.Stderr, "Graph err: %s, actions err: %s", graphErr, actionsErr)
os.Exit(1)
}
defer graphFile.Close() defer graphFile.Close()
actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleActionsFile))
maybeQuit(actionsErr, "actions err")
defer actionsFile.Close() defer actionsFile.Close()
ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile) ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
} }
func writeBuildGlobsNinjaFile(ctx *android.Context, buildDir string, config interface{}) []string { func writeBuildGlobsNinjaFile(ctx *android.Context) []string {
ctx.EventHandler.Begin("globs_ninja_file") ctx.EventHandler.Begin("globs_ninja_file")
defer ctx.EventHandler.End("globs_ninja_file") defer ctx.EventHandler.End("globs_ninja_file")
globDir := bootstrap.GlobDirectory(buildDir, globListDir) globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{ bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
GlobLister: ctx.Globs, GlobLister: ctx.Globs,
GlobFile: globFile, GlobFile: globFile,
GlobDir: globDir, GlobDir: globDir,
SrcDir: ctx.SrcDir(), SrcDir: ctx.SrcDir(),
}, config) }, ctx.Config())
return bootstrap.GlobFileListFiles(globDir) return bootstrap.GlobFileListFiles(globDir)
} }
@@ -290,83 +265,50 @@ func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDe
defer eventHandler.End("ninja_deps") defer eventHandler.End("ninja_deps")
depFile := shared.JoinPath(topDir, outputFile+".d") depFile := shared.JoinPath(topDir, outputFile+".d")
err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps) err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
if err != nil { maybeQuit(err, "error writing depfile '%s'", depFile)
fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err)
os.Exit(1)
}
}
// doChosenActivity runs Soong for a specific activity, like bp2build, queryview
// or the actual Soong build for the build.ninja file. Returns the top level
// output file of the specific activity.
func doChosenActivity(ctx *android.Context, configuration android.Config, extraNinjaDeps []string, metricsDir string) string {
if configuration.BuildMode == android.SymlinkForest {
return runSymlinkForestCreation(configuration, ctx, extraNinjaDeps, metricsDir)
} else if configuration.BuildMode == android.Bp2build {
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
return runBp2Build(configuration, ctx, extraNinjaDeps, metricsDir)
} else if configuration.BuildMode == android.ApiBp2build {
outputFile := runApiBp2build(configuration, ctx, extraNinjaDeps)
writeMetrics(configuration, ctx.EventHandler, metricsDir)
return outputFile
} else {
ctx.Register()
var outputFile string
if configuration.IsMixedBuildsEnabled() {
outputFile = runMixedModeBuild(configuration, ctx, extraNinjaDeps)
} else {
outputFile = runSoongOnlyBuild(configuration, ctx, extraNinjaDeps)
}
writeMetrics(configuration, ctx.EventHandler, metricsDir)
return outputFile
}
} }
// runSoongOnlyBuild runs the standard Soong build in a number of different modes. // runSoongOnlyBuild runs the standard Soong build in a number of different modes.
func runSoongOnlyBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) string { func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
ctx.EventHandler.Begin("soong_build") ctx.EventHandler.Begin("soong_build")
defer ctx.EventHandler.End("soong_build") defer ctx.EventHandler.End("soong_build")
var stopBefore bootstrap.StopBefore var stopBefore bootstrap.StopBefore
if configuration.BuildMode == android.GenerateModuleGraph { switch ctx.Config().BuildMode {
case android.GenerateModuleGraph:
stopBefore = bootstrap.StopBeforeWriteNinja stopBefore = bootstrap.StopBeforeWriteNinja
} else if configuration.BuildMode == android.GenerateQueryView || configuration.BuildMode == android.GenerateDocFile { case android.GenerateQueryView | android.GenerateDocFile:
stopBefore = bootstrap.StopBeforePrepareBuildActions stopBefore = bootstrap.StopBeforePrepareBuildActions
} else { default:
stopBefore = bootstrap.DoEverything stopBefore = bootstrap.DoEverything
} }
ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, configuration) ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
ninjaDeps = append(ninjaDeps, extraNinjaDeps...) ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) globListFiles := writeBuildGlobsNinjaFile(ctx)
ninjaDeps = append(ninjaDeps, globListFiles...) ninjaDeps = append(ninjaDeps, globListFiles...)
// Convert the Soong module graph into Bazel BUILD files. // Convert the Soong module graph into Bazel BUILD files.
if configuration.BuildMode == android.GenerateQueryView { switch ctx.Config().BuildMode {
case android.GenerateQueryView:
queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker" queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, configuration, ctx) runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps) writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
return queryviewMarkerFile return queryviewMarkerFile
} else if configuration.BuildMode == android.GenerateModuleGraph { case android.GenerateModuleGraph:
writeJsonModuleGraphAndActions(ctx, cmdlineArgs) writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps) writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.ModuleGraphFile return cmdlineArgs.ModuleGraphFile
} else if configuration.BuildMode == android.GenerateDocFile { case android.GenerateDocFile:
// TODO: we could make writeDocs() return the list of documentation files // TODO: we could make writeDocs() return the list of documentation files
// written and add them to the .d file. Then soong_docs would be re-run // written and add them to the .d file. Then soong_docs would be re-run
// whenever one is deleted. // whenever one is deleted.
if err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile)); err != nil { err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err) maybeQuit(err, "error building Soong documentation")
os.Exit(1)
}
writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps) writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
return cmdlineArgs.DocFile return cmdlineArgs.DocFile
} else { default:
// The actual output (build.ninja) was written in the RunBlueprint() call // The actual output (build.ninja) was written in the RunBlueprint() call
// above // above
writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
@@ -392,13 +334,8 @@ func parseAvailableEnv() map[string]string {
fmt.Fprintf(os.Stderr, "--available_env not set\n") fmt.Fprintf(os.Stderr, "--available_env not set\n")
os.Exit(1) os.Exit(1)
} }
result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile)) result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
if err != nil { maybeQuit(err, "error reading available environment file '%s'", availableEnvFile)
fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err)
os.Exit(1)
}
return result return result
} }
@@ -410,10 +347,7 @@ func main() {
availableEnv := parseAvailableEnv() availableEnv := parseAvailableEnv()
configuration, err := android.NewConfig(cmdlineArgs, availableEnv) configuration, err := android.NewConfig(cmdlineArgs, availableEnv)
if err != nil { maybeQuit(err, "")
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" { if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
configuration.SetAllowMissingDependencies() configuration.SetAllowMissingDependencies()
} }
@@ -427,12 +361,33 @@ func main() {
// Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will // Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
// change between every CI build, so tracking it would require re-running Soong for every build. // change between every CI build, so tracking it would require re-running Soong for every build.
logDir := availableEnv["LOG_DIR"] metricsDir := availableEnv["LOG_DIR"]
ctx := newContext(configuration) ctx := newContext(configuration)
finalOutputFile := doChosenActivity(ctx, configuration, extraNinjaDeps, logDir) var finalOutputFile string
// Run Soong for a specific activity, like bp2build, queryview
// or the actual Soong build for the build.ninja file.
switch configuration.BuildMode {
case android.SymlinkForest:
finalOutputFile = runSymlinkForestCreation(ctx, extraNinjaDeps, metricsDir)
case android.Bp2build:
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
finalOutputFile = runBp2Build(ctx, extraNinjaDeps, metricsDir)
case android.ApiBp2build:
finalOutputFile = runApiBp2build(ctx, extraNinjaDeps)
writeMetrics(configuration, ctx.EventHandler, metricsDir)
default:
ctx.Register()
if configuration.IsMixedBuildsEnabled() {
finalOutputFile = runMixedModeBuild(ctx, extraNinjaDeps)
} else {
finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps)
}
writeMetrics(configuration, ctx.EventHandler, metricsDir)
}
writeUsedEnvironmentFile(configuration, finalOutputFile) writeUsedEnvironmentFile(configuration, finalOutputFile)
} }
@@ -443,24 +398,19 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri
path := shared.JoinPath(topDir, usedEnvFile) path := shared.JoinPath(topDir, usedEnvFile)
data, err := shared.EnvFileContents(configuration.EnvDeps()) data, err := shared.EnvFileContents(configuration.EnvDeps())
if err != nil { maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
os.Exit(1)
}
if preexistingData, err := os.ReadFile(path); err != nil { if preexistingData, err := os.ReadFile(path); err != nil {
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "error reading used environment file '%s': %s\n", usedEnvFile, err) maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
os.Exit(1)
} }
} else if bytes.Equal(preexistingData, data) { } else if bytes.Equal(preexistingData, data) {
// used environment file is unchanged // used environment file is unchanged
return return
} }
if err = os.WriteFile(path, data, 0666); err != nil { err = os.WriteFile(path, data, 0666)
fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
os.Exit(1)
}
// Touch the output file so that it's not older than the file we just // Touch the output file so that it's not older than the file we just
// wrote. We can't write the environment file earlier because one an access // wrote. We can't write the environment file earlier because one an access
// new environment variables while writing it. // new environment variables while writing it.
@@ -469,88 +419,13 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri
func touch(path string) { func touch(path string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil { maybeQuit(err, "Error touching '%s'", path)
fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
os.Exit(1)
}
err = f.Close() err = f.Close()
if err != nil { maybeQuit(err, "Error touching '%s'", path)
fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
os.Exit(1)
}
currentTime := time.Now().Local() currentTime := time.Now().Local()
err = os.Chtimes(path, currentTime, currentTime) err = os.Chtimes(path, currentTime, currentTime)
if err != nil { maybeQuit(err, "error touching '%s'", path)
fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err)
os.Exit(1)
}
}
// Find BUILD files in the srcDir which are not in the allowlist
// (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir)
// and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
func getPathsToIgnoredBuildFiles(config android.Bp2BuildConversionAllowlist, topDir string, srcDirBazelFiles []string, verbose bool) []string {
paths := make([]string, 0)
for _, srcDirBazelFileRelativePath := range srcDirBazelFiles {
srcDirBazelFileFullPath := shared.JoinPath(topDir, srcDirBazelFileRelativePath)
fileInfo, err := os.Stat(srcDirBazelFileFullPath)
if err != nil {
// Warn about error, but continue trying to check files
fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", srcDirBazelFileFullPath, err)
continue
}
if fileInfo.IsDir() {
// Don't ignore entire directories
continue
}
if fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel" {
// Don't ignore this file - it is not a build file
continue
}
if config.ShouldKeepExistingBuildFileForDir(filepath.Dir(srcDirBazelFileRelativePath)) {
// Don't ignore this existing build file
continue
}
if verbose {
fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", srcDirBazelFileRelativePath)
}
paths = append(paths, srcDirBazelFileRelativePath)
}
return paths
}
// Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work
func getTemporaryExcludes() []string {
excludes := make([]string, 0)
// FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite symlink expansion error for Bazel
excludes = append(excludes, "external/autotest/venv/autotest_lib")
excludes = append(excludes, "external/autotest/autotest_lib")
excludes = append(excludes, "external/autotest/client/autotest_lib/client")
// FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
// It contains several symlinks back to real source dirs, and those source dirs contain BUILD files we want to ignore
excludes = append(excludes, "external/google-fruit/extras/bazel_root/third_party/fruit")
// FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
excludes = append(excludes, "frameworks/compile/slang")
// FIXME(b/260809113): 'prebuilts/clang/host/linux-x86/clang-dev' is a tool-generated symlink directory that contains a BUILD file.
// The bazel files finder code doesn't traverse into symlink dirs, and hence is not aware of this BUILD file and exclude it accordingly
// during symlink forest generation when checking against keepExistingBuildFiles allowlist.
//
// This is necessary because globs in //prebuilts/clang/host/linux-x86/BUILD
// currently assume no subpackages (keepExistingBuildFile is not recursive for that directory).
//
// This is a bandaid until we the symlink forest logic can intelligently exclude BUILD files found in source symlink dirs according
// to the keepExistingBuildFile allowlist.
excludes = append(excludes, "prebuilts/clang/host/linux-x86/clang-dev")
return excludes
} }
// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully) // Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)
@@ -561,12 +436,7 @@ func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
// Assume this was a relative path under topDir // Assume this was a relative path under topDir
bazelFinderFile = filepath.Join(topDir, bazelFinderFile) bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
} }
data, err := os.ReadFile(bazelFinderFile) return readFileLines(bazelFinderFile)
if err != nil {
return nil, err
}
files := strings.Split(strings.TrimSpace(string(data)), "\n")
return files, nil
} }
func bazelArtifacts() []string { func bazelArtifacts() []string {
@@ -588,36 +458,20 @@ func bazelArtifacts() []string {
// Ideally, bp2build would write a file that contains instructions to the // Ideally, bp2build would write a file that contains instructions to the
// symlink tree creation binary. Then the latter would not need to depend on // symlink tree creation binary. Then the latter would not need to depend on
// the very heavy-weight machinery of soong_build . // the very heavy-weight machinery of soong_build .
func runSymlinkForestCreation(configuration android.Config, ctx *android.Context, extraNinjaDeps []string, metricsDir string) string { func runSymlinkForestCreation(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
ctx.EventHandler.Do("symlink_forest", func() { ctx.EventHandler.Do("symlink_forest", func() {
var ninjaDeps []string var ninjaDeps []string
ninjaDeps = append(ninjaDeps, extraNinjaDeps...) ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
verbose := ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE")
generatedRoot := shared.JoinPath(configuration.SoongOutDir(), "bp2build")
workspaceRoot := shared.JoinPath(configuration.SoongOutDir(), "workspace")
excludes := bazelArtifacts()
if cmdlineArgs.OutDir[0] != '/' {
excludes = append(excludes, cmdlineArgs.OutDir)
}
existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)
if err != nil {
fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err)
os.Exit(1)
}
pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(configuration.Bp2buildPackageConfig, topDir, existingBazelRelatedFiles, configuration.IsEnvTrue("BP2BUILD_VERBOSE"))
excludes = append(excludes, pathsToIgnoredBuildFiles...)
excludes = append(excludes, getTemporaryExcludes()...)
// PlantSymlinkForest() returns all the directories that were readdir()'ed. // PlantSymlinkForest() returns all the directories that were readdir()'ed.
// Such a directory SHOULD be added to `ninjaDeps` so that a child directory // Such a directory SHOULD be added to `ninjaDeps` so that a child directory
// or file created/deleted under it would trigger an update of the symlink forest. // or file created/deleted under it would trigger an update of the symlink forest.
generatedRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "bp2build")
workspaceRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "workspace")
ctx.EventHandler.Do("plant", func() { ctx.EventHandler.Do("plant", func() {
symlinkForestDeps := bp2build.PlantSymlinkForest( symlinkForestDeps := bp2build.PlantSymlinkForest(
configuration.IsEnvTrue("BP2BUILD_VERBOSE"), topDir, workspaceRoot, generatedRoot, excludes) verbose, topDir, workspaceRoot, generatedRoot, excludedFromSymlinkForest(ctx, verbose))
ninjaDeps = append(ninjaDeps, symlinkForestDeps...) ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
}) })
@@ -633,21 +487,84 @@ func runSymlinkForestCreation(configuration android.Config, ctx *android.Context
//invocation of codegen. We should simply use a separate .pb file //invocation of codegen. We should simply use a separate .pb file
} }
writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir) writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
return cmdlineArgs.SymlinkForestMarker return cmdlineArgs.SymlinkForestMarker
} }
func excludedFromSymlinkForest(ctx *android.Context, verbose bool) []string {
excluded := bazelArtifacts()
if cmdlineArgs.OutDir[0] != '/' {
excluded = append(excluded, cmdlineArgs.OutDir)
}
// Find BUILD files in the srcDir which are not in the allowlist
// (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir)
// and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
existingBazelFiles, err := getExistingBazelRelatedFiles(topDir)
maybeQuit(err, "Error determining existing Bazel-related files")
for _, path := range existingBazelFiles {
fullPath := shared.JoinPath(topDir, path)
fileInfo, err2 := os.Stat(fullPath)
if err2 != nil {
// Warn about error, but continue trying to check files
fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fullPath, err2)
continue
}
// Exclude only files named 'BUILD' or 'BUILD.bazel' and unless forcibly kept
if fileInfo.IsDir() ||
(fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel") ||
ctx.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) {
// Don't ignore this existing build file
continue
}
if verbose {
fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", path)
}
excluded = append(excluded, path)
}
// Temporarily exclude stuff to make `bazel build //external/...` (and `bazel build //frameworks/...`) work
excluded = append(excluded,
// FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite
// symlink expansion error for Bazel
"external/autotest/venv/autotest_lib",
"external/autotest/autotest_lib",
"external/autotest/client/autotest_lib/client",
// FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
// It contains several symlinks back to real source dirs, and those source dirs contain
// BUILD files we want to ignore
"external/google-fruit/extras/bazel_root/third_party/fruit",
// FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
"frameworks/compile/slang",
// FIXME(b/260809113): 'prebuilts/clang/host/linux-x86/clang-dev' is a tool-generated symlink
// directory that contains a BUILD file. The bazel files finder code doesn't traverse into symlink dirs,
// and hence is not aware of this BUILD file and exclude it accordingly during symlink forest generation
// when checking against keepExistingBuildFiles allowlist.
//
// This is necessary because globs in //prebuilts/clang/host/linux-x86/BUILD
// currently assume no subpackages (keepExistingBuildFile is not recursive for that directory).
//
// This is a bandaid until we the symlink forest logic can intelligently exclude BUILD files found in
// source symlink dirs according to the keepExistingBuildFile allowlist.
"prebuilts/clang/host/linux-x86/clang-dev",
)
return excluded
}
// Run Soong in the bp2build mode. This creates a standalone context that registers // Run Soong in the bp2build mode. This creates a standalone context that registers
// an alternate pipeline of mutators and singletons specifically for generating // an alternate pipeline of mutators and singletons specifically for generating
// Bazel BUILD files instead of Ninja files. // Bazel BUILD files instead of Ninja files.
func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaDeps []string, metricsDir string) string { func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
var codegenMetrics *bp2build.CodegenMetrics var codegenMetrics *bp2build.CodegenMetrics
ctx.EventHandler.Do("bp2build", func() { ctx.EventHandler.Do("bp2build", func() {
// Propagate "allow misssing dependencies" bit. This is normally set in // Propagate "allow misssing dependencies" bit. This is normally set in
// newContext(), but we create ctx without calling that method. // newContext(), but we create ctx without calling that method.
ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies())
ctx.SetNameInterface(newNameResolver(configuration)) ctx.SetNameInterface(newNameResolver(ctx.Config()))
ctx.RegisterForBazelConversion() ctx.RegisterForBazelConversion()
ctx.SetModuleListFile(cmdlineArgs.ModuleListFile) ctx.SetModuleListFile(cmdlineArgs.ModuleListFile)
@@ -659,16 +576,17 @@ func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaD
// from the regular Modules. // from the regular Modules.
ctx.EventHandler.Do("bootstrap", func() { ctx.EventHandler.Do("bootstrap", func() {
blueprintArgs := cmdlineArgs blueprintArgs := cmdlineArgs
bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs.Args, bootstrap.StopBeforePrepareBuildActions, ctx.Context, configuration) bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs.Args,
bootstrap.StopBeforePrepareBuildActions, ctx.Context, ctx.Config())
ninjaDeps = append(ninjaDeps, bootstrapDeps...) ninjaDeps = append(ninjaDeps, bootstrapDeps...)
}) })
globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) globListFiles := writeBuildGlobsNinjaFile(ctx)
ninjaDeps = append(ninjaDeps, globListFiles...) ninjaDeps = append(ninjaDeps, globListFiles...)
// Run the code-generation phase to convert BazelTargetModules to BUILD files // Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion codegenMetrics to the user. // and print conversion codegenMetrics to the user.
codegenContext := bp2build.NewCodegenContext(configuration, ctx, bp2build.Bp2Build) codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build)
ctx.EventHandler.Do("codegen", func() { ctx.EventHandler.Do("codegen", func() {
codegenMetrics = bp2build.Codegen(codegenContext) codegenMetrics = bp2build.Codegen(codegenContext)
}) })
@@ -682,7 +600,7 @@ func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaD
// Only report metrics when in bp2build mode. The metrics aren't relevant // Only report metrics when in bp2build mode. The metrics aren't relevant
// for queryview, since that's a total repo-wide conversion and there's a // for queryview, since that's a total repo-wide conversion and there's a
// 1:1 mapping for each module. // 1:1 mapping for each module.
if configuration.IsEnvTrue("BP2BUILD_VERBOSE") { if ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE") {
codegenMetrics.Print() codegenMetrics.Print()
} }
writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir) writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
@@ -705,13 +623,22 @@ func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics, eventHandler
codegenMetrics.Write(metricsDir) codegenMetrics.Write(metricsDir)
} }
func readBazelPaths(configuration android.Config) ([]string, error) { func readFileLines(path string) ([]string, error) {
depsPath := configuration.Getenv("BAZEL_DEPS_FILE") data, err := os.ReadFile(path)
if err == nil {
data, err := os.ReadFile(depsPath) return strings.Split(strings.TrimSpace(string(data)), "\n"), nil
if err != nil {
return nil, err
} }
paths := strings.Split(strings.TrimSpace(string(data)), "\n") return nil, err
return paths, nil
}
func maybeQuit(err error, format string, args ...interface{}) {
if err == nil {
return
}
if format != "" {
fmt.Fprintln(os.Stderr, fmt.Sprintf(format, args...)+": "+err.Error())
} else {
fmt.Fprintln(os.Stderr, err)
}
os.Exit(1)
} }