diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index cff65f92e..beda1842a 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -25,6 +25,7 @@ import ( "android/soong/bp2build" "android/soong/shared" + "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/deptools" "github.com/google/blueprint/pathtools" @@ -43,6 +44,7 @@ var ( delveListen string delvePath string + moduleGraphFile string docFile string bazelQueryViewDir string bp2buildMarker string @@ -62,6 +64,7 @@ func init() { flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set") // Flags representing various modes soong_build can run in + flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output") flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") @@ -71,7 +74,6 @@ func init() { flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files") flag.StringVar(&cmdlineArgs.BuildDir, "b", ".", "the build output directory") flag.StringVar(&cmdlineArgs.NinjaBuildDir, "n", "", "the ninja builddir directory") - flag.StringVar(&cmdlineArgs.DepFile, "d", "", "the dependency file to output") flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file") flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file") flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file") @@ -149,11 +151,7 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps) - if err != nil { - fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err) - os.Exit(1) - } + writeDepFile(secondArgs.OutFile, ninjaDeps) } // Run the code-generation phase to convert BazelTargetModules to BUILD files. @@ -185,8 +183,8 @@ func writeMetrics(configuration android.Config) { } } -func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) { - f, err := os.Create(path) +func writeJsonModuleGraph(ctx *android.Context, path string) { + f, err := os.Create(shared.JoinPath(topDir, path)) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -194,7 +192,6 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa defer f.Close() ctx.Context.PrintJSONGraph(f) - writeFakeNinjaFile(extraNinjaDeps, configuration.SoongOutDir()) } func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string { @@ -208,6 +205,15 @@ func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.Mu return bootstrap.GlobFileListFiles(globDir) } +func writeDepFile(outputFile string, ninjaDeps []string) { + depFile := shared.JoinPath(topDir, outputFile+".d") + err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err) + os.Exit(1) + } +} + // doChosenActivity runs Soong for a specific activity, like bp2build, queryview // or the actual Soong build for the build.ninja file. Returns the top level // output file of the specific activity. @@ -215,10 +221,9 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str bazelConversionRequested := bp2buildMarker != "" mixedModeBuild := configuration.BazelContext.BazelEnabled() generateQueryView := bazelQueryViewDir != "" - jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH") blueprintArgs := cmdlineArgs - prepareBuildActions := !generateQueryView && jsonModuleFile == "" + prepareBuildActions := !generateQueryView && moduleGraphFile == "" if bazelConversionRequested { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. @@ -236,24 +241,21 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps) - if err != nil { - fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err) - os.Exit(1) + // Convert the Soong module graph into Bazel BUILD files. + if generateQueryView { + runQueryView(configuration, ctx) + return cmdlineArgs.OutFile // TODO: This is a lie + } else if moduleGraphFile != "" { + writeJsonModuleGraph(ctx, moduleGraphFile) + writeDepFile(moduleGraphFile, ninjaDeps) + return moduleGraphFile + } else { + // The actual output (build.ninja) was written in the RunBlueprint() call + // above + writeDepFile(cmdlineArgs.OutFile, ninjaDeps) } } - // Convert the Soong module graph into Bazel BUILD files. - if generateQueryView { - runQueryView(configuration, ctx) - return cmdlineArgs.OutFile // TODO: This is a lie - } - - if jsonModuleFile != "" { - writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps) - return cmdlineArgs.OutFile // TODO: This is a lie - } - writeMetrics(configuration) return cmdlineArgs.OutFile } @@ -348,29 +350,6 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri touch(shared.JoinPath(topDir, finalOutputFile)) } -// Workarounds to support running bp2build in a clean AOSP checkout with no -// prior builds, and exiting early as soon as the BUILD files get generated, -// therefore not creating build.ninja files that soong_ui and callers of -// soong_build expects. -// -// These files are: build.ninja and build.ninja.d. Since Kati hasn't been -// ran as well, and `nothing` is defined in a .mk file, there isn't a ninja -// target called `nothing`, so we manually create it here. -func writeFakeNinjaFile(extraNinjaDeps []string, soongOutDir string) { - extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ") - - ninjaFileName := "build.ninja" - ninjaFile := shared.JoinPath(topDir, soongOutDir, ninjaFileName) - ninjaFileD := shared.JoinPath(topDir, soongOutDir, ninjaFileName+".d") - // A workaround to create the 'nothing' ninja target so `m nothing` works, - // since bp2build runs without Kati, and the 'nothing' target is declared in - // a Makefile. - ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n phony_output = true\n"), 0666) - ioutil.WriteFile(ninjaFileD, - []byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFile, extraNinjaDepsString)), - 0666) -} - func touch(path string) { f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) if err != nil { @@ -550,12 +529,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) ninjaDeps = append(ninjaDeps, symlinkForestDeps...) - depFile := bp2buildMarker + ".d" - err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps) - if err != nil { - fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err) - os.Exit(1) - } + writeDepFile(bp2buildMarker, ninjaDeps) // Create an empty bp2build marker file. touch(shared.JoinPath(topDir, bp2buildMarker)) diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 76b2ee99a..610e42771 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -607,12 +607,36 @@ EOF function test_dump_json_module_graph() { setup - SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong - if [[ ! -r "$MOCK_TOP/modules.json" ]]; then + GENERATE_JSON_MODULE_GRAPH=1 run_soong + if [[ ! -r "out/soong//module-graph.json" ]]; then fail "JSON file was not created" fi } +function test_json_module_graph_back_and_forth_null_build() { + setup + + run_soong + local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja) + + GENERATE_JSON_MODULE_GRAPH=1 run_soong + local json_mtime1=$(stat -c "%y" out/soong/module-graph.json) + + run_soong + local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja) + if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then + fail "Output Ninja file changed after writing JSON module graph" + fi + + GENERATE_JSON_MODULE_GRAPH=1 run_soong + local json_mtime2=$(stat -c "%y" out/soong/module-graph.json) + if [[ "$json_mtime1" != "$json_mtime2" ]]; then + fail "JSON module graph file changed after writing Ninja file" + fi + +} + + function test_bp2build_bazel_workspace_structure { setup @@ -757,6 +781,7 @@ test_add_file_to_soong_build test_glob_during_bootstrapping test_soong_build_rerun_iff_environment_changes test_dump_json_module_graph +test_json_module_graph_back_and_forth_null_build test_write_to_source_tree test_bp2build_smoke test_bp2build_generates_marker_file diff --git a/ui/build/config.go b/ui/build/config.go index 956406d23..6a05f4fc4 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -747,6 +747,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string { return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker") } +func (c *configImpl) ModuleGraphFile() string { + return shared.JoinPath(c.SoongOutDir(), "module-graph.json") +} + func (c *configImpl) TempDir() string { return shared.TempDirForOutDir(c.SoongOutDir()) } @@ -919,7 +923,7 @@ func (c *configImpl) bazelBuildMode() bazelBuildMode { return mixedBuild } else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") { return generateBuildFiles - } else if v, ok := c.Environment().Get("SOONG_DUMP_JSON_MODULE_GRAPH"); ok && v != "" { + } else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") { return generateJsonModuleGraph } else { return noBazel diff --git a/ui/build/soong.go b/ui/build/soong.go index 06210b9f8..058f8197f 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -117,6 +117,7 @@ func bootstrapBlueprint(ctx Context, config Config) { bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja") bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja") + moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja") // The glob .ninja files are subninja'd. However, they are generated during // the build itself so we write an empty file so that the subninja doesn't @@ -181,9 +182,27 @@ func bootstrapBlueprint(ctx Context, config Config) { Outputs: []string{config.Bp2BuildMarkerFile()}, Args: bp2buildArgs, } + + moduleGraphArgs := []string{ + "--module_graph_file", config.ModuleGraphFile(), + "--globListDir", "globs.modulegraph", + "--globFile", moduleGraphGlobFile, + } + + moduleGraphArgs = append(moduleGraphArgs, commonArgs...) + moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...) + moduleGraphArgs = append(moduleGraphArgs, "Android.bp") + + moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{ + Inputs: []string{"Android.bp"}, + Outputs: []string{config.ModuleGraphFile()}, + Args: moduleGraphArgs, + } + args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{ bp2buildInvocation, mainSoongBuildInvocation, + moduleGraphInvocation, } blueprintCtx := blueprint.NewContext() @@ -307,6 +326,8 @@ func runSoong(ctx Context, config Config) { if config.bazelBuildMode() == generateBuildFiles { target = config.Bp2BuildMarkerFile() + } else if config.bazelBuildMode() == generateJsonModuleGraph { + target = config.ModuleGraphFile() } else { // This build generates /build.ninja, which is used later by build/soong/ui/build/build.go#Build(). target = config.MainNinjaFile()