Merge "Print the JSON module graph correctly."

cmd/soong_build/main.go

@@ -25,6 +25,7 @@ import (
 
 	"android/soong/bp2build"
 	"android/soong/shared"
+
 	"github.com/google/blueprint/bootstrap"
 	"github.com/google/blueprint/deptools"
 	"github.com/google/blueprint/pathtools"
@@ -43,6 +44,7 @@ var (
 	delveListen string
 	delvePath   string
 
+	moduleGraphFile   string
 	docFile           string
 	bazelQueryViewDir string
 	bp2buildMarker    string
@@ -62,6 +64,7 @@ func init() {
 	flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
 
 	// Flags representing various modes soong_build can run in
+	flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output")
 	flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
 	flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
 	flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
@@ -71,7 +74,6 @@ func init() {
 	flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
 	flag.StringVar(&cmdlineArgs.BuildDir, "b", ".", "the build output directory")
 	flag.StringVar(&cmdlineArgs.NinjaBuildDir, "n", "", "the ninja builddir directory")
-	flag.StringVar(&cmdlineArgs.DepFile, "d", "", "the dependency file to output")
 	flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
 	flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
 	flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
@@ -149,11 +151,7 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context,
 	globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration)
 	ninjaDeps = append(ninjaDeps, globListFiles...)
 
-	err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps)
+	writeDepFile(secondArgs.OutFile, ninjaDeps)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err)
-		os.Exit(1)
-	}
 }
 
 // Run the code-generation phase to convert BazelTargetModules to BUILD files.
@@ -185,8 +183,8 @@ func writeMetrics(configuration android.Config) {
 	}
 }
 
-func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) {
+func writeJsonModuleGraph(ctx *android.Context, path string) {
-	f, err := os.Create(path)
+	f, err := os.Create(shared.JoinPath(topDir, path))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err)
 		os.Exit(1)
@@ -194,7 +192,6 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa
 
 	defer f.Close()
 	ctx.Context.PrintJSONGraph(f)
-	writeFakeNinjaFile(extraNinjaDeps, configuration.SoongOutDir())
 }
 
 func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string {
@@ -208,6 +205,15 @@ func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.Mu
 	return bootstrap.GlobFileListFiles(globDir)
 }
 
+func writeDepFile(outputFile string, ninjaDeps []string) {
+	depFile := shared.JoinPath(topDir, outputFile+".d")
+	err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err)
+		os.Exit(1)
+	}
+}
+
 // doChosenActivity runs Soong for a specific activity, like bp2build, queryview
 // or the actual Soong build for the build.ninja file. Returns the top level
 // output file of the specific activity.
@@ -215,10 +221,9 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
 	bazelConversionRequested := bp2buildMarker != ""
 	mixedModeBuild := configuration.BazelContext.BazelEnabled()
 	generateQueryView := bazelQueryViewDir != ""
-	jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
 
 	blueprintArgs := cmdlineArgs
-	prepareBuildActions := !generateQueryView && jsonModuleFile == ""
+	prepareBuildActions := !generateQueryView && moduleGraphFile == ""
 	if bazelConversionRequested {
 		// Run the alternate pipeline of bp2build mutators and singleton to convert
 		// Blueprint to BUILD files before everything else.
@@ -236,24 +241,21 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
 		globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration)
 		ninjaDeps = append(ninjaDeps, globListFiles...)
 
-		err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps)
+		// Convert the Soong module graph into Bazel BUILD files.
-		if err != nil {
+		if generateQueryView {
-			fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err)
+			runQueryView(configuration, ctx)
-			os.Exit(1)
+			return cmdlineArgs.OutFile // TODO: This is a lie
+		} else if moduleGraphFile != "" {
+			writeJsonModuleGraph(ctx, moduleGraphFile)
+			writeDepFile(moduleGraphFile, ninjaDeps)
+			return moduleGraphFile
+		} else {
+			// The actual output (build.ninja) was written in the RunBlueprint() call
+			// above
+			writeDepFile(cmdlineArgs.OutFile, ninjaDeps)
 		}
 	}
 
-	// Convert the Soong module graph into Bazel BUILD files.
-	if generateQueryView {
-		runQueryView(configuration, ctx)
-		return cmdlineArgs.OutFile // TODO: This is a lie
-	}
-
-	if jsonModuleFile != "" {
-		writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
-		return cmdlineArgs.OutFile // TODO: This is a lie
-	}
-
 	writeMetrics(configuration)
 	return cmdlineArgs.OutFile
 }
@@ -348,29 +350,6 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri
 	touch(shared.JoinPath(topDir, finalOutputFile))
 }
 
-// Workarounds to support running bp2build in a clean AOSP checkout with no
-// prior builds, and exiting early as soon as the BUILD files get generated,
-// therefore not creating build.ninja files that soong_ui and callers of
-// soong_build expects.
-//
-// These files are: build.ninja and build.ninja.d. Since Kati hasn't been
-// ran as well, and `nothing` is defined in a .mk file, there isn't a ninja
-// target called `nothing`, so we manually create it here.
-func writeFakeNinjaFile(extraNinjaDeps []string, soongOutDir string) {
-	extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ")
-
-	ninjaFileName := "build.ninja"
-	ninjaFile := shared.JoinPath(topDir, soongOutDir, ninjaFileName)
-	ninjaFileD := shared.JoinPath(topDir, soongOutDir, ninjaFileName+".d")
-	// A workaround to create the 'nothing' ninja target so `m nothing` works,
-	// since bp2build runs without Kati, and the 'nothing' target is declared in
-	// a Makefile.
-	ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n  phony_output = true\n"), 0666)
-	ioutil.WriteFile(ninjaFileD,
-		[]byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFile, extraNinjaDepsString)),
-		0666)
-}
-
 func touch(path string) {
 	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
 	if err != nil {
@@ -550,12 +529,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 	ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
 	ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
 
-	depFile := bp2buildMarker + ".d"
+	writeDepFile(bp2buildMarker, ninjaDeps)
-	err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
-		os.Exit(1)
-	}
 
 	// Create an empty bp2build marker file.
 	touch(shared.JoinPath(topDir, bp2buildMarker))

tests/bootstrap_test.sh

@@ -607,12 +607,36 @@ EOF
 
 function test_dump_json_module_graph() {
   setup
-  SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
+  GENERATE_JSON_MODULE_GRAPH=1 run_soong
-  if [[ ! -r "$MOCK_TOP/modules.json" ]]; then
+  if [[ ! -r "out/soong//module-graph.json" ]]; then
     fail "JSON file was not created"
   fi
 }
 
+function test_json_module_graph_back_and_forth_null_build() {
+  setup
+
+  run_soong
+  local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+  GENERATE_JSON_MODULE_GRAPH=1 run_soong
+  local json_mtime1=$(stat -c "%y" out/soong/module-graph.json)
+
+  run_soong
+  local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja)
+  if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then
+    fail "Output Ninja file changed after writing JSON module graph"
+  fi
+
+  GENERATE_JSON_MODULE_GRAPH=1 run_soong
+  local json_mtime2=$(stat -c "%y" out/soong/module-graph.json)
+  if [[ "$json_mtime1" != "$json_mtime2" ]]; then
+    fail "JSON module graph file changed after writing Ninja file"
+  fi
+
+}
+
+
 function test_bp2build_bazel_workspace_structure {
   setup
 
@@ -757,6 +781,7 @@ test_add_file_to_soong_build
 test_glob_during_bootstrapping
 test_soong_build_rerun_iff_environment_changes
 test_dump_json_module_graph
+test_json_module_graph_back_and_forth_null_build
 test_write_to_source_tree
 test_bp2build_smoke
 test_bp2build_generates_marker_file

ui/build/config.go

@@ -747,6 +747,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string {
 	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
 }
 
+func (c *configImpl) ModuleGraphFile() string {
+	return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
+}
+
 func (c *configImpl) TempDir() string {
 	return shared.TempDirForOutDir(c.SoongOutDir())
 }
@@ -919,7 +923,7 @@ func (c *configImpl) bazelBuildMode() bazelBuildMode {
 		return mixedBuild
 	} else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") {
 		return generateBuildFiles
-	} else if v, ok := c.Environment().Get("SOONG_DUMP_JSON_MODULE_GRAPH"); ok && v != "" {
+	} else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") {
 		return generateJsonModuleGraph
 	} else {
 		return noBazel

ui/build/soong.go

@@ -117,6 +117,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
 
 	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
 	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
+	moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")
 
 	// The glob .ninja files are subninja'd. However, they are generated during
 	// the build itself so we write an empty file so that the subninja doesn't
@@ -181,9 +182,27 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		Outputs: []string{config.Bp2BuildMarkerFile()},
 		Args:    bp2buildArgs,
 	}
+
+	moduleGraphArgs := []string{
+		"--module_graph_file", config.ModuleGraphFile(),
+		"--globListDir", "globs.modulegraph",
+		"--globFile", moduleGraphGlobFile,
+	}
+
+	moduleGraphArgs = append(moduleGraphArgs, commonArgs...)
+	moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...)
+	moduleGraphArgs = append(moduleGraphArgs, "Android.bp")
+
+	moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{
+		Inputs:  []string{"Android.bp"},
+		Outputs: []string{config.ModuleGraphFile()},
+		Args:    moduleGraphArgs,
+	}
+
 	args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
 		bp2buildInvocation,
 		mainSoongBuildInvocation,
+		moduleGraphInvocation,
 	}
 
 	blueprintCtx := blueprint.NewContext()
@@ -307,6 +326,8 @@ func runSoong(ctx Context, config Config) {
 
 	if config.bazelBuildMode() == generateBuildFiles {
 		target = config.Bp2BuildMarkerFile()
+	} else if config.bazelBuildMode() == generateJsonModuleGraph {
+		target = config.ModuleGraphFile()
 	} else {
 		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
 		target = config.MainNinjaFile()