StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Print the JSON module graph correctly."

Browse Source
This commit is contained in:
Lukács T. Berki
2021-08-27 19:39:24 +00:00
committed by Gerrit Code Review
parent 560cb6685c e571dc3bd0
commit b8e5cb0511
4 changed files with 82 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
cmd/soong_build/main.go
View File

@@ -25,6 +25,7 @@ import (
"android/soong/bp2build"
"android/soong/shared"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools"
"github.com/google/blueprint/pathtools"
@@ -43,6 +44,7 @@ var (
delveListen string
delvePath string
moduleGraphFile string
docFile string
bazelQueryViewDir string
bp2buildMarker string
@@ -62,6 +64,7 @@ func init() {
flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
// Flags representing various modes soong_build can run in
flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output")
flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
@@ -71,7 +74,6 @@ func init() {
flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
flag.StringVar(&cmdlineArgs.BuildDir, "b", ".", "the build output directory")
flag.StringVar(&cmdlineArgs.NinjaBuildDir, "n", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.DepFile, "d", "", "the dependency file to output")
flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
@@ -149,11 +151,7 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context,
globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration)
ninjaDeps = append(ninjaDeps, globListFiles...)
err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps)
if err != nil {
fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err)
os.Exit(1)
}
writeDepFile(secondArgs.OutFile, ninjaDeps)
}
// Run the code-generation phase to convert BazelTargetModules to BUILD files.
@@ -185,8 +183,8 @@ func writeMetrics(configuration android.Config) {
}
}
func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) {
f, err := os.Create(path)
func writeJsonModuleGraph(ctx *android.Context, path string) {
f, err := os.Create(shared.JoinPath(topDir, path))
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
@@ -194,7 +192,6 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa
defer f.Close()
ctx.Context.PrintJSONGraph(f)
writeFakeNinjaFile(extraNinjaDeps, configuration.SoongOutDir())
}
func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string {
@@ -208,6 +205,15 @@ func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.Mu
return bootstrap.GlobFileListFiles(globDir)
}
func writeDepFile(outputFile string, ninjaDeps []string) {
depFile := shared.JoinPath(topDir, outputFile+".d")
err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
if err != nil {
fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err)
os.Exit(1)
}
}
// doChosenActivity runs Soong for a specific activity, like bp2build, queryview
// or the actual Soong build for the build.ninja file. Returns the top level
// output file of the specific activity.
@@ -215,10 +221,9 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
bazelConversionRequested := bp2buildMarker != ""
mixedModeBuild := configuration.BazelContext.BazelEnabled()
generateQueryView := bazelQueryViewDir != ""
jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
blueprintArgs := cmdlineArgs
prepareBuildActions := !generateQueryView && jsonModuleFile == ""
prepareBuildActions := !generateQueryView && moduleGraphFile == ""
if bazelConversionRequested {
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
@@ -236,24 +241,21 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration)
ninjaDeps = append(ninjaDeps, globListFiles...)
err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps)
if err != nil {
fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err)
os.Exit(1)
// Convert the Soong module graph into Bazel BUILD files.
if generateQueryView {
runQueryView(configuration, ctx)
return cmdlineArgs.OutFile // TODO: This is a lie
} else if moduleGraphFile != "" {
writeJsonModuleGraph(ctx, moduleGraphFile)
writeDepFile(moduleGraphFile, ninjaDeps)
return moduleGraphFile
} else {
// The actual output (build.ninja) was written in the RunBlueprint() call
// above
writeDepFile(cmdlineArgs.OutFile, ninjaDeps)
}
}
// Convert the Soong module graph into Bazel BUILD files.
if generateQueryView {
runQueryView(configuration, ctx)
return cmdlineArgs.OutFile // TODO: This is a lie
}
if jsonModuleFile != "" {
writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
return cmdlineArgs.OutFile // TODO: This is a lie
}
writeMetrics(configuration)
return cmdlineArgs.OutFile
}
@@ -348,29 +350,6 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri
touch(shared.JoinPath(topDir, finalOutputFile))
}
// Workarounds to support running bp2build in a clean AOSP checkout with no
// prior builds, and exiting early as soon as the BUILD files get generated,
// therefore not creating build.ninja files that soong_ui and callers of
// soong_build expects.
//
// These files are: build.ninja and build.ninja.d. Since Kati hasn't been
// ran as well, and `nothing` is defined in a .mk file, there isn't a ninja
// target called `nothing`, so we manually create it here.
func writeFakeNinjaFile(extraNinjaDeps []string, soongOutDir string) {
extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ")
ninjaFileName := "build.ninja"
ninjaFile := shared.JoinPath(topDir, soongOutDir, ninjaFileName)
ninjaFileD := shared.JoinPath(topDir, soongOutDir, ninjaFileName+".d")
// A workaround to create the 'nothing' ninja target so `m nothing` works,
// since bp2build runs without Kati, and the 'nothing' target is declared in
// a Makefile.
ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n phony_output = true\n"), 0666)
ioutil.WriteFile(ninjaFileD,
[]byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFile, extraNinjaDepsString)),
0666)
}
func touch(path string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
@@ -550,12 +529,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
depFile := bp2buildMarker + ".d"
err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
os.Exit(1)
}
writeDepFile(bp2buildMarker, ninjaDeps)
// Create an empty bp2build marker file.
touch(shared.JoinPath(topDir, bp2buildMarker))

29
tests/bootstrap_test.sh
View File

@@ -607,12 +607,36 @@ EOF
function test_dump_json_module_graph() {
setup
SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
if [[ ! -r "$MOCK_TOP/modules.json" ]]; then
GENERATE_JSON_MODULE_GRAPH=1 run_soong
if [[ ! -r "out/soong//module-graph.json" ]]; then
fail "JSON file was not created"
fi
}
function test_json_module_graph_back_and_forth_null_build() {
setup
run_soong
local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
GENERATE_JSON_MODULE_GRAPH=1 run_soong
local json_mtime1=$(stat -c "%y" out/soong/module-graph.json)
run_soong
local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja)
if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then
fail "Output Ninja file changed after writing JSON module graph"
fi
GENERATE_JSON_MODULE_GRAPH=1 run_soong
local json_mtime2=$(stat -c "%y" out/soong/module-graph.json)
if [[ "$json_mtime1" != "$json_mtime2" ]]; then
fail "JSON module graph file changed after writing Ninja file"
fi
}
function test_bp2build_bazel_workspace_structure {
setup
@@ -757,6 +781,7 @@ test_add_file_to_soong_build
test_glob_during_bootstrapping
test_soong_build_rerun_iff_environment_changes
test_dump_json_module_graph
test_json_module_graph_back_and_forth_null_build
test_write_to_source_tree
test_bp2build_smoke
test_bp2build_generates_marker_file

6
ui/build/config.go
View File

@@ -747,6 +747,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string {
return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
}
func (c *configImpl) ModuleGraphFile() string {
return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
}
func (c *configImpl) TempDir() string {
return shared.TempDirForOutDir(c.SoongOutDir())
}
@@ -919,7 +923,7 @@ func (c *configImpl) bazelBuildMode() bazelBuildMode {
return mixedBuild
} else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") {
return generateBuildFiles
} else if v, ok := c.Environment().Get("SOONG_DUMP_JSON_MODULE_GRAPH"); ok && v != "" {
} else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") {
return generateJsonModuleGraph
} else {
return noBazel

21
ui/build/soong.go
View File

@@ -117,6 +117,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")
// The glob .ninja files are subninja'd. However, they are generated during
// the build itself so we write an empty file so that the subninja doesn't
@@ -181,9 +182,27 @@ func bootstrapBlueprint(ctx Context, config Config) {
Outputs: []string{config.Bp2BuildMarkerFile()},
Args: bp2buildArgs,
}
moduleGraphArgs := []string{
"--module_graph_file", config.ModuleGraphFile(),
"--globListDir", "globs.modulegraph",
"--globFile", moduleGraphGlobFile,
}
moduleGraphArgs = append(moduleGraphArgs, commonArgs...)
moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...)
moduleGraphArgs = append(moduleGraphArgs, "Android.bp")
moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{
Inputs: []string{"Android.bp"},
Outputs: []string{config.ModuleGraphFile()},
Args: moduleGraphArgs,
}
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
bp2buildInvocation,
mainSoongBuildInvocation,
moduleGraphInvocation,
}
blueprintCtx := blueprint.NewContext()
@@ -307,6 +326,8 @@ func runSoong(ctx Context, config Config) {
if config.bazelBuildMode() == generateBuildFiles {
target = config.Bp2BuildMarkerFile()
} else if config.bazelBuildMode() == generateJsonModuleGraph {
target = config.ModuleGraphFile()
} else {
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
target = config.MainNinjaFile()