StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make running soong_build in alternate modes nicer.

Browse Source 
This includes the JSON graph generator and bp2build.

Before:
GENERATE_BAZEL_FILES=1 m nothing
GENERATE_JSON_MODULE_GRAPH=1 m nothing

Now:
m json-module-graph
m bp2build

They can now also be combined with other targets or each other.

The longer-term goal is to run "m queryview" and "m soong_docs" using
the same infrastructure. There are two alternate approaches:

1. Call soong_build from within the main Ninja invocation. This requires
two sequential soong_build invocations and is thus slower.
2. Do everything requested in the same soong_build invocation. This
would be faster, but one AFAIU can't tell Ninja that multiple possible
actions can build the same output so that doesn't work.

(1) is somewhat more desirable because soong_docs seems to be built
from build/make/core/main.mk ; I assume that that can be worked around
although I haven't checked where the output of "m soong_docs" goes.

Test: Presubmits.
Change-Id: If5ba36490d9f3f60733e6d6be9286eb2b67c3ff5
This commit is contained in:
Lukacs T. Berki
2021-09-02 17:23:06 +02:00
parent bd59c1490d
commit a1b9372ef7
6 changed files with 114 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
tests/bootstrap_test.sh
View File

@@ -522,7 +522,7 @@ EOF
function test_bp2build_smoke {
setup
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
[[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created"
[[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
}
@@ -531,7 +531,7 @@ function test_bp2build_generates_marker_file {
setup
create_mock_bazel
run_bp2build
run_soong bp2build
if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
fail "Marker file was not generated"
@@ -551,7 +551,7 @@ filegroup {
}
EOF
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
[[ -e out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not created"
[[ -L out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not symlinked"
@@ -565,7 +565,7 @@ filegroup {
}
EOF
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
[[ -e out/soong/bp2build/b/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not created"
[[ -L out/soong/workspace/b/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not symlinked"
}
@@ -573,10 +573,10 @@ EOF
function test_bp2build_null_build {
setup
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$mtime1" != "$mtime2" ]]; then
@@ -597,18 +597,35 @@ filegroup {
}
EOF
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
grep -q a1.txt "out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME}" || fail "a1.txt not in ${GENERATED_BUILD_FILE_NAME} file"
touch a/a2.txt
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
grep -q a2.txt "out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME}" || fail "a2.txt not in ${GENERATED_BUILD_FILE_NAME} file"
}
function test_multiple_soong_build_modes() {
setup
run_soong json-module-graph bp2build nothing
if [[ ! -f "out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
fail "bp2build marker file was not generated"
fi
if [[ ! -f "out/soong/module-graph.json" ]]; then
fail "JSON file was not created"
fi
if [[ ! -f "out/soong/build.ninja" ]]; then
fail "Main build.ninja file was not created"
fi
}
function test_dump_json_module_graph() {
setup
GENERATE_JSON_MODULE_GRAPH=1 run_soong
if [[ ! -r "out/soong//module-graph.json" ]]; then
run_soong json-module-graph
if [[ ! -r "out/soong/module-graph.json" ]]; then
fail "JSON file was not created"
fi
}
@@ -619,7 +636,7 @@ function test_json_module_graph_back_and_forth_null_build() {
run_soong
local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
GENERATE_JSON_MODULE_GRAPH=1 run_soong
run_soong json-module-graph
local json_mtime1=$(stat -c "%y" out/soong/module-graph.json)
run_soong
@@ -628,7 +645,7 @@ function test_json_module_graph_back_and_forth_null_build() {
fail "Output Ninja file changed after writing JSON module graph"
fi
GENERATE_JSON_MODULE_GRAPH=1 run_soong
run_soong json-module-graph
local json_mtime2=$(stat -c "%y" out/soong/module-graph.json)
if [[ "$json_mtime1" != "$json_mtime2" ]]; then
fail "JSON module graph file changed after writing Ninja file"
@@ -651,7 +668,7 @@ filegroup {
}
EOF
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
[[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
[[ -d out/soong/workspace/a/b ]] || fail "module directory not a directory"
[[ -L "out/soong/workspace/a/b/${GENERATED_BUILD_FILE_NAME}" ]] || fail "${GENERATED_BUILD_FILE_NAME} file not symlinked"
@@ -675,10 +692,10 @@ filegroup {
}
EOF
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
touch a/a2.txt # No reference in the .bp file needed
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
[[ -L out/soong/workspace/a/a2.txt ]] || fail "a/a2.txt not symlinked"
}
@@ -696,7 +713,7 @@ filegroup {
}
EOF
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
[[ -L "out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME}" ]] || fail "${GENERATED_BUILD_FILE_NAME} file not symlinked"
[[ "$(readlink -f out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME})" =~ "bp2build/a/${GENERATED_BUILD_FILE_NAME}"$ ]] \
|| fail "${GENERATED_BUILD_FILE_NAME} files symlinked to the wrong place"
@@ -725,7 +742,7 @@ filegroup {
}
EOF
if GENERATE_BAZEL_FILES=1 run_soong >& "$MOCK_TOP/errors"; then
if run_soong bp2build >& "$MOCK_TOP/errors"; then
fail "Build should have failed"
fi
@@ -739,7 +756,7 @@ function test_bp2build_back_and_forth_null_build {
run_soong
local output_mtime1=$(stat -c "%y" out/soong/build.ninja)
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
if [[ "$output_mtime1" != "$output_mtime2" ]]; then
fail "Output Ninja file changed when switching to bp2build"
@@ -757,7 +774,7 @@ function test_bp2build_back_and_forth_null_build {
fail "bp2build marker file changed when switching to regular build from bp2build"
fi
GENERATE_BAZEL_FILES=1 run_soong
run_soong bp2build
local output_mtime4=$(stat -c "%y" out/soong/build.ninja)
local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$output_mtime1" != "$output_mtime4" ]]; then
@@ -780,6 +797,7 @@ test_delete_android_bp
test_add_file_to_soong_build
test_glob_during_bootstrapping
test_soong_build_rerun_iff_environment_changes
test_multiple_soong_build_modes
test_dump_json_module_graph
test_json_module_graph_back_and_forth_null_build
test_write_to_source_tree

10
tests/bp2build_bazel_test.sh
View File

@@ -10,10 +10,10 @@ readonly GENERATED_BUILD_FILE_NAME="BUILD.bazel"
function test_bp2build_null_build() {
setup
run_bp2build
run_soong bp2build
local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
run_bp2build
run_soong bp2build
local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$output_mtime1" != "$output_mtime2" ]]; then
@@ -35,10 +35,10 @@ filegroup {
EOF
touch foo/bar/a.txt foo/bar/b.txt
run_bp2build
run_soong bp2build
local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
run_bp2build
run_soong bp2build
local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$output_mtime1" != "$output_mtime2" ]]; then
@@ -80,7 +80,7 @@ genrule {
}
EOF
run_bp2build
run_soong bp2build
if [[ ! -f "./out/soong/workspace/foo/convertible_soong_module/${GENERATED_BUILD_FILE_NAME}" ]]; then
fail "./out/soong/workspace/foo/convertible_soong_module/${GENERATED_BUILD_FILE_NAME} was not generated"

4
tests/lib.sh
View File

@@ -124,10 +124,6 @@ run_bazel() {
tools/bazel "$@"
}
run_bp2build() {
GENERATE_BAZEL_FILES=true build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests nothing
}
run_ninja() {
build/soong/soong_ui.bash --make-mode --skip-make --skip-soong-tests "$@"
}

20
ui/build/build.go
View File

@@ -248,6 +248,16 @@ func Build(ctx Context, config Config) {
what = what &^ RunNinja
}
if !config.SoongBuildInvocationNeeded() {
// This means that the output of soong_build is not needed and thus it would
// run unnecessarily. In addition, if this code wasn't there invocations
// with only special-cased target names like "m bp2build" would result in
// passing Ninja the empty target list and it would then build the default
// targets which is not what the user asked for.
what = what &^ RunNinja
what = what &^ RunKati
}
if config.StartGoma() {
startGoma(ctx, config)
}
@@ -278,16 +288,6 @@ func Build(ctx Context, config Config) {
if what&RunSoong != 0 {
runSoong(ctx, config)
if config.bazelBuildMode() == generateBuildFiles {
// Return early, if we're using Soong as solely the generator of BUILD files.
return
}
if config.bazelBuildMode() == generateJsonModuleGraph {
// Return early, if we're using Soong as solely the generator of the JSON module graph
return
}
}
if what&RunKati != 0 {

66
ui/build/config.go
View File

@@ -33,7 +33,8 @@ import (
type Config struct{ *configImpl }
type configImpl struct {
// From the environment
// Some targets that are implemented in soong_build
// (bp2build, json-module-graph) are not here and have their own bits below.
arguments []string
goma bool
environ *Environment
@@ -41,17 +42,19 @@ type configImpl struct {
buildDateTime string
// From the arguments
parallel int
keepGoing int
verbose bool
checkbuild bool
dist bool
skipConfig bool
skipKati bool
skipKatiNinja bool
skipSoong bool
skipNinja bool
skipSoongTests bool
parallel int
keepGoing int
verbose bool
checkbuild bool
dist bool
jsonModuleGraph bool
bp2build bool
skipConfig bool
skipKati bool
skipKatiNinja bool
skipSoong bool
skipNinja bool
skipSoongTests bool
// From the product config
katiArgs []string
@@ -109,9 +112,6 @@ const (
// Only generate build files (in a subdirectory of the out directory) and exit.
generateBuildFiles
// Only generate the Soong json module graph for use with jq, and exit.
generateJsonModuleGraph
// Generate synthetic build files and incorporate these files into a build which
// partially uses Bazel. Build metadata may come from Android.bp or BUILD files.
mixedBuild
@@ -639,6 +639,10 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
c.environ.Set(k, v)
} else if arg == "dist" {
c.dist = true
} else if arg == "json-module-graph" {
c.jsonModuleGraph = true
} else if arg == "bp2build" {
c.bp2build = true
} else {
if arg == "checkbuild" {
c.checkbuild = true
@@ -705,6 +709,26 @@ func (c *configImpl) Arguments() []string {
return c.arguments
}
func (c *configImpl) SoongBuildInvocationNeeded() bool {
if c.Dist() {
return true
}
if len(c.Arguments()) > 0 {
// Explicit targets requested that are not special targets like b2pbuild
// or the JSON module graph
return true
}
if !c.JsonModuleGraph() && !c.Bp2Build() {
// Command line was empty, the default Ninja target is built
return true
}
// build.ninja doesn't need to be generated
return false
}
func (c *configImpl) OutDir() string {
if outDir, ok := c.environ.Get("OUT_DIR"); ok {
return outDir
@@ -790,6 +814,14 @@ func (c *configImpl) Dist() bool {
return c.dist
}
func (c *configImpl) JsonModuleGraph() bool {
return c.jsonModuleGraph
}
func (c *configImpl) Bp2Build() bool {
return c.bp2build
}
func (c *configImpl) IsVerbose() bool {
return c.verbose
}
@@ -935,10 +967,6 @@ func (c *configImpl) UseBazel() bool {
func (c *configImpl) bazelBuildMode() bazelBuildMode {
if c.Environment().IsEnvTrue("USE_BAZEL_ANALYSIS") {
return mixedBuild
} else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") {
return generateBuildFiles
} else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") {
return generateJsonModuleGraph
} else {
return noBazel
}

26
ui/build/soong.go
View File

@@ -275,7 +275,7 @@ func runSoong(ctx Context, config Config) {
}
buildMode := config.bazelBuildMode()
integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
integratedBp2Build := buildMode == mixedBuild
// This is done unconditionally, but does not take a measurable amount of time
bootstrapBlueprint(ctx, config)
@@ -351,18 +351,22 @@ func runSoong(ctx Context, config Config) {
cmd.RunAndStreamOrFatal()
}
var target string
targets := make([]string, 0, 0)
if config.bazelBuildMode() == generateBuildFiles {
target = config.Bp2BuildMarkerFile()
} else if config.bazelBuildMode() == generateJsonModuleGraph {
target = config.ModuleGraphFile()
} else {
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
target = config.MainNinjaFile()
if config.JsonModuleGraph() {
targets = append(targets, config.ModuleGraphFile())
}
ninja("bootstrap", ".bootstrap/build.ninja", target)
if config.Bp2Build() {
targets = append(targets, config.Bp2BuildMarkerFile())
}
if config.SoongBuildInvocationNeeded() {
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
targets = append(targets, config.MainNinjaFile())
}
ninja("bootstrap", ".bootstrap/build.ninja", targets...)
var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
if shouldCollectBuildSoongMetrics(config) {
@@ -404,7 +408,7 @@ func runMicrofactory(ctx Context, config Config, relExePath string, pkg string,
func shouldCollectBuildSoongMetrics(config Config) bool {
// Do not collect metrics protobuf if the soong_build binary ran as the
// bp2build converter or the JSON graph dump.
return config.bazelBuildMode() != generateBuildFiles && config.bazelBuildMode() != generateJsonModuleGraph
return config.SoongBuildInvocationNeeded()
}
func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {