StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Separate the files of bp2build and soong_build.

Browse Source 
The following files are forked:

- build-globs.ninja
- The glob list files
- .d files for the glob list files
- The output file (build.ninja or the bp2build marker file)

This makes bp2build and soong_build not overwrite each other's files
they need for proper incrementality.

Test: Presubmits.

Change-Id: I69d192cbd36ecd9677f46f3fa095dfce6f872227
This commit is contained in:
Lukacs T. Berki
2021-08-12 14:03:55 +02:00
parent 15c82a10de
commit 56ebaf35b3
5 changed files with 107 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/soong_build/main.go
View File

@@ -472,10 +472,10 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
// Generate out/soong/.bootstrap/build-globs.ninja with the actions to generate flattened globfiles
// containing the globs seen during bp2build conversion
if blueprintArgs.GlobFile != "" {
bootstrap.WriteBuildGlobsNinjaFile(bootstrap.StageMain, bp2buildCtx.Context, blueprintArgs, configuration)
bootstrap.WriteBuildGlobsNinjaFile(blueprintArgs.GlobListDir, bp2buildCtx.Context, blueprintArgs, configuration)
}
// Add the depfile on the expanded globs in out/soong/.primary/globs
ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...)
ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration, blueprintArgs.GlobListDir)...)
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
@@ -528,12 +528,4 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
// Create an empty bp2build marker file.
touch(shared.JoinPath(topDir, bp2buildMarker))
// bp2build *always* writes a fake Ninja file containing just the nothing
// phony target if it ever re-runs. This allows bp2build to exit early with
// GENERATE_BAZEL_FILES=1 m nothing.
//
// If bp2build is invoked as part of an integrated mixed build, the fake
// build.ninja file will be rewritten later into the real file anyway.
writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
}

54
tests/bootstrap_test.sh
View File

@@ -144,7 +144,7 @@ EOF
run_soong
local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
local glob_deps_file=out/soong/.primary/globs/0.d
local glob_deps_file=out/soong/.bootstrap/globs/0.d
if [ -e "$glob_deps_file" ]; then
fail "Glob deps file unexpectedly written on first build"
@@ -526,18 +526,14 @@ function test_bp2build_smoke {
[[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
}
function test_bp2build_generates_fake_ninja_file {
function test_bp2build_generates_marker_file {
setup
create_mock_bazel
run_bp2build
if [[ ! -f "./out/soong/build.ninja" ]]; then
fail "./out/soong/build.ninja was not generated"
fi
if ! grep "build nothing: phony" "./out/soong/build.ninja"; then
fail "missing phony nothing target in out/soong/build.ninja"
if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
fail "Marker file was not generated"
fi
}
@@ -577,10 +573,10 @@ function test_bp2build_null_build {
setup
GENERATE_BAZEL_FILES=1 run_soong
local mtime1=$(stat -c "%y" out/soong/build.ninja)
local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
GENERATE_BAZEL_FILES=1 run_soong
local mtime2=$(stat -c "%y" out/soong/build.ninja)
local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$mtime1" != "$mtime2" ]]; then
fail "Output Ninja file changed on null build"
@@ -712,6 +708,41 @@ EOF
grep -q "b/${GENERATED_BUILD_FILE_NAME}' exist" "$MOCK_TOP/errors" || fail "Error for b/${GENERATED_BUILD_FILE_NAME} not found"
}
function test_bp2build_back_and_forth_null_build {
setup
run_soong
local output_mtime1=$(stat -c "%y" out/soong/build.ninja)
GENERATE_BAZEL_FILES=1 run_soong
local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
if [[ "$output_mtime1" != "$output_mtime2" ]]; then
fail "Output Ninja file changed when switching to bp2build"
fi
local marker_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
run_soong
local output_mtime3=$(stat -c "%y" out/soong/build.ninja)
local marker_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$output_mtime1" != "$output_mtime3" ]]; then
fail "Output Ninja file changed when switching to regular build from bp2build"
fi
if [[ "$marker_mtime1" != "$marker_mtime2" ]]; then
fail "bp2build marker file changed when switching to regular build from bp2build"
fi
GENERATE_BAZEL_FILES=1 run_soong
local output_mtime4=$(stat -c "%y" out/soong/build.ninja)
local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$output_mtime1" != "$output_mtime4" ]]; then
fail "Output Ninja file changed when switching back to bp2build"
fi
if [[ "$marker_mtime1" != "$marker_mtime3" ]]; then
fail "bp2build marker file changed when switching back to bp2build"
fi
}
test_smoke
test_null_build
test_null_build_after_docs
@@ -727,8 +758,9 @@ test_soong_build_rerun_iff_environment_changes
test_dump_json_module_graph
test_write_to_source_tree
test_bp2build_smoke
test_bp2build_generates_fake_ninja_file
test_bp2build_generates_marker_file
test_bp2build_null_build
test_bp2build_back_and_forth_null_build
test_bp2build_add_android_bp
test_bp2build_add_to_glob
test_bp2build_bazel_workspace_structure

54
tests/bp2build_bazel_test.sh
View File

@@ -48,60 +48,6 @@ EOF
test_bp2build_null_build_with_globs
function test_soong_after_bp2build_regenerates_build_globs_ninja() {
setup
mkdir -p foo/bar
cat > foo/bar/Android.bp <<'EOF'
filegroup {
name: "bp2build-files",
srcs: ["bp2build.*"],
bazel_module: { bp2build_available: true },
}
filegroup {
name: "soong-files",
srcs: ["soong.*"],
}
EOF
touch foo/bar/bp2build.txt foo/bar/soong.txt
build_globs_file="out/soong/.bootstrap/build-globs.ninja"
# Test: the build-globs file for bp2build should only contain the bp2build-files
# glob, whereas the build-globs file for soong should contain both bp2build-files
# and soong-files globs.
run_bp2build
local output_mtime1=$(stat -c "%y" "${build_globs_file}")
if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
fail "bp2build filegroup globs not found in bp2build's globs file"
fi
if grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
fail "soong filegroup globs unexpectedly found in bp2build's globs file"
fi
run_soong
local output_mtime2=$(stat -c "%y" "${build_globs_file}")
if [[ "$output_mtime1" == "$output_mtime2" ]]; then
fail "Output build-globs.ninja file did not change"
fi
if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
fail "bp2build filegroup globs not found in bp2build's globs file"
fi
if ! grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
fail "soong filegroup globs not found in bp2build's globs file"
fi
}
test_soong_after_bp2build_regenerates_build_globs_ninja
function test_bp2build_generates_all_buildfiles {
setup
create_mock_bazel

8
ui/build/config.go
View File

@@ -739,6 +739,14 @@ func (c *configImpl) SoongOutDir() string {
return filepath.Join(c.OutDir(), "soong")
}
func (c *configImpl) MainNinjaFile() string {
return shared.JoinPath(c.SoongOutDir(), "build.ninja")
}
func (c *configImpl) Bp2BuildMarkerFile() string {
return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
}
func (c *configImpl) TempDir() string {
return shared.TempDirForOutDir(c.SoongOutDir())
}

72
ui/build/soong.go
View File

@@ -114,17 +114,20 @@ func writeEmptyGlobFile(ctx Context, path string) {
}
}
func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
var args bootstrap.Args
mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
// .bootstrap/build.ninja "includes" .bootstrap/build-globs.ninja for incremental builds
// generate an empty glob before running any rule in .bootstrap/build.ninja
bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
// The glob .ninja files are subninja'd. However, they are generated during
// the build itself so we write an empty file so that the subninja doesn't
// fail on clean builds
writeEmptyGlobFile(ctx, bootstrapGlobFile)
writeEmptyGlobFile(ctx, bp2buildGlobFile)
bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
args.RunGoTests = !config.skipSoongTests
@@ -137,7 +140,7 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
// Building soong_build does not require a glob file
// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
args.GlobFile = ""
args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile}
args.GeneratingPrimaryBuilder = true
args.EmptyNinjaFile = config.EmptyNinjaFile()
@@ -146,42 +149,46 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
args.DelvePath = shared.ResolveDelveBinary()
}
commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())
mainSoongBuildInputs := []string{"Android.bp"}
if integratedBp2Build {
mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
if config.bazelBuildMode() == mixedBuild {
mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())
}
soongBuildArgs := []string{
"--globListDir", "globs",
"--globFile", bootstrapGlobFile,
}
soongBuildArgs := make([]string, 0)
soongBuildArgs = append(soongBuildArgs, commonArgs...)
soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
soongBuildArgs = append(soongBuildArgs, "Android.bp")
mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
Inputs: mainSoongBuildInputs,
Outputs: []string{mainNinjaFile},
Outputs: []string{config.MainNinjaFile()},
Args: soongBuildArgs,
}
if integratedBp2Build {
bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
bp2buildArgs := []string{
"--bp2build_marker", config.Bp2BuildMarkerFile(),
"--globListDir", "globs.bp2build",
"--globFile", bp2buildGlobFile,
}
bp2buildArgs = append(bp2buildArgs, commonArgs...)
bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
bp2buildArgs = append(bp2buildArgs, "Android.bp")
bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
Inputs: []string{"Android.bp"},
Outputs: []string{bp2BuildMarkerFile},
Outputs: []string{config.Bp2BuildMarkerFile()},
Args: bp2buildArgs,
}
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{bp2buildInvocation}
if config.bazelBuildMode() == mixedBuild {
args.PrimaryBuilderInvocations = append(args.PrimaryBuilderInvocations, mainSoongBuildInvocation)
}
} else {
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
bp2buildInvocation,
mainSoongBuildInvocation,
}
blueprintCtx := blueprint.NewContext()
@@ -231,7 +238,7 @@ func runSoong(ctx Context, config Config) {
integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
// This is done unconditionally, but does not take a measurable amount of time
bootstrapBlueprint(ctx, config, integratedBp2Build)
bootstrapBlueprint(ctx, config)
soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -268,7 +275,7 @@ func runSoong(ctx Context, config Config) {
runMicrofactory(ctx, config, ".minibootstrap/bpglob", "github.com/google/blueprint/bootstrap/bpglob",
map[string]string{"github.com/google/blueprint": "build/blueprint"})
ninja := func(name, file string) {
ninja := func(name, ninjaFile string, targets ...string) {
ctx.BeginTrace(metrics.RunSoong, name)
defer ctx.EndTrace()
@@ -276,8 +283,7 @@ func runSoong(ctx Context, config Config) {
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
defer nr.Close()
cmd := Command(ctx, config, "soong "+name,
config.PrebuiltBuildTool("ninja"),
ninjaArgs := []string{
"-d", "keepdepfile",
"-d", "stats",
"-o", "usesphonyoutputs=yes",
@@ -287,7 +293,12 @@ func runSoong(ctx Context, config Config) {
"-w", "missingoutfile=err",
"-j", strconv.Itoa(config.Parallel()),
"--frontend_file", fifo,
"-f", filepath.Join(config.SoongOutDir(), file))
"-f", filepath.Join(config.SoongOutDir(), ninjaFile),
}
ninjaArgs = append(ninjaArgs, targets...)
cmd := Command(ctx, config, "soong "+name,
config.PrebuiltBuildTool("ninja"), ninjaArgs...)
var ninjaEnv Environment
@@ -299,8 +310,17 @@ func runSoong(ctx Context, config Config) {
cmd.Sandbox = soongSandbox
cmd.RunAndStreamOrFatal()
}
var target string
if config.bazelBuildMode() == generateBuildFiles {
target = config.Bp2BuildMarkerFile()
} else {
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
ninja("bootstrap", ".bootstrap/build.ninja")
target = config.MainNinjaFile()
}
ninja("bootstrap", ".bootstrap/build.ninja", target)
var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
if shouldCollectBuildSoongMetrics(config) {