From c6012f36e133effe2c4f54b96b46425ea5c0cb9b Mon Sep 17 00:00:00 2001 From: "Lukacs T. Berki" Date: Mon, 6 Sep 2021 18:31:46 +0200 Subject: [PATCH] Invoke soong_docs from the bootstrap Ninja file. This makes soong_ui the only place where soong_build is invoked, thus greatly simplifying the conceptual model of the build. It comes with the slight limitation that now soong_docs (and queryview and the JSON module graph) are not Make targets anymore, but I suppose that's an acceptable loss. The only place where someone depended on soong_docs from a Makefile is removed in a separate change. Test: Presubmits. Change-Id: I3f9ac327725c15d84de725d05e3cdde1da3dcbe2 --- README.md | 6 ++- android/Android.bp | 1 - android/writedocs.go | 89 ------------------------------------ cmd/soong_build/main.go | 40 +++++++--------- cmd/soong_build/writedocs.go | 11 +++-- tests/bootstrap_test.sh | 33 ++++++++++--- ui/build/config.go | 13 +++++- ui/build/soong.go | 25 +++++++++- 8 files changed, 88 insertions(+), 130 deletions(-) delete mode 100644 android/writedocs.go diff --git a/README.md b/README.md index e92349eb7..d8dd26a05 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,10 @@ cc_binary { Every module must have a `name` property, and the value must be unique across all Android.bp files. -For a list of valid module types and their properties see -[$OUT_DIR/soong/docs/soong_build.html](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html). +The list of valid module types and their properties can be generated by calling +`m soong_docs`. It will be written to `$OUT_DIR/soong/docs/soong_build.html`. +This list for the current version of Soong can be found [here](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html). + ### File lists diff --git a/android/Android.bp b/android/Android.bp index 5901ed98f..f3a385025 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -80,7 +80,6 @@ bootstrap_go_package { "util.go", "variable.go", "visibility.go", - "writedocs.go", ], testSrcs: [ "android_test.go", diff --git a/android/writedocs.go b/android/writedocs.go deleted file mode 100644 index c380a3d84..000000000 --- a/android/writedocs.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/google/blueprint" -) - -func init() { - RegisterSingletonType("writedocs", DocsSingleton) -} - -func DocsSingleton() Singleton { - return &docsSingleton{} -} - -type docsSingleton struct{} - -func primaryBuilderPath(ctx SingletonContext) Path { - soongOutDir := absolutePath(ctx.Config().SoongOutDir()) - binary := absolutePath(os.Args[0]) - primaryBuilder, err := filepath.Rel(soongOutDir, binary) - if err != nil { - ctx.Errorf("path to primary builder %q is not in build dir %q (%q)", - os.Args[0], ctx.Config().SoongOutDir(), err) - } - - return PathForOutput(ctx, primaryBuilder) -} - -func (c *docsSingleton) GenerateBuildActions(ctx SingletonContext) { - var deps Paths - deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().moduleListFile)) - deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName)) - - // The dexpreopt configuration may not exist, but if it does, it's a dependency - // of soong_build. - dexpreoptConfigPath := ctx.Config().DexpreoptGlobalConfigPath(ctx) - if dexpreoptConfigPath.Valid() { - deps = append(deps, dexpreoptConfigPath.Path()) - } - - // Generate build system docs for the primary builder. Generating docs reads the source - // files used to build the primary builder, but that dependency will be picked up through - // the dependency on the primary builder itself. There are no dependencies on the - // Blueprints files, as any relevant changes to the Blueprints files would have caused - // a rebuild of the primary builder. - docsFile := PathForOutput(ctx, "docs", "soong_build.html") - primaryBuilder := primaryBuilderPath(ctx) - soongDocs := ctx.Rule(pctx, "soongDocs", - blueprint.RuleParams{ - Command: fmt.Sprintf("rm -f ${outDir}/* && %s --soong_docs %s %s", - primaryBuilder.String(), - docsFile.String(), - "\""+strings.Join(os.Args[1:], "\" \"")+"\""), - CommandDeps: []string{primaryBuilder.String()}, - Description: fmt.Sprintf("%s docs $out", primaryBuilder.Base()), - }, - "outDir") - - ctx.Build(pctx, BuildParams{ - Rule: soongDocs, - Output: docsFile, - Inputs: deps, - Args: map[string]string{ - "outDir": PathForOutput(ctx, "docs").String(), - }, - }) - - // Add a phony target for building the documentation - ctx.Phony("soong_docs", docsFile) -} diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 684160f5b..399efda3a 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -163,16 +163,6 @@ func runQueryView(queryviewDir, queryviewMarker string, configuration android.Co touch(shared.JoinPath(topDir, queryviewMarker)) } -func runSoongDocs(configuration android.Config) { - ctx := newContext(configuration) - soongDocsArgs := cmdlineArgs - bootstrap.RunBlueprint(soongDocsArgs, bootstrap.StopBeforePrepareBuildActions, ctx.Context, configuration) - if err := writeDocs(ctx, configuration, docFile); err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) - } -} - func writeMetrics(configuration android.Config) { metricsFile := filepath.Join(configuration.SoongOutDir(), "soong_build_metrics.pb") err := android.WriteMetrics(configuration, metricsFile) @@ -217,20 +207,22 @@ func writeDepFile(outputFile string, ninjaDeps []string) { // or the actual Soong build for the build.ninja file. Returns the top level // output file of the specific activity. func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string { - bazelConversionRequested := bp2buildMarker != "" mixedModeBuild := configuration.BazelContext.BazelEnabled() + generateBazelWorkspace := bp2buildMarker != "" generateQueryView := bazelQueryViewDir != "" + generateModuleGraphFile := moduleGraphFile != "" + generateDocFile := docFile != "" blueprintArgs := cmdlineArgs var stopBefore bootstrap.StopBefore - if !generateQueryView && moduleGraphFile == "" { + if !generateModuleGraphFile && !generateQueryView && !generateDocFile { stopBefore = bootstrap.DoEverything } else { stopBefore = bootstrap.StopBeforePrepareBuildActions } - if bazelConversionRequested { + if generateBazelWorkspace { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. runBp2Build(configuration, extraNinjaDeps) @@ -253,10 +245,20 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx) writeDepFile(queryviewMarkerFile, ninjaDeps) return queryviewMarkerFile - } else if moduleGraphFile != "" { + } else if generateModuleGraphFile { writeJsonModuleGraph(ctx, moduleGraphFile) writeDepFile(moduleGraphFile, ninjaDeps) return moduleGraphFile + } else if generateDocFile { + // TODO: we could make writeDocs() return the list of documentation files + // written and add them to the .d file. Then soong_docs would be re-run + // whenever one is deleted. + if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil { + fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err) + os.Exit(1) + } + writeDepFile(docFile, ninjaDeps) + return docFile } else { // The actual output (build.ninja) was written in the RunBlueprint() call // above @@ -320,16 +322,6 @@ func main() { extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve")) } - if docFile != "" { - // We don't write an used variables file when generating documentation - // because that is done from within the actual builds as a Ninja action and - // thus it would overwrite the actual used variables file so this is - // special-cased. - // TODO: Fix this by not passing --used_env to the soong_docs invocation - runSoongDocs(configuration) - return - } - finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) writeUsedEnvironmentFile(configuration, finalOutputFile) } diff --git a/cmd/soong_build/writedocs.go b/cmd/soong_build/writedocs.go index b7c260c6a..8d8f37f4b 100644 --- a/cmd/soong_build/writedocs.go +++ b/cmd/soong_build/writedocs.go @@ -15,13 +15,14 @@ package main import ( - "android/soong/android" "bytes" "html/template" "io/ioutil" "path/filepath" "sort" + "android/soong/android" + "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/bootstrap/bpdoc" ) @@ -95,13 +96,13 @@ func moduleTypeDocsToTemplates(moduleTypeList []*bpdoc.ModuleType) []moduleTypeT return result } -func getPackages(ctx *android.Context, config interface{}) ([]*bpdoc.Package, error) { +func getPackages(ctx *android.Context) ([]*bpdoc.Package, error) { moduleTypeFactories := android.ModuleTypeFactoriesForDocs() - return bootstrap.ModuleTypeDocs(ctx.Context, config, moduleTypeFactories) + return bootstrap.ModuleTypeDocs(ctx.Context, moduleTypeFactories) } -func writeDocs(ctx *android.Context, config interface{}, filename string) error { - packages, err := getPackages(ctx, config) +func writeDocs(ctx *android.Context, filename string) error { + packages, err := getPackages(ctx) if err != nil { return err } diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 95a193a37..a22adc5aa 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -472,17 +472,35 @@ EOF fi } -function test_null_build_after_docs { +function test_soong_docs_smoke() { setup - run_soong - local mtime1=$(stat -c "%y" out/soong/build.ninja) - prebuilts/build-tools/linux-x86/bin/ninja -f out/combined.ninja soong_docs + run_soong soong_docs + + [[ -e "out/soong/docs/soong_build.html" ]] || fail "Documentation for main page not created" + [[ -e "out/soong/docs/cc.html" ]] || fail "Documentation for C++ modules not created" +} + +function test_null_build_after_soong_docs() { + setup run_soong - local mtime2=$(stat -c "%y" out/soong/build.ninja) + local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja) - if [[ "$mtime1" != "$mtime2" ]]; then + run_soong soong_docs + local docs_mtime1=$(stat -c "%y" out/soong/docs/soong_build.html) + + run_soong soong_docs + local docs_mtime2=$(stat -c "%y" out/soong/docs/soong_build.html) + + if [[ "$docs_mtime1" != "$docs_mtime2" ]]; then + fail "Output Ninja file changed on null build" + fi + + run_soong + local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja) + + if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then fail "Output Ninja file changed on null build" fi } @@ -809,7 +827,8 @@ function test_queryview_null_build() { test_smoke test_null_build -test_null_build_after_docs +test_soong_docs_smoke +test_null_build_after_soong_docs test_soong_build_rebuilt_if_blueprint_changes test_glob_noop_incremental test_add_file_to_glob diff --git a/ui/build/config.go b/ui/build/config.go index 2cd7d55b6..d5d03c30e 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -50,6 +50,7 @@ type configImpl struct { jsonModuleGraph bool bp2build bool queryview bool + soongDocs bool skipConfig bool skipKati bool skipKatiNinja bool @@ -646,6 +647,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { c.bp2build = true } else if arg == "queryview" { c.queryview = true + } else if arg == "soong_docs" { + c.soongDocs = true } else { if arg == "checkbuild" { c.checkbuild = true @@ -723,7 +726,7 @@ func (c *configImpl) SoongBuildInvocationNeeded() bool { return true } - if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() { + if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() { // Command line was empty, the default Ninja target is built return true } @@ -788,6 +791,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string { return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker") } +func (c *configImpl) SoongDocsHtml() string { + return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") +} + func (c *configImpl) QueryviewMarkerFile() string { return shared.JoinPath(c.SoongOutDir(), "queryview.marker") } @@ -833,6 +840,10 @@ func (c *configImpl) Queryview() bool { return c.queryview } +func (c *configImpl) SoongDocs() bool { + return c.soongDocs +} + func (c *configImpl) IsVerbose() bool { return c.verbose } diff --git a/ui/build/soong.go b/ui/build/soong.go index 26afd43ea..ed3af1843 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -143,6 +143,7 @@ func bootstrapBlueprint(ctx Context, config Config) { bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja") bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja") queryviewGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.queryview.ninja") + soongDocsGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.soong_docs.ninja") moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja") // The glob .ninja files are subninja'd. However, they are generated during @@ -151,6 +152,7 @@ func bootstrapBlueprint(ctx Context, config Config) { writeEmptyGlobFile(ctx, bootstrapGlobFile) writeEmptyGlobFile(ctx, bp2buildGlobFile) writeEmptyGlobFile(ctx, queryviewGlobFile) + writeEmptyGlobFile(ctx, soongDocsGlobFile) writeEmptyGlobFile(ctx, moduleGraphGlobFile) bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") @@ -164,7 +166,7 @@ func bootstrapBlueprint(ctx Context, config Config) { // The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d) // Building soong_build does not require a glob file // Using "" instead of ".ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary - args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile} + args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile, soongDocsGlobFile} args.EmptyNinjaFile = config.EmptyNinjaFile() args.DelveListen = os.Getenv("SOONG_DELVE") @@ -226,6 +228,22 @@ func bootstrapBlueprint(ctx Context, config Config) { Args: queryviewArgs, } + soongDocsArgs := []string{ + "--soong_docs", config.SoongDocsHtml(), + "--globListDir", "soong_docs", + "--globFile", soongDocsGlobFile, + } + + soongDocsArgs = append(soongDocsArgs, commonArgs...) + soongDocsArgs = append(soongDocsArgs, environmentArgs(config, ".soong_docs")...) + soongDocsArgs = append(soongDocsArgs, "Android.bp") + + soongDocsInvocation := bootstrap.PrimaryBuilderInvocation{ + Inputs: []string{"Android.bp"}, + Outputs: []string{config.SoongDocsHtml()}, + Args: soongDocsArgs, + } + moduleGraphArgs := []string{ "--module_graph_file", config.ModuleGraphFile(), "--globListDir", "modulegraph", @@ -247,6 +265,7 @@ func bootstrapBlueprint(ctx Context, config Config) { mainSoongBuildInvocation, moduleGraphInvocation, queryviewInvocation, + soongDocsInvocation, } blueprintCtx := blueprint.NewContext() @@ -386,6 +405,10 @@ func runSoong(ctx Context, config Config) { targets = append(targets, config.QueryviewMarkerFile()) } + if config.SoongDocs() { + targets = append(targets, config.SoongDocsHtml()) + } + if config.SoongBuildInvocationNeeded() { // This build generates /build.ninja, which is used later by build/soong/ui/build/build.go#Build(). targets = append(targets, config.MainNinjaFile())