Invoke soong_docs from the bootstrap Ninja file.

This makes soong_ui the only place where soong_build is invoked, thus
greatly simplifying the conceptual model of the build.

It comes with the slight limitation that now soong_docs (and queryview
and the JSON module graph) are not Make targets anymore, but I suppose
that's an acceptable loss.

The only place where someone depended on soong_docs from a Makefile is
removed in a separate change.

Test: Presubmits.
Change-Id: I3f9ac327725c15d84de725d05e3cdde1da3dcbe2

README.md

@@ -33,8 +33,10 @@ cc_binary {
 Every module must have a `name` property, and the value must be unique across
 all Android.bp files.
 
-For a list of valid module types and their properties see
+The list of valid module types and their properties can be generated by calling
-[$OUT_DIR/soong/docs/soong_build.html](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html).
+`m soong_docs`. It will be written to `$OUT_DIR/soong/docs/soong_build.html`.
+This list for the current version of Soong can be found [here](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html).
+
 
 ### File lists
 

android/Android.bp

@@ -80,7 +80,6 @@ bootstrap_go_package {
         "util.go",
         "variable.go",
         "visibility.go",
-        "writedocs.go",
     ],
     testSrcs: [
         "android_test.go",

android/writedocs.go

@@ -1,89 +0,0 @@
-// Copyright 2015 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package android
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/google/blueprint"
-)
-
-func init() {
-	RegisterSingletonType("writedocs", DocsSingleton)
-}
-
-func DocsSingleton() Singleton {
-	return &docsSingleton{}
-}
-
-type docsSingleton struct{}
-
-func primaryBuilderPath(ctx SingletonContext) Path {
-	soongOutDir := absolutePath(ctx.Config().SoongOutDir())
-	binary := absolutePath(os.Args[0])
-	primaryBuilder, err := filepath.Rel(soongOutDir, binary)
-	if err != nil {
-		ctx.Errorf("path to primary builder %q is not in build dir %q (%q)",
-			os.Args[0], ctx.Config().SoongOutDir(), err)
-	}
-
-	return PathForOutput(ctx, primaryBuilder)
-}
-
-func (c *docsSingleton) GenerateBuildActions(ctx SingletonContext) {
-	var deps Paths
-	deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().moduleListFile))
-	deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName))
-
-	// The dexpreopt configuration may not exist, but if it does, it's a dependency
-	// of soong_build.
-	dexpreoptConfigPath := ctx.Config().DexpreoptGlobalConfigPath(ctx)
-	if dexpreoptConfigPath.Valid() {
-		deps = append(deps, dexpreoptConfigPath.Path())
-	}
-
-	// Generate build system docs for the primary builder.  Generating docs reads the source
-	// files used to build the primary builder, but that dependency will be picked up through
-	// the dependency on the primary builder itself.  There are no dependencies on the
-	// Blueprints files, as any relevant changes to the Blueprints files would have caused
-	// a rebuild of the primary builder.
-	docsFile := PathForOutput(ctx, "docs", "soong_build.html")
-	primaryBuilder := primaryBuilderPath(ctx)
-	soongDocs := ctx.Rule(pctx, "soongDocs",
-		blueprint.RuleParams{
-			Command: fmt.Sprintf("rm -f ${outDir}/* && %s --soong_docs %s %s",
-				primaryBuilder.String(),
-				docsFile.String(),
-				"\""+strings.Join(os.Args[1:], "\" \"")+"\""),
-			CommandDeps: []string{primaryBuilder.String()},
-			Description: fmt.Sprintf("%s docs $out", primaryBuilder.Base()),
-		},
-		"outDir")
-
-	ctx.Build(pctx, BuildParams{
-		Rule:   soongDocs,
-		Output: docsFile,
-		Inputs: deps,
-		Args: map[string]string{
-			"outDir": PathForOutput(ctx, "docs").String(),
-		},
-	})
-
-	// Add a phony target for building the documentation
-	ctx.Phony("soong_docs", docsFile)
-}

cmd/soong_build/main.go

@@ -163,16 +163,6 @@ func runQueryView(queryviewDir, queryviewMarker string, configuration android.Co
 	touch(shared.JoinPath(topDir, queryviewMarker))
 }
 
-func runSoongDocs(configuration android.Config) {
-	ctx := newContext(configuration)
-	soongDocsArgs := cmdlineArgs
-	bootstrap.RunBlueprint(soongDocsArgs, bootstrap.StopBeforePrepareBuildActions, ctx.Context, configuration)
-	if err := writeDocs(ctx, configuration, docFile); err != nil {
-		fmt.Fprintf(os.Stderr, "%s", err)
-		os.Exit(1)
-	}
-}
-
 func writeMetrics(configuration android.Config) {
 	metricsFile := filepath.Join(configuration.SoongOutDir(), "soong_build_metrics.pb")
 	err := android.WriteMetrics(configuration, metricsFile)
@@ -217,20 +207,22 @@ func writeDepFile(outputFile string, ninjaDeps []string) {
 // or the actual Soong build for the build.ninja file. Returns the top level
 // output file of the specific activity.
 func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string {
-	bazelConversionRequested := bp2buildMarker != ""
 	mixedModeBuild := configuration.BazelContext.BazelEnabled()
+	generateBazelWorkspace := bp2buildMarker != ""
 	generateQueryView := bazelQueryViewDir != ""
+	generateModuleGraphFile := moduleGraphFile != ""
+	generateDocFile := docFile != ""
 
 	blueprintArgs := cmdlineArgs
 
 	var stopBefore bootstrap.StopBefore
-	if !generateQueryView && moduleGraphFile == "" {
+	if !generateModuleGraphFile && !generateQueryView && !generateDocFile {
 		stopBefore = bootstrap.DoEverything
 	} else {
 		stopBefore = bootstrap.StopBeforePrepareBuildActions
 	}
 
-	if bazelConversionRequested {
+	if generateBazelWorkspace {
 		// Run the alternate pipeline of bp2build mutators and singleton to convert
 		// Blueprint to BUILD files before everything else.
 		runBp2Build(configuration, extraNinjaDeps)
@@ -253,10 +245,20 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
 			runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx)
 			writeDepFile(queryviewMarkerFile, ninjaDeps)
 			return queryviewMarkerFile
-		} else if moduleGraphFile != "" {
+		} else if generateModuleGraphFile {
 			writeJsonModuleGraph(ctx, moduleGraphFile)
 			writeDepFile(moduleGraphFile, ninjaDeps)
 			return moduleGraphFile
+		} else if generateDocFile {
+			// TODO: we could make writeDocs() return the list of documentation files
+			// written and add them to the .d file. Then soong_docs would be re-run
+			// whenever one is deleted.
+			if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil {
+				fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err)
+				os.Exit(1)
+			}
+			writeDepFile(docFile, ninjaDeps)
+			return docFile
 		} else {
 			// The actual output (build.ninja) was written in the RunBlueprint() call
 			// above
@@ -320,16 +322,6 @@ func main() {
 		extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
 	}
 
-	if docFile != "" {
-		// We don't write an used variables file when generating documentation
-		// because that is done from within the actual builds as a Ninja action and
-		// thus it would overwrite the actual used variables file so this is
-		// special-cased.
-		// TODO: Fix this by not passing --used_env to the soong_docs invocation
-		runSoongDocs(configuration)
-		return
-	}
-
 	finalOutputFile := doChosenActivity(configuration, extraNinjaDeps)
 	writeUsedEnvironmentFile(configuration, finalOutputFile)
 }

cmd/soong_build/writedocs.go

@@ -15,13 +15,14 @@
 package main
 
 import (
-	"android/soong/android"
 	"bytes"
 	"html/template"
 	"io/ioutil"
 	"path/filepath"
 	"sort"
 
+	"android/soong/android"
+
 	"github.com/google/blueprint/bootstrap"
 	"github.com/google/blueprint/bootstrap/bpdoc"
 )
@@ -95,13 +96,13 @@ func moduleTypeDocsToTemplates(moduleTypeList []*bpdoc.ModuleType) []moduleTypeT
 	return result
 }
 
-func getPackages(ctx *android.Context, config interface{}) ([]*bpdoc.Package, error) {
+func getPackages(ctx *android.Context) ([]*bpdoc.Package, error) {
 	moduleTypeFactories := android.ModuleTypeFactoriesForDocs()
-	return bootstrap.ModuleTypeDocs(ctx.Context, config, moduleTypeFactories)
+	return bootstrap.ModuleTypeDocs(ctx.Context, moduleTypeFactories)
 }
 
-func writeDocs(ctx *android.Context, config interface{}, filename string) error {
+func writeDocs(ctx *android.Context, filename string) error {
-	packages, err := getPackages(ctx, config)
+	packages, err := getPackages(ctx)
 	if err != nil {
 		return err
 	}

tests/bootstrap_test.sh

@@ -472,17 +472,35 @@ EOF
   fi
 }
 
-function test_null_build_after_docs {
+function test_soong_docs_smoke() {
   setup
-  run_soong
-  local mtime1=$(stat -c "%y" out/soong/build.ninja)
 
-  prebuilts/build-tools/linux-x86/bin/ninja -f out/combined.ninja soong_docs
+  run_soong soong_docs
+
+  [[ -e "out/soong/docs/soong_build.html" ]] || fail "Documentation for main page not created"
+  [[ -e "out/soong/docs/cc.html" ]] || fail "Documentation for C++ modules not created"
+}
+
+function test_null_build_after_soong_docs() {
+  setup
 
   run_soong
-  local mtime2=$(stat -c "%y" out/soong/build.ninja)
+  local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
 
-  if [[ "$mtime1" != "$mtime2" ]]; then
+  run_soong soong_docs
+  local docs_mtime1=$(stat -c "%y" out/soong/docs/soong_build.html)
+
+  run_soong soong_docs
+  local docs_mtime2=$(stat -c "%y" out/soong/docs/soong_build.html)
+
+  if [[ "$docs_mtime1" != "$docs_mtime2" ]]; then
+    fail "Output Ninja file changed on null build"
+  fi
+
+  run_soong
+  local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja)
+
+  if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then
     fail "Output Ninja file changed on null build"
   fi
 }
@@ -809,7 +827,8 @@ function test_queryview_null_build() {
 
 test_smoke
 test_null_build
-test_null_build_after_docs
+test_soong_docs_smoke
+test_null_build_after_soong_docs
 test_soong_build_rebuilt_if_blueprint_changes
 test_glob_noop_incremental
 test_add_file_to_glob

ui/build/config.go

@@ -50,6 +50,7 @@ type configImpl struct {
 	jsonModuleGraph bool
 	bp2build        bool
 	queryview       bool
+	soongDocs       bool
 	skipConfig      bool
 	skipKati        bool
 	skipKatiNinja   bool
@@ -646,6 +647,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
 			c.bp2build = true
 		} else if arg == "queryview" {
 			c.queryview = true
+		} else if arg == "soong_docs" {
+			c.soongDocs = true
 		} else {
 			if arg == "checkbuild" {
 				c.checkbuild = true
@@ -723,7 +726,7 @@ func (c *configImpl) SoongBuildInvocationNeeded() bool {
 		return true
 	}
 
-	if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() {
+	if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() {
 		// Command line was empty, the default Ninja target is built
 		return true
 	}
@@ -788,6 +791,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string {
 	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
 }
 
+func (c *configImpl) SoongDocsHtml() string {
+	return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
+}
+
 func (c *configImpl) QueryviewMarkerFile() string {
 	return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
 }
@@ -833,6 +840,10 @@ func (c *configImpl) Queryview() bool {
 	return c.queryview
 }
 
+func (c *configImpl) SoongDocs() bool {
+	return c.soongDocs
+}
+
 func (c *configImpl) IsVerbose() bool {
 	return c.verbose
 }

ui/build/soong.go

@@ -143,6 +143,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
 	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
 	queryviewGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.queryview.ninja")
+	soongDocsGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.soong_docs.ninja")
 	moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")
 
 	// The glob .ninja files are subninja'd. However, they are generated during
@@ -151,6 +152,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	writeEmptyGlobFile(ctx, bootstrapGlobFile)
 	writeEmptyGlobFile(ctx, bp2buildGlobFile)
 	writeEmptyGlobFile(ctx, queryviewGlobFile)
+	writeEmptyGlobFile(ctx, soongDocsGlobFile)
 	writeEmptyGlobFile(ctx, moduleGraphGlobFile)
 
 	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
@@ -164,7 +166,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
 	// Building soong_build does not require a glob file
 	// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
-	args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile}
+	args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile, soongDocsGlobFile}
 	args.EmptyNinjaFile = config.EmptyNinjaFile()
 
 	args.DelveListen = os.Getenv("SOONG_DELVE")
@@ -226,6 +228,22 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		Args:    queryviewArgs,
 	}
 
+	soongDocsArgs := []string{
+		"--soong_docs", config.SoongDocsHtml(),
+		"--globListDir", "soong_docs",
+		"--globFile", soongDocsGlobFile,
+	}
+
+	soongDocsArgs = append(soongDocsArgs, commonArgs...)
+	soongDocsArgs = append(soongDocsArgs, environmentArgs(config, ".soong_docs")...)
+	soongDocsArgs = append(soongDocsArgs, "Android.bp")
+
+	soongDocsInvocation := bootstrap.PrimaryBuilderInvocation{
+		Inputs:  []string{"Android.bp"},
+		Outputs: []string{config.SoongDocsHtml()},
+		Args:    soongDocsArgs,
+	}
+
 	moduleGraphArgs := []string{
 		"--module_graph_file", config.ModuleGraphFile(),
 		"--globListDir", "modulegraph",
@@ -247,6 +265,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		mainSoongBuildInvocation,
 		moduleGraphInvocation,
 		queryviewInvocation,
+		soongDocsInvocation,
 	}
 
 	blueprintCtx := blueprint.NewContext()
@@ -386,6 +405,10 @@ func runSoong(ctx Context, config Config) {
 		targets = append(targets, config.QueryviewMarkerFile())
 	}
 
+	if config.SoongDocs() {
+		targets = append(targets, config.SoongDocsHtml())
+	}
+
 	if config.SoongBuildInvocationNeeded() {
 		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
 		targets = append(targets, config.MainNinjaFile())