Allow choosing which soong_build invocations to run in the debugger

When `SOONG_DELVE_STEPS` enviromnment variable is set in addition
to `SOONG_DELVE`, only given soong_build invocations ('steps') are
run in the debugger.
Also, make bootstrapBlueprint code table-driven.

Test: treehugger
Change-Id: Ia4016240ca9e88c2a85c6d06851c5bba30e7b2b5

README.md

@@ -609,15 +609,15 @@ To load the code of Soong in IntelliJ:
   Content Root, then add the `build/blueprint` directory.
 * Optional: also add the `external/golang-protobuf` directory. In practice,
   IntelliJ seems to work well enough without this, too.
+
 ### Running Soong in a debugger
 
-To make `soong_build` wait for a debugger connection, install `dlv` and then
+Both the Android build driver (`soong_ui`) and Soong proper (`soong_build`) are
-start the build with `SOONG_DELVE=<listen addr>` in the environment.
+Go applications and can be debugged with the help of the standard Go debugger
-For example:
+called Delve. A client (e.g., IntelliJ IDEA) communicates with Delve via IP port
-```bash
+that Delve listens to (the port number is passed to it on invocation).
-SOONG_DELVE=5006 m nothing
-```
 
+#### Debugging Android Build Driver ####
 To make `soong_ui` wait for a debugger connection, use the `SOONG_UI_DELVE`
 variable:
 
@@ -625,11 +625,28 @@ variable:
 SOONG_UI_DELVE=5006 m nothing
 ```
 
+#### Debugging Soong Proper ####
 
-setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
+To make `soong_build` wait for a debugger connection, install `dlv` and then
+start the build with `SOONG_DELVE=<listen addr>` in the environment.
+For example:
+```bash
+SOONG_DELVE=5006 m nothing
+```
+Android build driver invokes `soong_build` multiple times, and by default each
+invocation is run in the debugger. Setting `SOONG_DELVE_STEPS` controls which
+invocations are run in the debugger, e.g., running
+```bash
+SOONG_DELVE=2345 SOONG_DELVE_STEPS='build,modulegraph' m
+```
+results in only `build` (main build step) and `modulegraph` being run in the debugger.
+The allowed step names are `api_bp2build`, `bp2build_files`, `bp2build_workspace`,
+`build`, `modulegraph`, `queryview`, `soong_docs`.
+
+Note setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
 is because in order to debug the binary, it needs to be built with debug
 symbols.
-
+#### Delve Troubleshooting ####
 To test the debugger connection, run this command:
 
 ```
@@ -648,15 +665,23 @@ using:
 sudo sysctl -w kernel.yama.ptrace_scope=0
 ```
 
+#### IntelliJ Setup ####
 To connect to the process using IntelliJ:
 
 * Run -> Edit Configurations...
 * Choose "Go Remote" on the left
 * Click on the "+" buttion on the top-left
-* Give it a nice name and set "Host" to localhost and "Port" to the port in the
+* Give it a nice _name_ and set "Host" to `localhost` and "Port" to the port in the
-  environment variable
+  environment variable (`SOONG_UI_DELVE` for `soong_ui`, `SOONG_DELVE` for
+  `soong_build`)
+* Set the breakpoints where you want application to stop
+* Run the build from the command line
+* In IntelliJ, click Run -> Debug _name_
+* Observe _Connecting..._ message in the debugger pane. It changes to
+  _Connected_ once the communication with the debugger has been established; the
+  terminal window where the build started will display
+  `API server listening at ...` message
 
-Debugging works far worse than debugging Java, but is sometimes useful.
 
 Sometimes the `dlv` process hangs on connection. A symptom of this is `dlv`
 spinning a core or two. In that case, `kill -9` `dlv` and try again.

ui/build/soong.go

@@ -18,7 +18,6 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -56,13 +55,13 @@ const (
 	bootstrapEpoch = 1
 )
 
-func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
+func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
 	data, err := shared.EnvFileContents(envDeps)
 	if err != nil {
 		return err
 	}
 
-	return ioutil.WriteFile(envFile, data, 0644)
+	return os.WriteFile(envFile, data, 0644)
 }
 
 // This uses Android.bp files and various tools to generate <builddir>/build.ninja.
@@ -141,7 +140,7 @@ func writeEmptyFile(ctx Context, path string) {
 	if exists, err := fileExists(path); err != nil {
 		ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
 	} else if !exists {
-		err = ioutil.WriteFile(path, nil, 0666)
+		err = os.WriteFile(path, nil, 0666)
 		if err != nil {
 			ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
 		}
@@ -157,24 +156,28 @@ func fileExists(path string) (bool, error) {
 	return true, nil
 }
 
-func primaryBuilderInvocation(
+type PrimaryBuilderFactory struct {
-	config Config,
+	name         string
-	name string,
+	description  string
-	output string,
+	config       Config
-	specificArgs []string,
+	output       string
-	description string) bootstrap.PrimaryBuilderInvocation {
+	specificArgs []string
+	debugPort    string
+}
+
+func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
 	commonArgs := make([]string, 0, 0)
 
-	if !config.skipSoongTests {
+	if !pb.config.skipSoongTests {
 		commonArgs = append(commonArgs, "-t")
 	}
 
-	commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list"))
+	commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
 	invocationEnv := make(map[string]string)
-	if os.Getenv("SOONG_DELVE") != "" {
+	if pb.debugPort != "" {
 		//debug mode
-		commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE"))
+		commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
-		commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary())
+			"--delve_path", shared.ResolveDelveBinary())
 		// GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
 		// is useful because the preemption happens by sending SIGURG to the OS
 		// thread hosting the goroutine in question and each signal results in
@@ -188,26 +191,26 @@ func primaryBuilderInvocation(
 	}
 
 	var allArgs []string
-	allArgs = append(allArgs, specificArgs...)
+	allArgs = append(allArgs, pb.specificArgs...)
 	allArgs = append(allArgs,
-		"--globListDir", name,
+		"--globListDir", pb.name,
-		"--globFile", config.NamedGlobFile(name))
+		"--globFile", pb.config.NamedGlobFile(pb.name))
 
 	allArgs = append(allArgs, commonArgs...)
-	allArgs = append(allArgs, environmentArgs(config, name)...)
+	allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
 	if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
-		allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+name)
+		allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
 	}
 	if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
-		allArgs = append(allArgs, "--memprofile", profileMem+"."+name)
+		allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
 	}
 	allArgs = append(allArgs, "Android.bp")
 
 	return bootstrap.PrimaryBuilderInvocation{
 		Inputs:      []string{"Android.bp"},
-		Outputs:     []string{output},
+		Outputs:     []string{pb.output},
 		Args:        allArgs,
-		Description: description,
+		Description: pb.description,
 		// NB: Changing the value of this environment variable will not result in a
 		// rebuild. The bootstrap Ninja file will change, but apparently Ninja does
 		// not consider changing the pool specified in a statement a change that's
@@ -270,90 +273,117 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	if config.bazelStagingMode {
 		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
 	}
+	queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
+	// The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
+	// The final workspace will be generated in out/soong/api_bp2build
+	apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
 
-	mainSoongBuildInvocation := primaryBuilderInvocation(
+	pbfs := []PrimaryBuilderFactory{
-		config,
+		{
-		soongBuildTag,
+			name:         soongBuildTag,
-		config.SoongNinjaFile(),
+			description:  fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
-		mainSoongBuildExtraArgs,
+			config:       config,
-		fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
+			output:       config.SoongNinjaFile(),
-	)
+			specificArgs: mainSoongBuildExtraArgs,
+		},
+		{
+			name:         bp2buildFilesTag,
+			description:  fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
+			config:       config,
+			output:       config.Bp2BuildFilesMarkerFile(),
+			specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()},
+		},
+		{
+			name:         bp2buildWorkspaceTag,
+			description:  "Creating Bazel symlink forest",
+			config:       config,
+			output:       config.Bp2BuildWorkspaceMarkerFile(),
+			specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()},
+		},
+		{
+			name:        jsonModuleGraphTag,
+			description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
+			config:      config,
+			output:      config.ModuleGraphFile(),
+			specificArgs: []string{
+				"--module_graph_file", config.ModuleGraphFile(),
+				"--module_actions_file", config.ModuleActionsFile(),
+			},
+		},
+		{
+			name:         queryviewTag,
+			description:  fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
+			config:       config,
+			output:       config.QueryviewMarkerFile(),
+			specificArgs: []string{"--bazel_queryview_dir", queryviewDir},
+		},
+		{
+			name:         apiBp2buildTag,
+			description:  fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
+			config:       config,
+			output:       config.ApiBp2buildMarkerFile(),
+			specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir},
+		},
+		{
+			name:         soongDocsTag,
+			description:  fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
+			config:       config,
+			output:       config.SoongDocsHtml(),
+			specificArgs: []string{"--soong_docs", config.SoongDocsHtml()},
+		},
+	}
 
+	// Figure out which invocations will be run under the debugger:
+	//   * SOONG_DELVE if set specifies listening port
+	//   * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
+	debuggedInvocations := make(map[string]bool)
+	delvePort := os.Getenv("SOONG_DELVE")
+	if delvePort != "" {
+		if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
+			var validSteps []string
+			for _, pbf := range pbfs {
+				debuggedInvocations[pbf.name] = false
+				validSteps = append(validSteps, pbf.name)
+
+			}
+			for _, step := range strings.Split(steps, ",") {
+				if _, ok := debuggedInvocations[step]; ok {
+					debuggedInvocations[step] = true
+				} else {
+					ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
+						"Valid steps are %v", step, validSteps)
+				}
+			}
+		} else {
+			//  SOONG_DELVE_STEPS is not set, run all steps in the debugger
+			for _, pbf := range pbfs {
+				debuggedInvocations[pbf.name] = true
+			}
+		}
+	}
+
+	var invocations []bootstrap.PrimaryBuilderInvocation
+	for _, pbf := range pbfs {
+		if debuggedInvocations[pbf.name] {
+			pbf.debugPort = delvePort
+		}
+		pbi := pbf.primaryBuilderInvocation()
+		// Some invocations require adjustment:
+		switch pbf.name {
+		case soongBuildTag:
 			if config.BazelBuildEnabled() {
 				// Mixed builds call Bazel from soong_build and they therefore need the
 				// Bazel workspace to be available. Make that so by adding a dependency on
 				// the bp2build marker file to the action that invokes soong_build .
-		mainSoongBuildInvocation.OrderOnlyInputs = append(mainSoongBuildInvocation.OrderOnlyInputs,
+				pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
-			config.Bp2BuildWorkspaceMarkerFile())
 			}
-
+		case bp2buildWorkspaceTag:
-	bp2buildInvocation := primaryBuilderInvocation(
+			pbi.Inputs = append(pbi.Inputs,
-		config,
-		bp2buildFilesTag,
 				config.Bp2BuildFilesMarkerFile(),
-		[]string{
+				filepath.Join(config.FileListDir(), "bazel.list"))
-			"--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
+		}
-		},
+		invocations = append(invocations, pbi)
-		fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
+	}
-	)
-
-	bp2buildWorkspaceInvocation := primaryBuilderInvocation(
-		config,
-		bp2buildWorkspaceTag,
-		config.Bp2BuildWorkspaceMarkerFile(),
-		[]string{
-			"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
-		},
-		fmt.Sprintf("Creating Bazel symlink forest"),
-	)
-
-	bp2buildWorkspaceInvocation.Inputs = append(bp2buildWorkspaceInvocation.Inputs,
-		config.Bp2BuildFilesMarkerFile(), filepath.Join(config.FileListDir(), "bazel.list"))
-
-	jsonModuleGraphInvocation := primaryBuilderInvocation(
-		config,
-		jsonModuleGraphTag,
-		config.ModuleGraphFile(),
-		[]string{
-			"--module_graph_file", config.ModuleGraphFile(),
-			"--module_actions_file", config.ModuleActionsFile(),
-		},
-		fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
-	)
-
-	queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
-	queryviewInvocation := primaryBuilderInvocation(
-		config,
-		queryviewTag,
-		config.QueryviewMarkerFile(),
-		[]string{
-			"--bazel_queryview_dir", queryviewDir,
-		},
-		fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
-	)
-
-	// The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
-	// The final workspace will be generated in out/soong/api_bp2build
-	apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
-	apiBp2buildInvocation := primaryBuilderInvocation(
-		config,
-		apiBp2buildTag,
-		config.ApiBp2buildMarkerFile(),
-		[]string{
-			"--bazel_api_bp2build_dir", apiBp2buildDir,
-		},
-		fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
-	)
-
-	soongDocsInvocation := primaryBuilderInvocation(
-		config,
-		soongDocsTag,
-		config.SoongDocsHtml(),
-		[]string{
-			"--soong_docs", config.SoongDocsHtml(),
-		},
-		fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
-	)
 
 	// The glob .ninja files are subninja'd. However, they are generated during
 	// the build itself so we write an empty file if the file does not exist yet
@@ -362,11 +392,11 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		writeEmptyFile(ctx, globFile)
 	}
 
-	var blueprintArgs bootstrap.Args
+	blueprintArgs := bootstrap.Args{
-
+		ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
-	blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
+		OutFile:        shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
-	blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja")
+		EmptyNinjaFile: false,
-	blueprintArgs.EmptyNinjaFile = false
+	}
 
 	blueprintCtx := blueprint.NewContext()
 	blueprintCtx.SetIgnoreUnknownModuleTypes(true)
@@ -376,16 +406,9 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		outDir:      config.OutDir(),
 		runGoTests:  !config.skipSoongTests,
 		// If we want to debug soong_build, we need to compile it for debugging
-		debugCompilation: os.Getenv("SOONG_DELVE") != "",
+		debugCompilation:          delvePort != "",
 		subninjas:                 bootstrapGlobFileList(config),
-		primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{
+		primaryBuilderInvocations: invocations,
-			mainSoongBuildInvocation,
-			bp2buildInvocation,
-			bp2buildWorkspaceInvocation,
-			jsonModuleGraphInvocation,
-			queryviewInvocation,
-			apiBp2buildInvocation,
-			soongDocsInvocation},
 	}
 
 	// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little