diff --git a/README.md b/README.md index 18cf7b2ad..7f1846351 100644 --- a/README.md +++ b/README.md @@ -609,15 +609,15 @@ To load the code of Soong in IntelliJ: Content Root, then add the `build/blueprint` directory. * Optional: also add the `external/golang-protobuf` directory. In practice, IntelliJ seems to work well enough without this, too. + ### Running Soong in a debugger -To make `soong_build` wait for a debugger connection, install `dlv` and then -start the build with `SOONG_DELVE=` in the environment. -For example: -```bash -SOONG_DELVE=5006 m nothing -``` +Both the Android build driver (`soong_ui`) and Soong proper (`soong_build`) are +Go applications and can be debugged with the help of the standard Go debugger +called Delve. A client (e.g., IntelliJ IDEA) communicates with Delve via IP port +that Delve listens to (the port number is passed to it on invocation). +#### Debugging Android Build Driver #### To make `soong_ui` wait for a debugger connection, use the `SOONG_UI_DELVE` variable: @@ -625,11 +625,28 @@ variable: SOONG_UI_DELVE=5006 m nothing ``` +#### Debugging Soong Proper #### -setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This +To make `soong_build` wait for a debugger connection, install `dlv` and then +start the build with `SOONG_DELVE=` in the environment. +For example: +```bash +SOONG_DELVE=5006 m nothing +``` +Android build driver invokes `soong_build` multiple times, and by default each +invocation is run in the debugger. Setting `SOONG_DELVE_STEPS` controls which +invocations are run in the debugger, e.g., running +```bash +SOONG_DELVE=2345 SOONG_DELVE_STEPS='build,modulegraph' m +``` +results in only `build` (main build step) and `modulegraph` being run in the debugger. +The allowed step names are `api_bp2build`, `bp2build_files`, `bp2build_workspace`, +`build`, `modulegraph`, `queryview`, `soong_docs`. + +Note setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This is because in order to debug the binary, it needs to be built with debug symbols. - +#### Delve Troubleshooting #### To test the debugger connection, run this command: ``` @@ -648,15 +665,23 @@ using: sudo sysctl -w kernel.yama.ptrace_scope=0 ``` +#### IntelliJ Setup #### To connect to the process using IntelliJ: * Run -> Edit Configurations... * Choose "Go Remote" on the left * Click on the "+" buttion on the top-left -* Give it a nice name and set "Host" to localhost and "Port" to the port in the - environment variable +* Give it a nice _name_ and set "Host" to `localhost` and "Port" to the port in the + environment variable (`SOONG_UI_DELVE` for `soong_ui`, `SOONG_DELVE` for + `soong_build`) +* Set the breakpoints where you want application to stop +* Run the build from the command line +* In IntelliJ, click Run -> Debug _name_ +* Observe _Connecting..._ message in the debugger pane. It changes to + _Connected_ once the communication with the debugger has been established; the + terminal window where the build started will display + `API server listening at ...` message -Debugging works far worse than debugging Java, but is sometimes useful. Sometimes the `dlv` process hangs on connection. A symptom of this is `dlv` spinning a core or two. In that case, `kill -9` `dlv` and try again. diff --git a/ui/build/soong.go b/ui/build/soong.go index abaf5aecd..837f0a4f4 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" "io/fs" - "io/ioutil" "os" "path/filepath" "strconv" @@ -56,13 +55,13 @@ const ( bootstrapEpoch = 1 ) -func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error { +func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error { data, err := shared.EnvFileContents(envDeps) if err != nil { return err } - return ioutil.WriteFile(envFile, data, 0644) + return os.WriteFile(envFile, data, 0644) } // This uses Android.bp files and various tools to generate /build.ninja. @@ -141,7 +140,7 @@ func writeEmptyFile(ctx Context, path string) { if exists, err := fileExists(path); err != nil { ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err) } else if !exists { - err = ioutil.WriteFile(path, nil, 0666) + err = os.WriteFile(path, nil, 0666) if err != nil { ctx.Fatalf("Failed to create empty file '%s': %s", path, err) } @@ -157,24 +156,28 @@ func fileExists(path string) (bool, error) { return true, nil } -func primaryBuilderInvocation( - config Config, - name string, - output string, - specificArgs []string, - description string) bootstrap.PrimaryBuilderInvocation { +type PrimaryBuilderFactory struct { + name string + description string + config Config + output string + specificArgs []string + debugPort string +} + +func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation { commonArgs := make([]string, 0, 0) - if !config.skipSoongTests { + if !pb.config.skipSoongTests { commonArgs = append(commonArgs, "-t") } - commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list")) + commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list")) invocationEnv := make(map[string]string) - if os.Getenv("SOONG_DELVE") != "" { + if pb.debugPort != "" { //debug mode - commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE")) - commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary()) + commonArgs = append(commonArgs, "--delve_listen", pb.debugPort, + "--delve_path", shared.ResolveDelveBinary()) // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This // is useful because the preemption happens by sending SIGURG to the OS // thread hosting the goroutine in question and each signal results in @@ -188,26 +191,26 @@ func primaryBuilderInvocation( } var allArgs []string - allArgs = append(allArgs, specificArgs...) + allArgs = append(allArgs, pb.specificArgs...) allArgs = append(allArgs, - "--globListDir", name, - "--globFile", config.NamedGlobFile(name)) + "--globListDir", pb.name, + "--globFile", pb.config.NamedGlobFile(pb.name)) allArgs = append(allArgs, commonArgs...) - allArgs = append(allArgs, environmentArgs(config, name)...) + allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...) if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" { - allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+name) + allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name) } if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" { - allArgs = append(allArgs, "--memprofile", profileMem+"."+name) + allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name) } allArgs = append(allArgs, "Android.bp") return bootstrap.PrimaryBuilderInvocation{ Inputs: []string{"Android.bp"}, - Outputs: []string{output}, + Outputs: []string{pb.output}, Args: allArgs, - Description: description, + Description: pb.description, // NB: Changing the value of this environment variable will not result in a // rebuild. The bootstrap Ninja file will change, but apparently Ninja does // not consider changing the pool specified in a statement a change that's @@ -270,90 +273,117 @@ func bootstrapBlueprint(ctx Context, config Config) { if config.bazelStagingMode { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging") } - - mainSoongBuildInvocation := primaryBuilderInvocation( - config, - soongBuildTag, - config.SoongNinjaFile(), - mainSoongBuildExtraArgs, - fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()), - ) - - if config.BazelBuildEnabled() { - // Mixed builds call Bazel from soong_build and they therefore need the - // Bazel workspace to be available. Make that so by adding a dependency on - // the bp2build marker file to the action that invokes soong_build . - mainSoongBuildInvocation.OrderOnlyInputs = append(mainSoongBuildInvocation.OrderOnlyInputs, - config.Bp2BuildWorkspaceMarkerFile()) - } - - bp2buildInvocation := primaryBuilderInvocation( - config, - bp2buildFilesTag, - config.Bp2BuildFilesMarkerFile(), - []string{ - "--bp2build_marker", config.Bp2BuildFilesMarkerFile(), - }, - fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()), - ) - - bp2buildWorkspaceInvocation := primaryBuilderInvocation( - config, - bp2buildWorkspaceTag, - config.Bp2BuildWorkspaceMarkerFile(), - []string{ - "--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(), - }, - fmt.Sprintf("Creating Bazel symlink forest"), - ) - - bp2buildWorkspaceInvocation.Inputs = append(bp2buildWorkspaceInvocation.Inputs, - config.Bp2BuildFilesMarkerFile(), filepath.Join(config.FileListDir(), "bazel.list")) - - jsonModuleGraphInvocation := primaryBuilderInvocation( - config, - jsonModuleGraphTag, - config.ModuleGraphFile(), - []string{ - "--module_graph_file", config.ModuleGraphFile(), - "--module_actions_file", config.ModuleActionsFile(), - }, - fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()), - ) - queryviewDir := filepath.Join(config.SoongOutDir(), "queryview") - queryviewInvocation := primaryBuilderInvocation( - config, - queryviewTag, - config.QueryviewMarkerFile(), - []string{ - "--bazel_queryview_dir", queryviewDir, - }, - fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir), - ) - // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files) // The final workspace will be generated in out/soong/api_bp2build apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build") - apiBp2buildInvocation := primaryBuilderInvocation( - config, - apiBp2buildTag, - config.ApiBp2buildMarkerFile(), - []string{ - "--bazel_api_bp2build_dir", apiBp2buildDir, - }, - fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir), - ) - soongDocsInvocation := primaryBuilderInvocation( - config, - soongDocsTag, - config.SoongDocsHtml(), - []string{ - "--soong_docs", config.SoongDocsHtml(), + pbfs := []PrimaryBuilderFactory{ + { + name: soongBuildTag, + description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()), + config: config, + output: config.SoongNinjaFile(), + specificArgs: mainSoongBuildExtraArgs, }, - fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()), - ) + { + name: bp2buildFilesTag, + description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()), + config: config, + output: config.Bp2BuildFilesMarkerFile(), + specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()}, + }, + { + name: bp2buildWorkspaceTag, + description: "Creating Bazel symlink forest", + config: config, + output: config.Bp2BuildWorkspaceMarkerFile(), + specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()}, + }, + { + name: jsonModuleGraphTag, + description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()), + config: config, + output: config.ModuleGraphFile(), + specificArgs: []string{ + "--module_graph_file", config.ModuleGraphFile(), + "--module_actions_file", config.ModuleActionsFile(), + }, + }, + { + name: queryviewTag, + description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir), + config: config, + output: config.QueryviewMarkerFile(), + specificArgs: []string{"--bazel_queryview_dir", queryviewDir}, + }, + { + name: apiBp2buildTag, + description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir), + config: config, + output: config.ApiBp2buildMarkerFile(), + specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir}, + }, + { + name: soongDocsTag, + description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()), + config: config, + output: config.SoongDocsHtml(), + specificArgs: []string{"--soong_docs", config.SoongDocsHtml()}, + }, + } + + // Figure out which invocations will be run under the debugger: + // * SOONG_DELVE if set specifies listening port + // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are + debuggedInvocations := make(map[string]bool) + delvePort := os.Getenv("SOONG_DELVE") + if delvePort != "" { + if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" { + var validSteps []string + for _, pbf := range pbfs { + debuggedInvocations[pbf.name] = false + validSteps = append(validSteps, pbf.name) + + } + for _, step := range strings.Split(steps, ",") { + if _, ok := debuggedInvocations[step]; ok { + debuggedInvocations[step] = true + } else { + ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+ + "Valid steps are %v", step, validSteps) + } + } + } else { + // SOONG_DELVE_STEPS is not set, run all steps in the debugger + for _, pbf := range pbfs { + debuggedInvocations[pbf.name] = true + } + } + } + + var invocations []bootstrap.PrimaryBuilderInvocation + for _, pbf := range pbfs { + if debuggedInvocations[pbf.name] { + pbf.debugPort = delvePort + } + pbi := pbf.primaryBuilderInvocation() + // Some invocations require adjustment: + switch pbf.name { + case soongBuildTag: + if config.BazelBuildEnabled() { + // Mixed builds call Bazel from soong_build and they therefore need the + // Bazel workspace to be available. Make that so by adding a dependency on + // the bp2build marker file to the action that invokes soong_build . + pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile()) + } + case bp2buildWorkspaceTag: + pbi.Inputs = append(pbi.Inputs, + config.Bp2BuildFilesMarkerFile(), + filepath.Join(config.FileListDir(), "bazel.list")) + } + invocations = append(invocations, pbi) + } // The glob .ninja files are subninja'd. However, they are generated during // the build itself so we write an empty file if the file does not exist yet @@ -362,11 +392,11 @@ func bootstrapBlueprint(ctx Context, config Config) { writeEmptyFile(ctx, globFile) } - var blueprintArgs bootstrap.Args - - blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") - blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja") - blueprintArgs.EmptyNinjaFile = false + blueprintArgs := bootstrap.Args{ + ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"), + OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"), + EmptyNinjaFile: false, + } blueprintCtx := blueprint.NewContext() blueprintCtx.SetIgnoreUnknownModuleTypes(true) @@ -376,16 +406,9 @@ func bootstrapBlueprint(ctx Context, config Config) { outDir: config.OutDir(), runGoTests: !config.skipSoongTests, // If we want to debug soong_build, we need to compile it for debugging - debugCompilation: os.Getenv("SOONG_DELVE") != "", - subninjas: bootstrapGlobFileList(config), - primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{ - mainSoongBuildInvocation, - bp2buildInvocation, - bp2buildWorkspaceInvocation, - jsonModuleGraphInvocation, - queryviewInvocation, - apiBp2buildInvocation, - soongDocsInvocation}, + debugCompilation: delvePort != "", + subninjas: bootstrapGlobFileList(config), + primaryBuilderInvocations: invocations, } // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little