From 2fcc48e4a36d10af71dcdb374d54d3d9c100dd67 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 9 Aug 2024 15:17:01 -0700 Subject: [PATCH] Use rsp files to pass turbine inputs to RBE Turbine rules that run in RBE fail when they have long lists of flags. They work locally because the list of flags is placed in an RSP file, but the list of inputs still appears on the command line when RBE is enabled to pass them to rewrapper. When the list of srcjars or classpath arguments are too long, move the list of files into an rsp file, and pass the rsp file to RBE instead of the list of files. Bug; 308016794 Test: builds Flag: EXEMPT refactor Change-Id: I5ee610a91302ce94ec77b6f13b28a29bc63fd962 --- java/builder.go | 89 ++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/java/builder.go b/java/builder.go index 5d84d0b43..591523545 100644 --- a/java/builder.go +++ b/java/builder.go @@ -156,7 +156,7 @@ var ( turbine, turbineRE = pctx.RemoteStaticRules("turbine", blueprint.RuleParams{ Command: `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} $outputFlags ` + - `--sources @$out.rsp --source_jars $srcJars ` + + `--sources @$out.rsp ` + `--javacopts ${config.CommonJdkFlags} ` + `$javacFlags -source $javaVersion -target $javaVersion -- $turbineFlags && ` + `(for o in $outputs; do if cmp -s $${o}.tmp $${o} ; then rm $${o}.tmp ; else mv $${o}.tmp $${o} ; fi; done )`, @@ -170,13 +170,13 @@ var ( }, &remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "turbine"}, ExecStrategy: "${config.RETurbineExecStrategy}", - Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$implicits"}, - RSPFiles: []string{"${out}.rsp"}, + Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$rbeInputs"}, + RSPFiles: []string{"$out.rsp", "$rspFiles"}, OutputFiles: []string{"$rbeOutputs"}, ToolchainInputs: []string{"${config.JavaCmd}"}, Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, }, - []string{"javacFlags", "turbineFlags", "outputFlags", "javaVersion", "outputs", "rbeOutputs", "srcJars"}, []string{"implicits"}) + []string{"javacFlags", "turbineFlags", "outputFlags", "javaVersion", "outputs", "rbeOutputs"}, []string{"rbeInputs", "rspFiles"}) jar, jarRE = pctx.RemoteStaticRules("jar", blueprint.RuleParams{ @@ -428,53 +428,72 @@ func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath, idx }) } -func turbineFlags(ctx android.ModuleContext, flags javaBuilderFlags, dir string) (string, android.Paths) { - var deps android.Paths +func turbineFlags(ctx android.ModuleContext, flags javaBuilderFlags, dir string, srcJars android.Paths) (string, android.Paths, android.Paths, android.Paths) { + var implicits android.Paths + var rbeInputs android.Paths + var rspFiles android.Paths classpath := flags.classpath - var bootClasspath string + srcJarArgs := strings.Join(srcJars.Strings(), " ") + implicits = append(implicits, srcJars...) + const srcJarArgsLimit = 32 * 1024 + if len(srcJarArgs) > srcJarArgsLimit { + srcJarRspFile := android.PathForModuleOut(ctx, "turbine", "srcjars.rsp") + android.WriteFileRule(ctx, srcJarRspFile, srcJarArgs) + srcJarArgs = "@" + srcJarRspFile.String() + implicits = append(implicits, srcJarRspFile) + rbeInputs = append(rbeInputs, srcJarRspFile) + } else { + rbeInputs = append(rbeInputs, srcJars...) + } + + var bootClasspathFlags string if flags.javaVersion.usesJavaModules() { var systemModuleDeps android.Paths - bootClasspath, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device()) - deps = append(deps, systemModuleDeps...) + bootClasspathFlags, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device()) + implicits = append(implicits, systemModuleDeps...) + rbeInputs = append(rbeInputs, systemModuleDeps...) classpath = append(flags.java9Classpath, classpath...) } else { - deps = append(deps, flags.bootClasspath...) + implicits = append(implicits, flags.bootClasspath...) + rbeInputs = append(rbeInputs, flags.bootClasspath...) if len(flags.bootClasspath) == 0 && ctx.Device() { // explicitly specify -bootclasspath "" if the bootclasspath is empty to // ensure turbine does not fall back to the default bootclasspath. - bootClasspath = `--bootclasspath ""` + bootClasspathFlags = `--bootclasspath ""` } else { - bootClasspath = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ") + bootClasspathFlags = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ") } } - deps = append(deps, classpath...) - turbineFlags := bootClasspath + " " + classpath.FormTurbineClassPath("--classpath ") - - const flagsLimit = 32 * 1024 - if len(turbineFlags) > flagsLimit { - flagsRspFile := android.PathForModuleOut(ctx, dir, "turbine-flags.rsp") - android.WriteFileRule(ctx, flagsRspFile, turbineFlags) - turbineFlags = "@" + flagsRspFile.String() - deps = append(deps, flagsRspFile) + classpathFlags := classpath.FormTurbineClassPath("") + implicits = append(implicits, classpath...) + const classpathLimit = 32 * 1024 + if len(classpathFlags) > classpathLimit { + classpathRspFile := android.PathForModuleOut(ctx, dir, "classpath.rsp") + android.WriteFileRule(ctx, classpathRspFile, classpathFlags) + classpathFlags = "@" + classpathRspFile.String() + implicits = append(implicits, classpathRspFile) + rspFiles = append(rspFiles, classpathRspFile) + rbeInputs = append(rbeInputs, classpathRspFile) + } else { + rbeInputs = append(rbeInputs, classpath...) } - return turbineFlags, deps + turbineFlags := "--source_jars " + srcJarArgs + " " + bootClasspathFlags + " --classpath " + classpathFlags + + return turbineFlags, implicits, rbeInputs, rspFiles } func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath, srcFiles, srcJars android.Paths, flags javaBuilderFlags) { - turbineFlags, deps := turbineFlags(ctx, flags, "turbine") - - deps = append(deps, srcJars...) + turbineFlags, implicits, rbeInputs, rspFiles := turbineFlags(ctx, flags, "turbine", srcJars) rule := turbine args := map[string]string{ "javacFlags": flags.javacFlags, - "srcJars": strings.Join(srcJars.Strings(), " "), "javaVersion": flags.javaVersion.String(), "turbineFlags": turbineFlags, "outputFlags": "--output " + outputFile.String() + ".tmp", @@ -482,15 +501,16 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android. } if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_TURBINE") { rule = turbineRE - args["implicits"] = strings.Join(deps.Strings(), ",") + args["rbeInputs"] = strings.Join(rbeInputs.Strings(), ",") args["rbeOutputs"] = outputFile.String() + ".tmp" + args["rspFiles"] = strings.Join(rspFiles.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "turbine", Output: outputFile, Inputs: srcFiles, - Implicits: deps, + Implicits: implicits, Args: args, }) } @@ -499,11 +519,10 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android. func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.WritablePath, srcFiles, srcJars android.Paths, flags javaBuilderFlags) { - turbineFlags, deps := turbineFlags(ctx, flags, "kapt") + turbineFlags, implicits, rbeInputs, rspFiles := turbineFlags(ctx, flags, "turbine-apt", srcJars) - deps = append(deps, srcJars...) - - deps = append(deps, flags.processorPath...) + implicits = append(implicits, flags.processorPath...) + rbeInputs = append(rbeInputs, flags.processorPath...) turbineFlags += " " + flags.processorPath.FormTurbineClassPath("--processorpath ") turbineFlags += " --processors " + strings.Join(flags.processors, " ") @@ -514,7 +533,6 @@ func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.Wr rule := turbine args := map[string]string{ "javacFlags": flags.javacFlags, - "srcJars": strings.Join(srcJars.Strings(), " "), "javaVersion": flags.javaVersion.String(), "turbineFlags": turbineFlags, "outputFlags": outputFlags, @@ -522,8 +540,9 @@ func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.Wr } if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_TURBINE") { rule = turbineRE - args["implicits"] = strings.Join(deps.Strings(), ",") + args["rbeInputs"] = strings.Join(rbeInputs.Strings(), ",") args["rbeOutputs"] = outputSrcJar.String() + ".tmp," + outputResJar.String() + ".tmp" + args["rspFiles"] = strings.Join(rspFiles.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, @@ -531,7 +550,7 @@ func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.Wr Output: outputs[0], ImplicitOutputs: outputs[1:], Inputs: srcFiles, - Implicits: deps, + Implicits: implicits, Args: args, }) }