Use rsp files to pass turbine inputs to RBE
Turbine rules that run in RBE fail when they have long lists of flags. They work locally because the list of flags is placed in an RSP file, but the list of inputs still appears on the command line when RBE is enabled to pass them to rewrapper. When the list of srcjars or classpath arguments are too long, move the list of files into an rsp file, and pass the rsp file to RBE instead of the list of files. Bug; 308016794 Test: builds Flag: EXEMPT refactor Change-Id: I5ee610a91302ce94ec77b6f13b28a29bc63fd962
This commit is contained in:
Colin Cross
@@ -156,7 +156,7 @@ var (
|
||||
turbine, turbineRE = pctx.RemoteStaticRules("turbine",
|
||||
blueprint.RuleParams{
|
||||
Command: `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} $outputFlags ` +
|
||||
`--sources @$out.rsp --source_jars $srcJars ` +
|
||||
`--sources @$out.rsp ` +
|
||||
`--javacopts ${config.CommonJdkFlags} ` +
|
||||
`$javacFlags -source $javaVersion -target $javaVersion -- $turbineFlags && ` +
|
||||
`(for o in $outputs; do if cmp -s $${o}.tmp $${o} ; then rm $${o}.tmp ; else mv $${o}.tmp $${o} ; fi; done )`,
|
||||
@@ -170,13 +170,13 @@ var (
|
||||
},
|
||||
&remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "turbine"},
|
||||
ExecStrategy: "${config.RETurbineExecStrategy}",
|
||||
Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$implicits"},
|
||||
RSPFiles: []string{"${out}.rsp"},
|
||||
Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$rbeInputs"},
|
||||
RSPFiles: []string{"$out.rsp", "$rspFiles"},
|
||||
OutputFiles: []string{"$rbeOutputs"},
|
||||
ToolchainInputs: []string{"${config.JavaCmd}"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
[]string{"javacFlags", "turbineFlags", "outputFlags", "javaVersion", "outputs", "rbeOutputs", "srcJars"}, []string{"implicits"})
|
||||
[]string{"javacFlags", "turbineFlags", "outputFlags", "javaVersion", "outputs", "rbeOutputs"}, []string{"rbeInputs", "rspFiles"})
|
||||
|
||||
jar, jarRE = pctx.RemoteStaticRules("jar",
|
||||
blueprint.RuleParams{
|
||||
@@ -428,53 +428,72 @@ func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath, idx
|
||||
})
|
||||
}
|
||||
|
||||
func turbineFlags(ctx android.ModuleContext, flags javaBuilderFlags, dir string) (string, android.Paths) {
|
||||
var deps android.Paths
|
||||
func turbineFlags(ctx android.ModuleContext, flags javaBuilderFlags, dir string, srcJars android.Paths) (string, android.Paths, android.Paths, android.Paths) {
|
||||
var implicits android.Paths
|
||||
var rbeInputs android.Paths
|
||||
var rspFiles android.Paths
|
||||
|
||||
classpath := flags.classpath
|
||||
|
||||
var bootClasspath string
|
||||
srcJarArgs := strings.Join(srcJars.Strings(), " ")
|
||||
implicits = append(implicits, srcJars...)
|
||||
const srcJarArgsLimit = 32 * 1024
|
||||
if len(srcJarArgs) > srcJarArgsLimit {
|
||||
srcJarRspFile := android.PathForModuleOut(ctx, "turbine", "srcjars.rsp")
|
||||
android.WriteFileRule(ctx, srcJarRspFile, srcJarArgs)
|
||||
srcJarArgs = "@" + srcJarRspFile.String()
|
||||
implicits = append(implicits, srcJarRspFile)
|
||||
rbeInputs = append(rbeInputs, srcJarRspFile)
|
||||
} else {
|
||||
rbeInputs = append(rbeInputs, srcJars...)
|
||||
}
|
||||
|
||||
var bootClasspathFlags string
|
||||
if flags.javaVersion.usesJavaModules() {
|
||||
var systemModuleDeps android.Paths
|
||||
bootClasspath, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device())
|
||||
deps = append(deps, systemModuleDeps...)
|
||||
bootClasspathFlags, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device())
|
||||
implicits = append(implicits, systemModuleDeps...)
|
||||
rbeInputs = append(rbeInputs, systemModuleDeps...)
|
||||
classpath = append(flags.java9Classpath, classpath...)
|
||||
} else {
|
||||
deps = append(deps, flags.bootClasspath...)
|
||||
implicits = append(implicits, flags.bootClasspath...)
|
||||
rbeInputs = append(rbeInputs, flags.bootClasspath...)
|
||||
if len(flags.bootClasspath) == 0 && ctx.Device() {
|
||||
// explicitly specify -bootclasspath "" if the bootclasspath is empty to
|
||||
// ensure turbine does not fall back to the default bootclasspath.
|
||||
bootClasspath = `--bootclasspath ""`
|
||||
bootClasspathFlags = `--bootclasspath ""`
|
||||
} else {
|
||||
bootClasspath = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ")
|
||||
bootClasspathFlags = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ")
|
||||
}
|
||||
}
|
||||
|
||||
deps = append(deps, classpath...)
|
||||
turbineFlags := bootClasspath + " " + classpath.FormTurbineClassPath("--classpath ")
|
||||
|
||||
const flagsLimit = 32 * 1024
|
||||
if len(turbineFlags) > flagsLimit {
|
||||
flagsRspFile := android.PathForModuleOut(ctx, dir, "turbine-flags.rsp")
|
||||
android.WriteFileRule(ctx, flagsRspFile, turbineFlags)
|
||||
turbineFlags = "@" + flagsRspFile.String()
|
||||
deps = append(deps, flagsRspFile)
|
||||
classpathFlags := classpath.FormTurbineClassPath("")
|
||||
implicits = append(implicits, classpath...)
|
||||
const classpathLimit = 32 * 1024
|
||||
if len(classpathFlags) > classpathLimit {
|
||||
classpathRspFile := android.PathForModuleOut(ctx, dir, "classpath.rsp")
|
||||
android.WriteFileRule(ctx, classpathRspFile, classpathFlags)
|
||||
classpathFlags = "@" + classpathRspFile.String()
|
||||
implicits = append(implicits, classpathRspFile)
|
||||
rspFiles = append(rspFiles, classpathRspFile)
|
||||
rbeInputs = append(rbeInputs, classpathRspFile)
|
||||
} else {
|
||||
rbeInputs = append(rbeInputs, classpath...)
|
||||
}
|
||||
|
||||
return turbineFlags, deps
|
||||
turbineFlags := "--source_jars " + srcJarArgs + " " + bootClasspathFlags + " --classpath " + classpathFlags
|
||||
|
||||
return turbineFlags, implicits, rbeInputs, rspFiles
|
||||
}
|
||||
|
||||
func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath,
|
||||
srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
|
||||
|
||||
turbineFlags, deps := turbineFlags(ctx, flags, "turbine")
|
||||
|
||||
deps = append(deps, srcJars...)
|
||||
turbineFlags, implicits, rbeInputs, rspFiles := turbineFlags(ctx, flags, "turbine", srcJars)
|
||||
|
||||
rule := turbine
|
||||
args := map[string]string{
|
||||
"javacFlags": flags.javacFlags,
|
||||
"srcJars": strings.Join(srcJars.Strings(), " "),
|
||||
"javaVersion": flags.javaVersion.String(),
|
||||
"turbineFlags": turbineFlags,
|
||||
"outputFlags": "--output " + outputFile.String() + ".tmp",
|
||||
@@ -482,15 +501,16 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.
|
||||
}
|
||||
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_TURBINE") {
|
||||
rule = turbineRE
|
||||
args["implicits"] = strings.Join(deps.Strings(), ",")
|
||||
args["rbeInputs"] = strings.Join(rbeInputs.Strings(), ",")
|
||||
args["rbeOutputs"] = outputFile.String() + ".tmp"
|
||||
args["rspFiles"] = strings.Join(rspFiles.Strings(), ",")
|
||||
}
|
||||
ctx.Build(pctx, android.BuildParams{
|
||||
Rule: rule,
|
||||
Description: "turbine",
|
||||
Output: outputFile,
|
||||
Inputs: srcFiles,
|
||||
Implicits: deps,
|
||||
Implicits: implicits,
|
||||
Args: args,
|
||||
})
|
||||
}
|
||||
@@ -499,11 +519,10 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.
|
||||
func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.WritablePath,
|
||||
srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
|
||||
|
||||
turbineFlags, deps := turbineFlags(ctx, flags, "kapt")
|
||||
turbineFlags, implicits, rbeInputs, rspFiles := turbineFlags(ctx, flags, "turbine-apt", srcJars)
|
||||
|
||||
deps = append(deps, srcJars...)
|
||||
|
||||
deps = append(deps, flags.processorPath...)
|
||||
implicits = append(implicits, flags.processorPath...)
|
||||
rbeInputs = append(rbeInputs, flags.processorPath...)
|
||||
turbineFlags += " " + flags.processorPath.FormTurbineClassPath("--processorpath ")
|
||||
turbineFlags += " --processors " + strings.Join(flags.processors, " ")
|
||||
|
||||
@@ -514,7 +533,6 @@ func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.Wr
|
||||
rule := turbine
|
||||
args := map[string]string{
|
||||
"javacFlags": flags.javacFlags,
|
||||
"srcJars": strings.Join(srcJars.Strings(), " "),
|
||||
"javaVersion": flags.javaVersion.String(),
|
||||
"turbineFlags": turbineFlags,
|
||||
"outputFlags": outputFlags,
|
||||
@@ -522,8 +540,9 @@ func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.Wr
|
||||
}
|
||||
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_TURBINE") {
|
||||
rule = turbineRE
|
||||
args["implicits"] = strings.Join(deps.Strings(), ",")
|
||||
args["rbeInputs"] = strings.Join(rbeInputs.Strings(), ",")
|
||||
args["rbeOutputs"] = outputSrcJar.String() + ".tmp," + outputResJar.String() + ".tmp"
|
||||
args["rspFiles"] = strings.Join(rspFiles.Strings(), ",")
|
||||
}
|
||||
ctx.Build(pctx, android.BuildParams{
|
||||
Rule: rule,
|
||||
@@ -531,7 +550,7 @@ func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.Wr
|
||||
Output: outputs[0],
|
||||
ImplicitOutputs: outputs[1:],
|
||||
Inputs: srcFiles,
|
||||
Implicits: deps,
|
||||
Implicits: implicits,
|
||||
Args: args,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user