Merge changes Iab4e09d9,Icf2f24dd,I15be5ef1,Ic0db9619

* changes: Run lint actions in sbox Support sbox-in-RBE Move android package on top of remotexec Support sandboxing inputs in RuleBuilder
2021-03-19 17:25:14 +00:00
parent 00d54280d0 1661aff8be
commit d2db7953a0
18 changed files with 367 additions and 207 deletions
--- a/java/app_builder.go
+++ b/java/app_builder.go
@@ -30,7 +30,7 @@ import (
 )

 var (
-	Signapk, SignapkRE = remoteexec.StaticRules(pctx, "signapk",
+	Signapk, SignapkRE = pctx.RemoteStaticRules("signapk",
 		blueprint.RuleParams{
 			Command: `rm -f $out && $reTemplate${config.JavaCmd} ${config.JavaVmFlags} -Djava.library.path=$$(dirname ${config.SignapkJniLibrary}) ` +
 				`-jar ${config.SignapkCmd} $flags $certificates $in $out`,
--- a/java/builder.go
+++ b/java/builder.go
@@ -40,7 +40,7 @@ var (
 	// (if the rule produces .class files) or a .srcjar file (if the rule produces .java files).
 	// .srcjar files are unzipped into a temporary directory when compiled with javac.
 	// TODO(b/143658984): goma can't handle the --system argument to javac.
-	javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac",
+	javac, javacRE = pctx.MultiCommandRemoteStaticRules("javac",
 		blueprint.RuleParams{
 			Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" "$out" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +
 				`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +
@@ -129,7 +129,7 @@ var (
 		},
 		"abis", "allow-prereleased", "screen-densities", "sdk-version", "stem", "apkcerts", "partition")

-	turbine, turbineRE = remoteexec.StaticRules(pctx, "turbine",
+	turbine, turbineRE = pctx.RemoteStaticRules("turbine",
 		blueprint.RuleParams{
 			Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
 				`$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} --output $out.tmp ` +
@@ -157,7 +157,7 @@ var (
 			Platform:          map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
 		}, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"})

-	jar, jarRE = remoteexec.StaticRules(pctx, "jar",
+	jar, jarRE = pctx.RemoteStaticRules("jar",
 		blueprint.RuleParams{
 			Command:        `$reTemplate${config.SoongZipCmd} -jar -o $out @$out.rsp`,
 			CommandDeps:    []string{"${config.SoongZipCmd}"},
@@ -172,7 +172,7 @@ var (
 			Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
 		}, []string{"jarArgs"}, nil)

-	zip, zipRE = remoteexec.StaticRules(pctx, "zip",
+	zip, zipRE = pctx.RemoteStaticRules("zip",
 		blueprint.RuleParams{
 			Command:        `${config.SoongZipCmd} -o $out @$out.rsp`,
 			CommandDeps:    []string{"${config.SoongZipCmd}"},
@@ -244,7 +244,6 @@ var (
 func init() {
 	pctx.Import("android/soong/android")
 	pctx.Import("android/soong/java/config")
-	pctx.Import("android/soong/remoteexec")
 }

 type javaBuilderFlags struct {
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -149,14 +149,14 @@ func init() {
 	pctx.HostBinToolVariable("SoongJavacWrapper", "soong_javac_wrapper")
 	pctx.HostBinToolVariable("DexpreoptGen", "dexpreopt_gen")

-	pctx.VariableFunc("REJavaPool", remoteexec.EnvOverrideFunc("RBE_JAVA_POOL", "java16"))
-	pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy))
-	pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy))
-	pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy))
-	pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
-	pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
-	pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
-	pctx.VariableFunc("REZipExecStrategy", remoteexec.EnvOverrideFunc("RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
+	pctx.StaticVariableWithEnvOverride("REJavaPool", "RBE_JAVA_POOL", "java16")
+	pctx.StaticVariableWithEnvOverride("REJavacExecStrategy", "RBE_JAVAC_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)
+	pctx.StaticVariableWithEnvOverride("RED8ExecStrategy", "RBE_D8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)
+	pctx.StaticVariableWithEnvOverride("RER8ExecStrategy", "RBE_R8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)
+	pctx.StaticVariableWithEnvOverride("RETurbineExecStrategy", "RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy)
+	pctx.StaticVariableWithEnvOverride("RESignApkExecStrategy", "RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy)
+	pctx.StaticVariableWithEnvOverride("REJarExecStrategy", "RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy)
+	pctx.StaticVariableWithEnvOverride("REZipExecStrategy", "RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy)

 	pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar")

--- a/java/dex.go
+++ b/java/dex.go
@@ -83,7 +83,7 @@ func (d *dexer) effectiveOptimizeEnabled() bool {
 	return BoolDefault(d.dexProperties.Optimize.Enabled, d.dexProperties.Optimize.EnabledByDefault)
 }

-var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8",
+var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8",
 	blueprint.RuleParams{
 		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
 			`$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +
@@ -111,7 +111,7 @@ var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8",
 		},
 	}, []string{"outDir", "d8Flags", "zipFlags"}, nil)

-var r8, r8RE = remoteexec.MultiCommandStaticRules(pctx, "r8",
+var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8",
 	blueprint.RuleParams{
 		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
 			`rm -f "$outDict" && rm -rf "${outUsageDir}" && ` +
--- a/java/droiddoc.go
+++ b/java/droiddoc.go
@@ -1235,7 +1235,7 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi
 			ToolchainInputs:      []string{config.JavaCmd(ctx).String()},
 			Platform:             map[string]string{remoteexec.PoolKey: pool},
 			EnvironmentVariables: []string{"ANDROID_SDK_HOME"},
-		}).NoVarTemplate(ctx.Config()))
+		}).NoVarTemplate(ctx.Config().RBEWrapper()))
 	}

 	cmd.BuiltTool("metalava").
@@ -1678,14 +1678,17 @@ func StubsDefaultsFactory() android.Module {
 func zipSyncCmd(ctx android.ModuleContext, rule *android.RuleBuilder,
 	srcJarDir android.ModuleOutPath, srcJars android.Paths) android.OutputPath {

-	rule.Command().Text("rm -rf").Text(srcJarDir.String())
-	rule.Command().Text("mkdir -p").Text(srcJarDir.String())
+	cmd := rule.Command()
+	cmd.Text("rm -rf").Text(cmd.PathForOutput(srcJarDir))
+	cmd = rule.Command()
+	cmd.Text("mkdir -p").Text(cmd.PathForOutput(srcJarDir))
 	srcJarList := srcJarDir.Join(ctx, "list")

 	rule.Temporary(srcJarList)

-	rule.Command().BuiltTool("zipsync").
-		FlagWithArg("-d ", srcJarDir.String()).
+	cmd = rule.Command()
+	cmd.BuiltTool("zipsync").
+		FlagWithArg("-d ", cmd.PathForOutput(srcJarDir)).
 		FlagWithOutput("-l ", srcJarList).
 		FlagWithArg("-f ", `"*.java"`).
 		Inputs(srcJars)
--- a/java/java_test.go
+++ b/java/java_test.go
@@ -25,12 +25,12 @@ import (
 	"strings"
 	"testing"

-	"android/soong/genrule"
 	"github.com/google/blueprint/proptools"

 	"android/soong/android"
 	"android/soong/cc"
 	"android/soong/dexpreopt"
+	"android/soong/genrule"
 	"android/soong/python"
 )

@@ -1301,9 +1301,9 @@ func TestJavaLint(t *testing.T) {
 	})

 	foo := ctx.ModuleForTests("foo", "android_common")
-	rule := foo.Rule("lint")

-	if !strings.Contains(rule.RuleParams.Command, "--baseline lint-baseline.xml") {
+	sboxProto := android.RuleBuilderSboxProtoForTests(t, foo.Output("lint.sbox.textproto"))
+	if !strings.Contains(*sboxProto.Commands[0].Command, "--baseline lint-baseline.xml") {
 		t.Error("did not pass --baseline flag")
 	}
 }
@@ -1323,9 +1323,9 @@ func TestJavaLintWithoutBaseline(t *testing.T) {
       `, map[string][]byte{})

 	foo := ctx.ModuleForTests("foo", "android_common")
-	rule := foo.Rule("lint")

-	if strings.Contains(rule.RuleParams.Command, "--baseline") {
+	sboxProto := android.RuleBuilderSboxProtoForTests(t, foo.Output("lint.sbox.textproto"))
+	if strings.Contains(*sboxProto.Commands[0].Command, "--baseline") {
 		t.Error("passed --baseline flag for non existent file")
 	}
 }
@@ -1381,9 +1381,9 @@ func TestJavaLintUsesCorrectBpConfig(t *testing.T) {
 	})

 	foo := ctx.ModuleForTests("foo", "android_common")
-	rule := foo.Rule("lint")

-	if !strings.Contains(rule.RuleParams.Command, "--baseline mybaseline.xml") {
+	sboxProto := android.RuleBuilderSboxProtoForTests(t, foo.Output("lint.sbox.textproto"))
+	if !strings.Contains(*sboxProto.Commands[0].Command, "--baseline mybaseline.xml") {
 		t.Error("did not use the correct file for baseline")
 	}
 }
--- a/java/lint.go
+++ b/java/lint.go
@@ -218,7 +218,7 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
 		// The list of resources may be too long to put on the command line, but
 		// we can't use the rsp file because it is already being used for srcs.
 		// Insert a second rule to write out the list of resources to a file.
-		resourcesList = android.PathForModuleOut(ctx, "lint", "resources.list")
+		resourcesList = android.PathForModuleOut(ctx, "resources.list")
 		resListRule := android.NewRuleBuilder(pctx, ctx)
 		resListRule.Command().Text("cp").
 			FlagWithRspFileInputList("", resourcesList.ReplaceExtension(ctx, "rsp"), l.resources).
@@ -233,7 +233,7 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
 	cacheDir := android.PathForModuleOut(ctx, "lint", "cache")
 	homeDir := android.PathForModuleOut(ctx, "lint", "home")

-	srcJarDir := android.PathForModuleOut(ctx, "lint-srcjars")
+	srcJarDir := android.PathForModuleOut(ctx, "lint", "srcjars")
 	srcJarList := zipSyncCmd(ctx, rule, srcJarDir, l.srcJars)
 	// TODO(ccross): this is a little fishy.  The files extracted from the srcjars are referenced
 	// by the project.xml and used by the later lint rule, but the lint rule depends on the srcjars,
@@ -248,6 +248,7 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
 		FlagWithRspFileInputList("", srcsListRsp, l.srcs).
 		Output(srcsList)
 	trackRSPDependency(l.srcs, srcsList)
+	rule.Temporary(srcsList)

 	cmd := rule.Command().
 		BuiltTool("lint-project-xml").
@@ -262,11 +263,11 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
 		cmd.Flag("--test")
 	}
 	if l.manifest != nil {
-		cmd.FlagWithArg("--manifest ", l.manifest.String())
+		cmd.FlagWithArg("--manifest ", cmd.PathForInput(l.manifest))
 		trackInputDependency(l.manifest)
 	}
 	if l.mergedManifest != nil {
-		cmd.FlagWithArg("--merged_manifest ", l.mergedManifest.String())
+		cmd.FlagWithArg("--merged_manifest ", cmd.PathForInput(l.mergedManifest))
 		trackInputDependency(l.mergedManifest)
 	}

@@ -279,23 +280,17 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
 	}

 	if l.classes != nil {
-		cmd.FlagWithArg("--classes ", l.classes.String())
+		cmd.FlagWithArg("--classes ", cmd.PathForInput(l.classes))
 		trackInputDependency(l.classes)
 	}

-	cmd.FlagForEachArg("--classpath ", l.classpath.Strings())
+	cmd.FlagForEachArg("--classpath ", cmd.PathsForInputs(l.classpath))
 	trackInputDependency(l.classpath...)

-	cmd.FlagForEachArg("--extra_checks_jar ", l.extraLintCheckJars.Strings())
+	cmd.FlagForEachArg("--extra_checks_jar ", cmd.PathsForInputs(l.extraLintCheckJars))
 	trackInputDependency(l.extraLintCheckJars...)

-	if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_LINT") &&
-		lintRBEExecStrategy(ctx) != remoteexec.LocalExecStrategy {
-		// TODO(b/181912787): remove these and use "." instead.
-		cmd.FlagWithArg("--root_dir ", "/b/f/w")
-	} else {
-		cmd.FlagWithArg("--root_dir ", "$PWD")
-	}
+	cmd.FlagWithArg("--root_dir ", "$PWD")

 	// The cache tag in project.xml is relative to the root dir, or the project.xml file if
 	// the root dir is not set.
@@ -325,7 +320,7 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru

 // generateManifest adds a command to the rule to write a simple manifest that contains the
 // minSdkVersion and targetSdkVersion for modules (like java_library) that don't have a manifest.
-func (l *linter) generateManifest(ctx android.ModuleContext, rule *android.RuleBuilder) android.Path {
+func (l *linter) generateManifest(ctx android.ModuleContext, rule *android.RuleBuilder) android.WritablePath {
 	manifestPath := android.PathForModuleOut(ctx, "lint", "AndroidManifest.xml")

 	rule.Command().Text("(").
@@ -356,18 +351,36 @@ func (l *linter) lint(ctx android.ModuleContext) {
 		}
 	}

-	rule := android.NewRuleBuilder(pctx, ctx)
+	rule := android.NewRuleBuilder(pctx, ctx).
+		Sbox(android.PathForModuleOut(ctx, "lint"),
+			android.PathForModuleOut(ctx, "lint.sbox.textproto")).
+		SandboxInputs()
+
+	if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_LINT") {
+		pool := ctx.Config().GetenvWithDefault("RBE_LINT_POOL", "java16")
+		rule.Remoteable(android.RemoteRuleSupports{RBE: true})
+		rule.Rewrapper(&remoteexec.REParams{
+			Labels:          map[string]string{"type": "tool", "name": "lint"},
+			ExecStrategy:    lintRBEExecStrategy(ctx),
+			ToolchainInputs: []string{config.JavaCmd(ctx).String()},
+			EnvironmentVariables: []string{
+				"LANG",
+			},
+			Platform: map[string]string{remoteexec.PoolKey: pool},
+		})
+	}

 	if l.manifest == nil {
 		manifest := l.generateManifest(ctx, rule)
 		l.manifest = manifest
+		rule.Temporary(manifest)
 	}

 	lintPaths := l.writeLintProjectXML(ctx, rule)

-	html := android.PathForModuleOut(ctx, "lint-report.html")
-	text := android.PathForModuleOut(ctx, "lint-report.txt")
-	xml := android.PathForModuleOut(ctx, "lint-report.xml")
+	html := android.PathForModuleOut(ctx, "lint", "lint-report.html")
+	text := android.PathForModuleOut(ctx, "lint", "lint-report.txt")
+	xml := android.PathForModuleOut(ctx, "lint", "lint-report.xml")

 	depSetsBuilder := NewLintDepSetBuilder().Direct(html, text, xml)

@@ -397,43 +410,7 @@ func (l *linter) lint(ctx android.ModuleContext) {
 		FlagWithInput("SDK_ANNOTATIONS=", annotationsZipPath).
 		FlagWithInput("LINT_OPTS=-DLINT_API_DATABASE=", apiVersionsXMLPath)

-	if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_LINT") {
-		pool := ctx.Config().GetenvWithDefault("RBE_LINT_POOL", "java16")
-		// TODO(b/181912787): this should be local fallback once the hack that passes /b/f/w in project.xml
-		// is removed.
-		execStrategy := lintRBEExecStrategy(ctx)
-		labels := map[string]string{"type": "tool", "name": "lint"}
-		rule.Remoteable(android.RemoteRuleSupports{RBE: true})
-		remoteInputs := lintPaths.remoteInputs
-		remoteInputs = append(remoteInputs,
-			lintPaths.projectXML,
-			lintPaths.configXML,
-			lintPaths.homeDir,
-			lintPaths.cacheDir,
-			ctx.Config().HostJavaToolPath(ctx, "lint.jar"),
-			annotationsZipPath,
-			apiVersionsXMLPath,
-		)
-
-		cmd.Text((&remoteexec.REParams{
-			Labels:          labels,
-			ExecStrategy:    execStrategy,
-			ToolchainInputs: []string{config.JavaCmd(ctx).String()},
-			Inputs:          remoteInputs.Strings(),
-			OutputFiles:     android.Paths{html, text, xml}.Strings(),
-			RSPFile:         strings.Join(lintPaths.remoteRSPInputs.Strings(), ","),
-			EnvironmentVariables: []string{
-				"JAVA_OPTS",
-				"ANDROID_SDK_HOME",
-				"SDK_ANNOTATIONS",
-				"LINT_OPTS",
-				"LANG",
-			},
-			Platform: map[string]string{remoteexec.PoolKey: pool},
-		}).NoVarTemplate(ctx.Config()))
-	}
-
-	cmd.BuiltTool("lint").
+	cmd.BuiltTool("lint").ImplicitTool(ctx.Config().HostJavaToolPath(ctx, "lint.jar")).
 		Flag("--quiet").
 		FlagWithInput("--project ", lintPaths.projectXML).
 		FlagWithInput("--config ", lintPaths.configXML).
@@ -450,6 +427,9 @@ func (l *linter) lint(ctx android.ModuleContext) {
 		Implicit(apiVersionsXMLPath).
 		Implicits(lintPaths.deps)

+	rule.Temporary(lintPaths.projectXML)
+	rule.Temporary(lintPaths.configXML)
+
 	if checkOnly := ctx.Config().Getenv("ANDROID_LINT_CHECK"); checkOnly != "" {
 		cmd.FlagWithArg("--check ", checkOnly)
 	}