Use jars containg sources for java generators

srcFileLists was an ill-fated attempt to deal with generators that
produce a set of java sources that is not known ahead of time.
For example, the list of files produced by protoc depends on the
package statement in the .proto file.  srcFileLists put the list
of generated files into a file, which was then passed to javac
using the @file syntax.  This worked, but it was too easy to cause
missing dependencies, and will not work well in a future distributed
build environment.

Switch to putting generated sources into a jar, and then pass them
jar to javac using -sourcepath.

Test: m checkbuild
Change-Id: Iaab7a588a6c1239f7bf46e4f1b102b3ef517619b

java/app.go

@@ -89,7 +89,7 @@ func (a *AndroidApp) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 		publicResourcesFile, proguardOptionsFile, aaptJavaFileList :=
 			CreateResourceJavaFiles(ctx, aaptRJavaFlags, aaptDeps)
 		a.aaptJavaFileList = aaptJavaFileList
-		a.ExtraSrcLists = append(a.ExtraSrcLists, aaptJavaFileList)
+		// TODO(ccross):  export aapt generated java files as a src jar
 
 		if a.appProperties.Export_package_resources {
 			aaptPackageFlags := append([]string(nil), aaptFlags...)

java/builder.go

@@ -40,7 +40,7 @@ var (
 		blueprint.RuleParams{
 			Command: `rm -rf "$outDir" "$annoDir" && mkdir -p "$outDir" "$annoDir" && ` +
 				`${config.JavacWrapper}${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` +
-				`$javacFlags $bootClasspath $classpath ` +
+				`$javacFlags $sourcepath $bootClasspath $classpath ` +
 				`-source $javaVersion -target $javaVersion ` +
 				`-d $outDir -s $annoDir @$out.rsp && ` +
 				`${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
@@ -48,13 +48,13 @@ var (
 			Rspfile:        "$out.rsp",
 			RspfileContent: "$in",
 		},
-		"javacFlags", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion")
+		"javacFlags", "sourcepath", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion")
 
 	errorprone = pctx.AndroidStaticRule("errorprone",
 		blueprint.RuleParams{
 			Command: `rm -rf "$outDir" "$annoDir" && mkdir -p "$outDir" "$annoDir" && ` +
 				`${config.ErrorProneCmd} ` +
-				`$javacFlags $bootClasspath $classpath ` +
+				`$javacFlags $sourcepath $bootClasspath $classpath ` +
 				`-source $javaVersion -target $javaVersion ` +
 				`-d $outDir -s $annoDir @$out.rsp && ` +
 				`${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
@@ -67,7 +67,7 @@ var (
 			Rspfile:        "$out.rsp",
 			RspfileContent: "$in",
 		},
-		"javacFlags", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion")
+		"javacFlags", "sourcepath", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion")
 
 	jar = pctx.AndroidStaticRule("jar",
 		blueprint.RuleParams{
@@ -136,31 +136,28 @@ type javaBuilderFlags struct {
 }
 
 func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
-	srcFiles, srcFileLists android.Paths,
+	srcFiles android.Paths, srcJars classpath,
 	flags javaBuilderFlags, deps android.Paths) {
 
-	transformJavaToClasses(ctx, outputFile, srcFiles, srcFileLists, flags, deps,
+	transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, deps,
 		"", "javac", javac)
 }
 
 func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
-	srcFiles, srcFileLists android.Paths,
+	srcFiles android.Paths, srcJars classpath,
 	flags javaBuilderFlags) {
 
 	if config.ErrorProneJar == "" {
 		ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?")
 	}
 
-	transformJavaToClasses(ctx, outputFile, srcFiles, srcFileLists, flags, nil,
+	transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, nil,
 		"-errorprone", "errorprone", errorprone)
 }
 
 // transformJavaToClasses takes source files and converts them to a jar containing .class files.
-// srcFiles is a list of paths to sources, srcFileLists is a list of paths to files that contain
-// paths to sources.  There is no dependency on the sources passed through srcFileLists, those
-// must be added through the deps argument, which contains a list of paths that should be added
-// as implicit dependencies.  flags contains various command line flags to be passed to the
-// compiler.
+// srcFiles is a list of paths to sources, srcJars is a list of paths to jar files that contain
+// sources.  flags contains various command line flags to be passed to the compiler.
 //
 // This method may be used for different compilers, including javac and Error Prone.  The rule
 // argument specifies which command line to use and desc sets the description of the rule that will
@@ -168,16 +165,11 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
 // suffix will be appended to various intermediate files and directories to avoid collisions when
 // this function is called twice in the same module directory.
 func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
-	srcFiles, srcFileLists android.Paths,
+	srcFiles android.Paths, srcJars classpath,
 	flags javaBuilderFlags, deps android.Paths,
 	intermediatesSuffix, desc string, rule blueprint.Rule) {
 
-	javacFlags := flags.javacFlags
-	if len(srcFileLists) > 0 {
-		javacFlags += " " + android.JoinWithPrefix(srcFileLists.Strings(), "@")
-	}
-
-	deps = append(deps, srcFileLists...)
+	deps = append(deps, srcJars...)
 
 	var bootClasspath string
 	if flags.javaVersion == "1.9" {
@@ -197,8 +189,9 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab
 		Inputs:      srcFiles,
 		Implicits:   deps,
 		Args: map[string]string{
-			"javacFlags":    javacFlags,
+			"javacFlags":    flags.javacFlags,
 			"bootClasspath": bootClasspath,
+			"sourcepath":    srcJars.JavaSourcepath(),
 			"classpath":     flags.classpath.JavaClasspath(),
 			"outDir":        android.PathForModuleOut(ctx, "classes"+intermediatesSuffix).String(),
 			"annoDir":       android.PathForModuleOut(ctx, "anno"+intermediatesSuffix).String(),
@@ -316,6 +309,16 @@ func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath,
 
 type classpath []android.Path
 
+// Returns a -sourcepath argument in the form javac expects.  If the list is empty returns
+// -sourcepath "" to ensure javac does not fall back to searching the classpath for sources.
+func (x *classpath) JavaSourcepath() string {
+	if len(*x) > 0 {
+		return "-sourcepath " + strings.Join(x.Strings(), ":")
+	} else {
+		return `-sourcepath ""`
+	}
+}
+
 // Returns a -classpath argument in the form java or javac expects
 func (x *classpath) JavaClasspath() string {
 	if len(*x) > 0 {

java/gen.go

@@ -85,7 +85,7 @@ func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Pat
 }
 
 func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
-	flags javaBuilderFlags) (android.Paths, android.Paths) {
+	flags javaBuilderFlags) (android.Paths, classpath) {
 
 	var protoFiles android.Paths
 	outSrcFiles := make(android.Paths, 0, len(srcFiles))
@@ -106,16 +106,17 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 		}
 	}
 
-	var outSrcFileLists android.Paths
+	var outSrcJars classpath
 
 	if len(protoFiles) > 0 {
-		protoFileList := genProto(ctx, protoFiles,
+		protoSrcJar := android.PathForModuleGen(ctx, "proto.src.jar")
+		genProto(ctx, protoSrcJar, protoFiles,
 			flags.protoFlags, flags.protoOutFlag, "")
 
-		outSrcFileLists = append(outSrcFileLists, protoFileList)
+		outSrcJars = append(outSrcJars, protoSrcJar)
 	}
 
-	return outSrcFiles, outSrcFileLists
+	return outSrcFiles, outSrcJars
 }
 
 func LogtagsSingleton() blueprint.Singleton {

java/java.go

@@ -169,9 +169,9 @@ type Module struct {
 
 	logtagsSrcs android.Paths
 
-	// filelists of extra source files that should be included in the javac command line,
+	// jars containing source files that should be included in the javac command line,
 	// for example R.java generated by aapt for android apps
-	ExtraSrcLists android.Paths
+	ExtraSrcJars android.Paths
 
 	// installed file for binary dependency
 	installFile android.Path
@@ -370,7 +370,7 @@ type deps struct {
 	staticJars         android.Paths
 	staticJarResources android.Paths
 	aidlIncludeDirs    android.Paths
-	srcFileLists       android.Paths
+	srcJars            android.Paths
 	systemModules      android.Path
 	aidlPreprocess     android.OptionalPath
 }
@@ -422,7 +422,7 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps {
 			if ctx.ModuleName() == "framework" {
 				// framework.jar has a one-off dependency on the R.java and Manifest.java files
 				// generated by framework-res.apk
-				deps.srcFileLists = append(deps.srcFileLists, module.(*AndroidApp).aaptJavaFileList)
+				// TODO(ccross): aapt java files should go in a src jar
 			}
 		default:
 			panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName()))
@@ -483,13 +483,12 @@ func (j *Module) compile(ctx android.ModuleContext) {
 		flags = protoFlags(ctx, &j.protoProperties, flags)
 	}
 
-	var srcFileLists android.Paths
+	var srcJars classpath
+	srcFiles, srcJars = j.genSources(ctx, srcFiles, flags)
 
-	srcFiles, srcFileLists = j.genSources(ctx, srcFiles, flags)
+	srcJars = append(srcJars, deps.srcJars...)
 
-	srcFileLists = append(srcFileLists, deps.srcFileLists...)
-
-	srcFileLists = append(srcFileLists, j.ExtraSrcLists...)
+	srcJars = append(srcJars, j.ExtraSrcJars...)
 
 	var jars android.Paths
 
@@ -502,13 +501,13 @@ func (j *Module) compile(ctx android.ModuleContext) {
 			// TODO(ccross): Once we always compile with javac9 we may be able to conditionally
 			//    enable error-prone without affecting the output class files.
 			errorprone := android.PathForModuleOut(ctx, "classes-errorprone.list")
-			RunErrorProne(ctx, errorprone, srcFiles, srcFileLists, flags)
+			RunErrorProne(ctx, errorprone, srcFiles, srcJars, flags)
 			extraJarDeps = append(extraJarDeps, errorprone)
 		}
 
 		// Compile java sources into .class files
 		classes := android.PathForModuleOut(ctx, "classes-compiled.jar")
-		TransformJavaToClasses(ctx, classes, srcFiles, srcFileLists, flags, extraJarDeps)
+		TransformJavaToClasses(ctx, classes, srcFiles, srcJars, flags, extraJarDeps)
 		if ctx.Failed() {
 			return
 		}

java/proto.go

@@ -30,20 +30,21 @@ var (
 		blueprint.RuleParams{
 			Command: `rm -rf $outDir && mkdir -p $outDir && ` +
 				`$protocCmd $protoOut=$protoOutFlags:$outDir $protoFlags $in && ` +
-				`find $outDir -name "*.java" > $out`,
-			CommandDeps: []string{"$protocCmd"},
+				`${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`,
+			CommandDeps: []string{
+				"$protocCmd",
+				"${config.SoongZipCmd}",
+			},
 		}, "protoFlags", "protoOut", "protoOutFlags", "outDir")
 )
 
-func genProto(ctx android.ModuleContext, protoFiles android.Paths,
-	protoFlags string, protoOut, protoOutFlags string) android.WritablePath {
-
-	protoFileList := android.PathForModuleGen(ctx, "proto.filelist")
+func genProto(ctx android.ModuleContext, outputSrcJar android.WritablePath,
+	protoFiles android.Paths, protoFlags string, protoOut, protoOutFlags string) {
 
 	ctx.ModuleBuild(pctx, android.ModuleBuildParams{
 		Rule:        proto,
 		Description: "protoc " + protoFiles[0].Rel(),
-		Output:      protoFileList,
+		Output:      outputSrcJar,
 		Inputs:      protoFiles,
 		Args: map[string]string{
 			"outDir":        android.ProtoDir(ctx).String(),
@@ -52,8 +53,6 @@ func genProto(ctx android.ModuleContext, protoFiles android.Paths,
 			"protoFlags":    protoFlags,
 		},
 	})
-
-	return protoFileList
 }
 
 func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) {