Merge "Support source code cross-referencing for C++ and Java"

android/config.go

@@ -756,6 +756,14 @@ func (c *config) RunErrorProne() bool {
 	return c.IsEnvTrue("RUN_ERROR_PRONE")
 }
 
+func (c *config) XrefCorpusName() string {
+	return c.Getenv("XREF_CORPUS")
+}
+
+func (c *config) EmitXrefRules() bool {
+	return c.XrefCorpusName() != ""
+}
+
 // Returns true if -source 1.9 -target 1.9 is being passed to javac
 func (c *config) TargetOpenJDK9() bool {
 	return c.targetOpenJDK9

cc/builder.go

@@ -221,6 +221,17 @@ var (
 			Rspfile:        "$out.rsp",
 			RspfileContent: "$in",
 		})
+
+	_ = pctx.SourcePathVariable("cxxExtractor",
+		"prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/cxx_extractor")
+	_ = pctx.VariableFunc("kytheCorpus",
+		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
+	kytheExtract = pctx.StaticRule("kythe",
+		blueprint.RuleParams{
+			Command:     "rm -f $out && KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out $cxxExtractor $cFlags $in ",
+			CommandDeps: []string{"$cxxExtractor"},
+		},
+		"cFlags")
 )
 
 func init() {
@@ -257,6 +268,7 @@ type builderFlags struct {
 	tidy            bool
 	coverage        bool
 	sAbiDump        bool
+	emitXrefs       bool
 
 	systemIncludeFlags string
 
@@ -281,6 +293,7 @@ type Objects struct {
 	tidyFiles     android.Paths
 	coverageFiles android.Paths
 	sAbiDumpFiles android.Paths
+	kytheFiles    android.Paths
 }
 
 func (a Objects) Copy() Objects {
@@ -289,6 +302,7 @@ func (a Objects) Copy() Objects {
 		tidyFiles:     append(android.Paths{}, a.tidyFiles...),
 		coverageFiles: append(android.Paths{}, a.coverageFiles...),
 		sAbiDumpFiles: append(android.Paths{}, a.sAbiDumpFiles...),
+		kytheFiles:    append(android.Paths{}, a.kytheFiles...),
 	}
 }
 
@@ -298,6 +312,7 @@ func (a Objects) Append(b Objects) Objects {
 		tidyFiles:     append(a.tidyFiles, b.tidyFiles...),
 		coverageFiles: append(a.coverageFiles, b.coverageFiles...),
 		sAbiDumpFiles: append(a.sAbiDumpFiles, b.sAbiDumpFiles...),
+		kytheFiles:    append(a.kytheFiles, b.kytheFiles...),
 	}
 }
 
@@ -314,6 +329,10 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and
 	if flags.coverage {
 		coverageFiles = make(android.Paths, 0, len(srcFiles))
 	}
+	var kytheFiles android.Paths
+	if flags.emitXrefs {
+		kytheFiles = make(android.Paths, 0, len(srcFiles))
+	}
 
 	commonFlags := strings.Join([]string{
 		flags.globalFlags,
@@ -401,6 +420,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and
 		coverage := flags.coverage
 		dump := flags.sAbiDump
 		rule := cc
+		emitXref := flags.emitXrefs
 
 		switch srcFile.Ext() {
 		case ".s":
@@ -412,6 +432,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and
 			tidy = false
 			coverage = false
 			dump = false
+			emitXref = false
 		case ".c":
 			ccCmd = "clang"
 			moduleCflags = cflags
@@ -450,6 +471,22 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and
 			},
 		})
 
+		if emitXref {
+			kytheFile := android.ObjPathWithExt(ctx, subdir, srcFile, "kzip")
+			ctx.Build(pctx, android.BuildParams{
+				Rule:        kytheExtract,
+				Description: "Xref C++ extractor " + srcFile.Rel(),
+				Output:      kytheFile,
+				Input:       srcFile,
+				Implicits:   cFlagsDeps,
+				OrderOnly:   pathDeps,
+				Args: map[string]string{
+					"cFlags": moduleCflags,
+				},
+			})
+			kytheFiles = append(kytheFiles, kytheFile)
+		}
+
 		if tidy {
 			tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy")
 			tidyFiles = append(tidyFiles, tidyFile)
@@ -493,6 +530,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and
 		tidyFiles:     tidyFiles,
 		coverageFiles: coverageFiles,
 		sAbiDumpFiles: sAbiDumpFiles,
+		kytheFiles:    kytheFiles,
 	}
 }
 

cc/cc.go

@@ -77,6 +77,7 @@ func init() {
 		ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel()
 	})
 
+	android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory)
 	pctx.Import("android/soong/cc/config")
 }
 
@@ -162,6 +163,7 @@ type Flags struct {
 	Tidy      bool
 	Coverage  bool
 	SAbiDump  bool
+	EmitXrefs bool // If true, generate Ninja rules to generate emitXrefs input files for Kythe
 
 	RequiredInstructionSet string
 	DynamicLinker          string
@@ -346,6 +348,10 @@ type dependencyTag struct {
 	explicitlyVersioned bool
 }
 
+type xref interface {
+	XrefCcFiles() android.Paths
+}
+
 var (
 	sharedDepTag          = dependencyTag{name: "shared", library: true}
 	sharedExportDepTag    = dependencyTag{name: "shared", library: true, reexportFlags: true}
@@ -427,6 +433,8 @@ type Module struct {
 	staticVariant *Module
 
 	makeLinkType string
+	// Kythe (source file indexer) paths for this compilation module
+	kytheFiles android.Paths
 }
 
 func (c *Module) OutputFile() android.OptionalPath {
@@ -657,6 +665,10 @@ func installToBootstrap(name string, config android.Config) bool {
 	return isBionic(name)
 }
 
+func (c *Module) XrefCcFiles() android.Paths {
+	return c.kytheFiles
+}
+
 type baseModuleContext struct {
 	android.BaseModuleContext
 	moduleContextImpl
@@ -995,6 +1007,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
 
 	flags := Flags{
 		Toolchain: c.toolchain(ctx),
+		EmitXrefs: ctx.Config().EmitXrefRules(),
 	}
 	if c.compiler != nil {
 		flags = c.compiler.compilerFlags(ctx, flags, deps)
@@ -1060,6 +1073,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
 		if ctx.Failed() {
 			return
 		}
+		c.kytheFiles = objs.kytheFiles
 	}
 
 	if c.linker != nil {
@@ -2366,6 +2380,31 @@ func getCurrentNdkPrebuiltVersion(ctx DepsContext) string {
 	return ctx.Config().PlatformSdkVersion()
 }
 
+func kytheExtractAllFactory() android.Singleton {
+	return &kytheExtractAllSingleton{}
+}
+
+type kytheExtractAllSingleton struct {
+}
+
+func (ks *kytheExtractAllSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+	var xrefTargets android.Paths
+	ctx.VisitAllModules(func(module android.Module) {
+		if ccModule, ok := module.(xref); ok {
+			xrefTargets = append(xrefTargets, ccModule.XrefCcFiles()...)
+		}
+	})
+	// TODO(asmundak): Perhaps emit a rule to output a warning if there were no xrefTargets
+	if len(xrefTargets) > 0 {
+		ctx.Build(pctx, android.BuildParams{
+			Rule:   blueprint.Phony,
+			Output: android.PathForPhony(ctx, "xref_cxx"),
+			Inputs: xrefTargets,
+			//Default: true,
+		})
+	}
+}
+
 var Bool = proptools.Bool
 var BoolDefault = proptools.BoolDefault
 var BoolPtr = proptools.BoolPtr

cc/util.go

@@ -74,6 +74,7 @@ func flagsToBuilderFlags(in Flags) builderFlags {
 		coverage:        in.Coverage,
 		tidy:            in.Tidy,
 		sAbiDump:        in.SAbiDump,
+		emitXrefs:       in.EmitXrefs,
 
 		systemIncludeFlags: strings.Join(in.SystemIncludeFlags, " "),
 

java/builder.go

@@ -62,6 +62,37 @@ var (
 		"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",
 		"outDir", "annoDir", "javaVersion")
 
+	_ = pctx.VariableFunc("kytheCorpus",
+		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
+	// Run it with -add-opens=java.base/java.nio=ALL-UNNAMED to avoid JDK9's warning about
+	// "Illegal reflective access by com.google.protobuf.Utf8$UnsafeProcessor ...
+	// to field java.nio.Buffer.address"
+	kytheExtract = pctx.AndroidStaticRule("kythe",
+		blueprint.RuleParams{
+			Command: `${config.ZipSyncCmd} -d $srcJarDir ` +
+				`-l $srcJarDir/list -f "*.java" $srcJars && ` +
+				`( [ ! -s $srcJarDir/list -a ! -s $out.rsp ] || ` +
+				`KYTHE_ROOT_DIRECTORY=. KYTHE_OUTPUT_FILE=$out ` +
+				`KYTHE_CORPUS=${kytheCorpus} ` +
+				`${config.SoongJavacWrapper} ${config.JavaCmd} ` +
+				`--add-opens=java.base/java.nio=ALL-UNNAMED ` +
+				`-jar ${config.JavaKytheExtractorJar} ` +
+				`${config.JavacHeapFlags} ${config.CommonJdkFlags} ` +
+				`$processorpath $processor $javacFlags $bootClasspath $classpath ` +
+				`-source $javaVersion -target $javaVersion ` +
+				`-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list)`,
+			CommandDeps: []string{
+				"${config.JavaCmd}",
+				"${config.JavaKytheExtractorJar}",
+				"${config.ZipSyncCmd}",
+			},
+			CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},
+			Rspfile:          "$out.rsp",
+			RspfileContent:   "$in",
+		},
+		"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",
+		"outDir", "annoDir", "javaVersion")
+
 	turbine = pctx.AndroidStaticRule("turbine",
 		blueprint.RuleParams{
 			Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
@@ -196,6 +227,61 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
 		"errorprone", "errorprone")
 }
 
+// Emits the rule to generate Xref input file (.kzip file) for the given set of source files and source jars
+// to compile with given set of builder flags, etc.
+func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath,
+	srcFiles, srcJars android.Paths,
+	flags javaBuilderFlags, deps android.Paths,
+	intermediatesDir string) {
+
+	deps = append(deps, srcJars...)
+
+	var bootClasspath string
+	if flags.javaVersion == "1.9" {
+		var systemModuleDeps android.Paths
+		bootClasspath, systemModuleDeps = flags.systemModules.FormJavaSystemModulesPath(ctx.Device())
+		deps = append(deps, systemModuleDeps...)
+	} else {
+		deps = append(deps, flags.bootClasspath...)
+		if len(flags.bootClasspath) == 0 && ctx.Device() {
+			// explicitly specify -bootclasspath "" if the bootclasspath is empty to
+			// ensure java does not fall back to the default bootclasspath.
+			bootClasspath = `-bootclasspath ""`
+		} else {
+			bootClasspath = flags.bootClasspath.FormJavaClassPath("-bootclasspath")
+		}
+	}
+
+	deps = append(deps, flags.classpath...)
+	deps = append(deps, flags.processorPath...)
+
+	processor := "-proc:none"
+	if flags.processor != "" {
+		processor = "-processor " + flags.processor
+	}
+
+	ctx.Build(pctx,
+		android.BuildParams{
+			Rule:        kytheExtract,
+			Description: "Xref Java extractor",
+			Output:      xrefFile,
+			Inputs:      srcFiles,
+			Implicits:   deps,
+			Args: map[string]string{
+				"annoDir":       android.PathForModuleOut(ctx, intermediatesDir, "anno").String(),
+				"bootClasspath": bootClasspath,
+				"classpath":     flags.classpath.FormJavaClassPath("-classpath"),
+				"javacFlags":    flags.javacFlags,
+				"javaVersion":   flags.javaVersion,
+				"outDir":        android.PathForModuleOut(ctx, "javac", "classes.xref").String(),
+				"processorpath": flags.processorPath.FormJavaClassPath("-processorpath"),
+				"processor":     processor,
+				"srcJarDir":     android.PathForModuleOut(ctx, intermediatesDir, "srcjars.xref").String(),
+				"srcJars":       strings.Join(srcJars.Strings(), " "),
+			},
+		})
+}
+
 func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath,
 	srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
 

java/config/config.go

@@ -94,6 +94,7 @@ func init() {
 	pctx.SourcePathVariable("JlinkCmd", "${JavaToolchain}/jlink")
 	pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod")
 	pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar")
+	pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar")
 	pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime")
 
 	pctx.SourcePathVariable("GenKotlinBuildFileCmd", "build/soong/scripts/gen-kotlin-build-file.sh")

java/java.go

@@ -50,6 +50,7 @@ func init() {
 	android.RegisterModuleType("dex_import", DexImportFactory)
 
 	android.RegisterSingletonType("logtags", LogtagsSingleton)
+	android.RegisterSingletonType("kythe_java_extract", kytheExtractJavaFactory)
 }
 
 // TODO:
@@ -343,6 +344,9 @@ type Module struct {
 
 	hiddenAPI
 	dexpreopter
+
+	// list of the xref extraction files
+	kytheFiles android.Paths
 }
 
 func (j *Module) OutputFiles(tag string) (android.Paths, error) {
@@ -383,6 +387,10 @@ type SrcDependency interface {
 	CompiledSrcJars() android.Paths
 }
 
+type xref interface {
+	XrefJavaFiles() android.Paths
+}
+
 func (j *Module) CompiledSrcs() android.Paths {
 	return j.compiledJavaSrcs
 }
@@ -391,6 +399,10 @@ func (j *Module) CompiledSrcJars() android.Paths {
 	return j.compiledSrcJars
 }
 
+func (j *Module) XrefJavaFiles() android.Paths {
+	return j.kytheFiles
+}
+
 var _ SrcDependency = (*Module)(nil)
 
 func InitJavaModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) {
@@ -1139,6 +1151,12 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) {
 			TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps)
 			jars = append(jars, classes)
 		}
+		if ctx.Config().EmitXrefRules() {
+			extractionFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".kzip")
+			emitXrefRule(ctx, extractionFile, uniqueSrcFiles, srcJars, flags, extraJarDeps, "xref")
+			j.kytheFiles = append(j.kytheFiles, extractionFile)
+
+		}
 		if ctx.Failed() {
 			return
 		}
@@ -2213,6 +2231,30 @@ func DefaultsFactory(props ...interface{}) android.Module {
 	return module
 }
 
+func kytheExtractJavaFactory() android.Singleton {
+	return &kytheExtractJavaSingleton{}
+}
+
+type kytheExtractJavaSingleton struct {
+}
+
+func (ks *kytheExtractJavaSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+	var xrefTargets android.Paths
+	ctx.VisitAllModules(func(module android.Module) {
+		if javaModule, ok := module.(xref); ok {
+			xrefTargets = append(xrefTargets, javaModule.XrefJavaFiles()...)
+		}
+	})
+	// TODO(asmundak): perhaps emit a rule to output a warning if there were no xrefTargets
+	if len(xrefTargets) > 0 {
+		ctx.Build(pctx, android.BuildParams{
+			Rule:   blueprint.Phony,
+			Output: android.PathForPhony(ctx, "xref_java"),
+			Inputs: xrefTargets,
+		})
+	}
+}
+
 var Bool = proptools.Bool
 var BoolDefault = proptools.BoolDefault
 var String = proptools.String