Add cross-referencing support for Rust

Bug: 222044478
Test: run TARGET_BUILD_VARIANT=userdebug TARGET_PRODUCT=cf_x86_64_phone XREF_CORPUS=googleplex-android.googlesource.com/codesearch/android build/soong/build_kzip.bash
Change-Id: Ia12eed53fafd0cbbbf2cc499fa20a0f5a40031de

build_kzip.bash

@@ -36,7 +36,7 @@ export KYTHE_JAVA_SOURCE_BATCH_SIZE KYTHE_KZIP_ENCODING
 declare -r out="${OUT_DIR:-out}"
 
 # Build extraction files for C++ and Java. Build `merge_zips` which we use later.
-build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java
+build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java xref_rust
 
 # Build extraction file for Go the files in build/{blueprint,soong} directories.
 declare -r abspath_out=$(realpath "${out}")
@@ -44,7 +44,7 @@ declare -r go_extractor=$(realpath prebuilts/build-tools/linux-x86/bin/go_extrac
 declare -r go_root=$(realpath prebuilts/go/linux-x86)
 declare -r source_root=$PWD
 
-# TODO(asmundak): Until b/182183061 is fixed, default corpus has to be specified 
+# TODO(asmundak): Until b/182183061 is fixed, default corpus has to be specified
 # in the rules file. Generate this file on the fly with corpus value set from the
 # environment variable.
 for dir in blueprint soong; do

rust/binary.go

@@ -128,11 +128,11 @@ func (binary *binaryDecorator) preferRlib() bool {
 	return Bool(binary.baseCompiler.Properties.Prefer_rlib) || Bool(binary.Properties.Static_executable)
 }
 
-func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
 	fileName := binary.getStem(ctx) + ctx.toolchain().ExecutableSuffix()
 	srcPath, _ := srcPathFromModuleSrcs(ctx, binary.baseCompiler.Properties.Srcs)
 	outputFile := android.PathForModuleOut(ctx, fileName)
-	ret := outputFile
+	ret := buildOutput{outputFile: outputFile}
 
 	flags.RustFlags = append(flags.RustFlags, deps.depFlags...)
 	flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...)
@@ -147,8 +147,7 @@ func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps Path
 	}
 	binary.baseCompiler.unstrippedOutputFile = outputFile
 
-	TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile)
+	ret.kytheFile = TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile).kytheFile
-
 	return ret
 }
 

rust/builder.go

@@ -83,10 +83,37 @@ var (
 			RspfileContent: "$in",
 		},
 		"outDir")
+
+	// Cross-referencing:
+	_ = pctx.SourcePathVariable("rustExtractor",
+		"prebuilts/build-tools/${config.HostPrebuiltTag}/bin/rust_extractor")
+	_ = pctx.VariableFunc("kytheCorpus",
+		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
+	_ = pctx.VariableFunc("kytheCuEncoding",
+		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCuEncoding() })
+	_            = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json")
+	kytheExtract = pctx.AndroidStaticRule("kythe",
+		blueprint.RuleParams{
+			Command: `KYTHE_CORPUS=${kytheCorpus} ` +
+				`KYTHE_OUTPUT_FILE=$out ` +
+				`KYTHE_VNAMES=$kytheVnames ` +
+				`KYTHE_KZIP_ENCODING=${kytheCuEncoding} ` +
+				`KYTHE_CANONICALIZE_VNAME_PATHS=prefer-relative ` +
+				`$rustExtractor $envVars ` +
+				`$rustcCmd ` +
+				`-C linker=${config.RustLinker} ` +
+				`-C link-args="${crtBegin} ${config.RustLinkerArgs} ${linkFlags} ${crtEnd}" ` +
+				`$in ${libFlags} $rustcFlags`,
+			CommandDeps:    []string{"$rustExtractor", "$kytheVnames"},
+			Rspfile:        "${out}.rsp",
+			RspfileContent: "$in",
+		},
+		"rustcFlags", "linkFlags", "libFlags", "crtBegin", "crtEnd", "envVars")
 )
 
 type buildOutput struct {
 	outputFile android.Path
+	kytheFile  android.Path
 }
 
 func init() {
@@ -324,6 +351,25 @@ func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, fl
 		},
 	})
 
+	if flags.EmitXrefs {
+		kytheFile := android.PathForModuleOut(ctx, outputFile.Base()+".kzip")
+		ctx.Build(pctx, android.BuildParams{
+			Rule:        kytheExtract,
+			Description: "Xref Rust extractor " + main.Rel(),
+			Output:      kytheFile,
+			Inputs:      inputs,
+			Implicits:   implicits,
+			Args: map[string]string{
+				"rustcFlags": strings.Join(rustcFlags, " "),
+				"linkFlags":  strings.Join(linkFlags, " "),
+				"libFlags":   strings.Join(libFlags, " "),
+				"crtBegin":   strings.Join(deps.CrtBegin.Strings(), " "),
+				"crtEnd":     strings.Join(deps.CrtEnd.Strings(), " "),
+				"envVars":    strings.Join(envVars, " "),
+			},
+		})
+		output.kytheFile = kytheFile
+	}
 	return output
 }
 

rust/compiler.go

@@ -304,6 +304,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag
 	flags.GlobalRustFlags = append(flags.GlobalRustFlags, config.GlobalRustFlags...)
 	flags.GlobalRustFlags = append(flags.GlobalRustFlags, ctx.toolchain().ToolchainRustFlags())
 	flags.GlobalLinkFlags = append(flags.GlobalLinkFlags, ctx.toolchain().ToolchainLinkFlags())
+	flags.EmitXrefs = ctx.Config().EmitXrefRules()
 
 	if ctx.Host() && !ctx.Windows() {
 		rpathPrefix := `\$$ORIGIN/`
@@ -324,7 +325,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag
 	return flags
 }
 
-func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
 	panic(fmt.Errorf("baseCrater doesn't know how to crate things!"))
 }
 

rust/library.go

@@ -474,8 +474,9 @@ func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags) F
 	return flags
 }
 
-func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
-	var outputFile, ret android.ModuleOutPath
+	var outputFile android.ModuleOutPath
+	var ret buildOutput
 	var fileName string
 	srcPath := library.srcPath(ctx, deps)
 
@@ -487,19 +488,19 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa
 	if library.rlib() {
 		fileName = library.getStem(ctx) + ctx.toolchain().RlibSuffix()
 		outputFile = android.PathForModuleOut(ctx, fileName)
-		ret = outputFile
+		ret.outputFile = outputFile
 	} else if library.dylib() {
 		fileName = library.getStem(ctx) + ctx.toolchain().DylibSuffix()
 		outputFile = android.PathForModuleOut(ctx, fileName)
-		ret = outputFile
+		ret.outputFile = outputFile
 	} else if library.static() {
 		fileName = library.getStem(ctx) + ctx.toolchain().StaticLibSuffix()
 		outputFile = android.PathForModuleOut(ctx, fileName)
-		ret = outputFile
+		ret.outputFile = outputFile
 	} else if library.shared() {
 		fileName = library.sharedLibFilename(ctx)
 		outputFile = android.PathForModuleOut(ctx, fileName)
-		ret = outputFile
+		ret.outputFile = outputFile
 	}
 
 	if !library.rlib() && !library.static() && library.stripper.NeedsStrip(ctx) {
@@ -524,13 +525,13 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa
 
 	// Call the appropriate builder for this library type
 	if library.rlib() {
-		TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile)
+		ret.kytheFile = TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile).kytheFile
 	} else if library.dylib() {
-		TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile)
+		ret.kytheFile = TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile).kytheFile
 	} else if library.static() {
-		TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile)
+		ret.kytheFile = TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile).kytheFile
 	} else if library.shared() {
-		TransformSrctoShared(ctx, srcPath, deps, flags, outputFile)
+		ret.kytheFile = TransformSrctoShared(ctx, srcPath, deps, flags, outputFile).kytheFile
 	}
 
 	if library.rlib() || library.dylib() {
@@ -572,7 +573,7 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa
 	return ret
 }
 
-func (library *libraryDecorator) srcPath(ctx ModuleContext, deps PathDeps) android.Path {
+func (library *libraryDecorator) srcPath(ctx ModuleContext, _ PathDeps) android.Path {
 	if library.sourceProvider != nil {
 		// Assume the first source from the source provider is the library entry point.
 		return library.sourceProvider.Srcs()[0]

rust/prebuilt.go

@@ -145,7 +145,7 @@ func (prebuilt *prebuiltLibraryDecorator) compilerProps() []interface{} {
 		&prebuilt.Properties)
 }
 
-func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
 	prebuilt.flagExporter.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...)
 	prebuilt.flagExporter.setProvider(ctx)
 
@@ -154,7 +154,7 @@ func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags
 		ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)")
 	}
 	prebuilt.baseCompiler.unstrippedOutputFile = srcPath
-	return srcPath
+	return buildOutput{outputFile: srcPath}
 }
 
 func (prebuilt *prebuiltLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags,
@@ -202,7 +202,7 @@ func (prebuilt *prebuiltProcMacroDecorator) compilerProps() []interface{} {
 		&prebuilt.Properties)
 }
 
-func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
 	prebuilt.flagExporter.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...)
 	prebuilt.flagExporter.setProvider(ctx)
 
@@ -211,7 +211,7 @@ func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Fla
 		ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)")
 	}
 	prebuilt.baseCompiler.unstrippedOutputFile = srcPath
-	return srcPath
+	return buildOutput{outputFile: srcPath}
 }
 
 func (prebuilt *prebuiltProcMacroDecorator) rustdoc(ctx ModuleContext, flags Flags,

rust/proc_macro.go

@@ -70,14 +70,14 @@ func (procMacro *procMacroDecorator) compilerFlags(ctx ModuleContext, flags Flag
 	return flags
 }
 
-func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
 	fileName := procMacro.getStem(ctx) + ctx.toolchain().ProcMacroSuffix()
 	outputFile := android.PathForModuleOut(ctx, fileName)
 
 	srcPath, _ := srcPathFromModuleSrcs(ctx, procMacro.baseCompiler.Properties.Srcs)
-	TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile)
+	ret := TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile)
 	procMacro.baseCompiler.unstrippedOutputFile = outputFile
-	return outputFile
+	return ret
 }
 
 func (procMacro *procMacroDecorator) getStem(ctx ModuleContext) string {

rust/rust.go

@@ -15,6 +15,7 @@
 package rust
 
 import (
+	"android/soong/bloaty"
 	"fmt"
 	"strings"
 
@@ -22,7 +23,6 @@ import (
 	"github.com/google/blueprint/proptools"
 
 	"android/soong/android"
-	"android/soong/bloaty"
 	"android/soong/cc"
 	cc_config "android/soong/cc/config"
 	"android/soong/fuzz"
@@ -52,6 +52,7 @@ func init() {
 	})
 	pctx.Import("android/soong/rust/config")
 	pctx.ImportAs("cc_config", "android/soong/cc/config")
+	android.InitRegistrationContext.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory)
 }
 
 type Flags struct {
@@ -64,6 +65,7 @@ type Flags struct {
 	Toolchain       config.Toolchain
 	Coverage        bool
 	Clippy          bool
+	EmitXrefs       bool // If true, emit rules to aid cross-referencing
 }
 
 type BaseProperties struct {
@@ -161,6 +163,9 @@ type Module struct {
 	// Output file to be installed, may be stripped or unstripped.
 	outputFile android.OptionalPath
 
+	// Cross-reference input file
+	kytheFiles android.Paths
+
 	docTimestampFile android.OptionalPath
 
 	hideApexVariantFromMake bool
@@ -394,6 +399,10 @@ func (mod *Module) SplitPerApiLevel() bool {
 	return false
 }
 
+func (mod *Module) XrefRustFiles() android.Paths {
+	return mod.kytheFiles
+}
+
 type Deps struct {
 	Dylibs          []string
 	Rlibs           []string
@@ -457,7 +466,7 @@ type compiler interface {
 	cfgFlags(ctx ModuleContext, flags Flags) Flags
 	featureFlags(ctx ModuleContext, flags Flags) Flags
 	compilerProps() []interface{}
-	compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path
+	compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput
 	compilerDeps(ctx DepsContext, deps Deps) Deps
 	crateName() string
 	rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath
@@ -493,6 +502,10 @@ type exportedFlagsProducer interface {
 	exportLinkObjects(...string)
 }
 
+type xref interface {
+	XrefRustFiles() android.Paths
+}
+
 type flagExporter struct {
 	linkDirs    []string
 	linkObjects []string
@@ -904,11 +917,14 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
 
 	if mod.compiler != nil && !mod.compiler.Disabled() {
 		mod.compiler.initialize(ctx)
-		outputFile := mod.compiler.compile(ctx, flags, deps)
+		buildOutput := mod.compiler.compile(ctx, flags, deps)
 		if ctx.Failed() {
 			return
 		}
-		mod.outputFile = android.OptionalPathForPath(outputFile)
+		mod.outputFile = android.OptionalPathForPath(buildOutput.outputFile)
+		if buildOutput.kytheFile != nil {
+			mod.kytheFiles = append(mod.kytheFiles, buildOutput.kytheFile)
+		}
 		bloaty.MeasureSizeForPaths(ctx, mod.compiler.strippedOutputFilePath(), android.OptionalPathForPath(mod.compiler.unstrippedOutputFilePath()))
 
 		mod.docTimestampFile = mod.compiler.rustdoc(ctx, flags, deps)
@@ -1618,6 +1634,25 @@ func libNameFromFilePath(filepath android.Path) (string, bool) {
 	return "", false
 }
 
+func kytheExtractRustFactory() android.Singleton {
+	return &kytheExtractRustSingleton{}
+}
+
+type kytheExtractRustSingleton struct {
+}
+
+func (k kytheExtractRustSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+	var xrefTargets android.Paths
+	ctx.VisitAllModules(func(module android.Module) {
+		if rustModule, ok := module.(xref); ok {
+			xrefTargets = append(xrefTargets, rustModule.XrefRustFiles()...)
+		}
+	})
+	if len(xrefTargets) > 0 {
+		ctx.Phony("xref_rust", xrefTargets...)
+	}
+}
+
 var Bool = proptools.Bool
 var BoolDefault = proptools.BoolDefault
 var String = proptools.String

rust/snapshot_prebuilt.go

@@ -69,7 +69,7 @@ func snapshotLibraryFactory(image cc.SnapshotImage, moduleSuffix string) (*Modul
 	return module, prebuilt
 }
 
-func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
+func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput {
 	var variant string
 	if library.static() {
 		variant = cc.SnapshotStaticSuffix
@@ -85,11 +85,11 @@ func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags,
 	}
 
 	if !library.MatchesWithDevice(ctx.DeviceConfig()) {
-		return nil
+		return buildOutput{}
 	}
 	outputFile := android.PathForModuleSrc(ctx, *library.properties.Src)
 	library.unstrippedOutputFile = outputFile
-	return outputFile
+	return buildOutput{outputFile: outputFile}
 }
 
 func (library *snapshotLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath {

rust/testing.go

@@ -194,6 +194,7 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) {
 		ctx.BottomUp("rust_begin", BeginMutator).Parallel()
 	})
 	ctx.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton)
+	ctx.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory)
 	ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) {
 		ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel()
 	})