From 1028d5a53f003d497d9e850780f7d0c28729b343 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 19 Aug 2024 21:45:48 +0000 Subject: [PATCH 1/2] Create .kzip files for kotlin translation units This CL creates the build rules to invoke the standalone kotlin extractor and create .kzip files for each kotlin translation unit. These will be indexed by the internal g3 kotlin indexers Implementation details - Create a `kotlinKytheExtract` static rule. Its inputs will be source .kt/.srcjar files and other verbatim args passed to kotlinc. The extrator will serialize this information into a .kzip file. The .kzip file should contain the necessary information to "replay" the compilation - Create a xref_kotlin phony rule to build all .kzip files corresponding to kotlin translation units. This implementation has one limitation. Since the kotlin indexers "replay" the compilation using its own version of kotlinc-jvm, there might be indexing issues caused by kotlinc version skew between android builds and indexing builds. `-kotlin-home` can likely solve that, but this CL defers that for now. Bug: 265428637 Test: Built the phony `xref_kotlin`, and ran the indexer locally using go/kythe-git-local-run#indexing-to-local-xrefs-server Change-Id: Ifb370f2df8aa46f71df3fe1ae7d7aa5da773cc8b --- java/base.go | 5 +++-- java/config/config.go | 1 + java/java.go | 10 +++++++++ java/kotlin.go | 50 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/java/base.go b/java/base.go index a63533127..30485d0ca 100644 --- a/java/base.go +++ b/java/base.go @@ -538,7 +538,8 @@ type Module struct { linter // list of the xref extraction files - kytheFiles android.Paths + kytheFiles android.Paths + kytheKotlinFiles android.Paths hideApexVariantFromMake bool @@ -1372,7 +1373,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath kotlinJar := android.PathForModuleOut(ctx, "kotlin", jarName) kotlinHeaderJar := android.PathForModuleOut(ctx, "kotlin_headers", jarName) - kotlinCompile(ctx, kotlinJar, kotlinHeaderJar, uniqueSrcFiles, kotlinCommonSrcFiles, srcJars, flags) + j.kotlinCompile(ctx, kotlinJar, kotlinHeaderJar, uniqueSrcFiles, kotlinCommonSrcFiles, srcJars, flags) if ctx.Failed() { return } diff --git a/java/config/config.go b/java/config/config.go index a50c1b42d..ca712827f 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -144,6 +144,7 @@ func init() { pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod") pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar") pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar") + pctx.SourcePathVariable("KotlinKytheExtractor", "prebuilts/build-tools/${hostPrebuiltTag}/bin/kotlinc_extractor") pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime") pctx.SourcePathVariable("ResourceProcessorBusyBox", "prebuilts/bazel/common/android_tools/android_tools/all_android_tools_deploy.jar") diff --git a/java/java.go b/java/java.go index 797a90dfc..99f44374f 100644 --- a/java/java.go +++ b/java/java.go @@ -356,12 +356,17 @@ type UsesLibraryDependency interface { // TODO(jungjw): Move this to kythe.go once it's created. type xref interface { XrefJavaFiles() android.Paths + XrefKotlinFiles() android.Paths } func (j *Module) XrefJavaFiles() android.Paths { return j.kytheFiles } +func (j *Module) XrefKotlinFiles() android.Paths { + return j.kytheKotlinFiles +} + func (d dependencyTag) PropagateAconfigValidation() bool { return d.static } @@ -3304,15 +3309,20 @@ type kytheExtractJavaSingleton struct { func (ks *kytheExtractJavaSingleton) GenerateBuildActions(ctx android.SingletonContext) { var xrefTargets android.Paths + var xrefKotlinTargets android.Paths ctx.VisitAllModules(func(module android.Module) { if javaModule, ok := module.(xref); ok { xrefTargets = append(xrefTargets, javaModule.XrefJavaFiles()...) + xrefKotlinTargets = append(xrefKotlinTargets, javaModule.XrefKotlinFiles()...) } }) // TODO(asmundak): perhaps emit a rule to output a warning if there were no xrefTargets if len(xrefTargets) > 0 { ctx.Phony("xref_java", xrefTargets...) } + if len(xrefKotlinTargets) > 0 { + ctx.Phony("xref_kotlin", xrefKotlinTargets...) + } } var Bool = proptools.Bool diff --git a/java/kotlin.go b/java/kotlin.go index c28bc3f54..79781a06d 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -64,6 +64,29 @@ var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.RemoteRuleSupports "kotlincFlags", "classpath", "srcJars", "commonSrcFilesArg", "srcJarDir", "classesDir", "headerClassesDir", "headerJar", "kotlinJvmTarget", "kotlinBuildFile", "emptyDir", "name") +var kotlinKytheExtract = pctx.AndroidStaticRule("kotlinKythe", + // TODO (b/265428637): To prevent kotlinc version skew between android builds and internal kotlin indexers (g3), consider embedding the kotlinc used by android into the kzip file. + // This has an impact on .kzip sizes, so defer that for now. + blueprint.RuleParams{ + Command: `rm -rf "$srcJarDir" && mkdir -p "$srcJarDir" && ` + + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" -f "*.kt" $srcJars && ` + + `${config.KotlinKytheExtractor} -corpus ${kytheCorpus} --srcs @$out.rsp --srcs @"$srcJarDir/list" $commonSrcFilesList --cp @$classpath -o $out --kotlin_out $outJar ` + + // wrap the additional kotlin args. + // Skip Xbuild file, pass the cp explicitly. + // Skip header jars, those should not have an effect on kythe results. + ` --args '${config.KotlincGlobalFlags} ` + + ` ${config.KotlincSuppressJDK9Warnings} ${config.JavacHeapFlags} ` + + ` $kotlincFlags -jvm-target $kotlinJvmTarget'`, + CommandDeps: []string{ + "${config.KotlinKytheExtractor}", + "${config.ZipSyncCmd}", + }, + Rspfile: "$out.rsp", + RspfileContent: "$in", + }, + "classpath", "kotlincFlags", "commonSrcFilesList", "kotlinJvmTarget", "outJar", "srcJars", "srcJarDir", +) + func kotlinCommonSrcsList(ctx android.ModuleContext, commonSrcFiles android.Paths) android.OptionalPath { if len(commonSrcFiles) > 0 { // The list of common_srcs may be too long to put on the command line, but @@ -81,7 +104,7 @@ func kotlinCommonSrcsList(ctx android.ModuleContext, commonSrcFiles android.Path } // kotlinCompile takes .java and .kt sources and srcJars, and compiles the .kt sources into a classes jar in outputFile. -func kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile android.WritablePath, +func (j *Module) kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile android.WritablePath, srcFiles, commonSrcFiles, srcJars android.Paths, flags javaBuilderFlags) { @@ -127,6 +150,31 @@ func kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile andro "name": kotlinName, }, }) + + // Emit kythe xref rule + if (ctx.Config().EmitXrefRules()) && ctx.Module() == ctx.PrimaryModule() { + extractionFile := outputFile.ReplaceExtension(ctx, "kzip") + args := map[string]string{ + "classpath": classpathRspFile.String(), + "kotlincFlags": flags.kotlincFlags, + "kotlinJvmTarget": flags.javaVersion.StringForKotlinc(), + "outJar": outputFile.String(), + "srcJars": strings.Join(srcJars.Strings(), " "), + "srcJarDir": android.PathForModuleOut(ctx, "kotlinc", "srcJars.xref").String(), + } + if commonSrcsList.Valid() { + args["commonSrcFilesList"] = "--srcs @" + commonSrcsList.String() + } + ctx.Build(pctx, android.BuildParams{ + Rule: kotlinKytheExtract, + Description: "kotlinKythe", + Output: extractionFile, + Inputs: srcFiles, + Implicits: deps, + Args: args, + }) + j.kytheKotlinFiles = append(j.kytheKotlinFiles, extractionFile) + } } var kaptStubs = pctx.AndroidRemoteStaticRule("kaptStubs", android.RemoteRuleSupports{Goma: true}, From 600afbe381409e77654f73672491492f4c2aeecd Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 4 Sep 2024 23:49:27 +0000 Subject: [PATCH 2/2] Add kotlin kzips to build_kzip.bash Test: Built the phony xref_kotlin target locally Bug: 265428637 Change-Id: Ib5d6679f17ea45beea8302bc2257e1e4bd0ed7eb --- build_kzip.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/build_kzip.bash b/build_kzip.bash index 4c42048dd..850aedaf0 100755 --- a/build_kzip.bash +++ b/build_kzip.bash @@ -40,6 +40,7 @@ kzip_targets=( merge_zips xref_cxx xref_java + xref_kotlin # TODO: b/286390153 - reenable rust # xref_rust )