Pass filename mappings to C++ and Java extractors.

Android builds by default put artifacts into out/ subdirectory of
the source tree, causing the extractor to record their names as
relative. The indexer considers such files as sources, which is wrong.
Fortunately, the extractor can be fed a set of filename rewriting
rules (see build/tools/vnames.json).
Also, undo previous unsuccessful attempt use to absolute path for the
output directory to distinguish between source code and artifacts.

Bug: 141385476
Test: run the build, inspect compilation units of the kzip file
Change-Id: I89ec3aed8fd14f43ea6e0b226d54f643346f6125

build_kzip.bash

@@ -1,33 +1,27 @@
-# /bin/bash -uv
+#! /bin/bash -uv
 #
 # Build kzip files (source files for the indexing pipeline) for the given configuration,
 # merge them and place the resulting all.kzip into $DIST_DIR.
 # It is assumed that the current directory is the top of the source tree.
 # The following environment variables affect the result:
-#   TARGET_PRODUCT        target device name, e.g., 'aosp_blueline'
-#   TARGET_BUILD_VARIANT  variant, e.g., `userdebug`
-#   OUT_DIR               absolute path  where the build is happening ($PWD/out if not specified})
-#   DIST_DIR              where the resulting all.kzip will be placed
-#   XREF_CORPUS           source code repository URI, e.g., 'android.googlesource.com/platform/superproject'
 #   BUILD_NUMBER          build number, used to generate unique ID (will use UUID if not set)
+#   DIST_DIR              where the resulting all.kzip will be placed
+#   OUT_DIR               output directory (out if not specified})
+#   TARGET_BUILD_VARIANT  variant, e.g., `userdebug`
+#   TARGET_PRODUCT        target device name, e.g., 'aosp_blueline'
+#   XREF_CORPUS           source code repository URI, e.g., 'android.googlesource.com/platform/superproject'
 
-# If OUT_DIR is not set, the build will use out/ as output directory, which is
-# a relative path. Make it absolute, otherwise the indexer will not know that it
-# contains only generated files.
-: ${OUT_DIR:=$PWD/out}
-[[ "$OUT_DIR" =~ ^/ ]] || { echo "$OUT_DIR is not an absolute path"; exit 1; }
 : ${BUILD_NUMBER:=$(uuidgen)}
 
-# The extraction might fail for some source files, so run with -k
+# The extraction might fail for some source files, so run with -k and then check that
-OUT_DIR=$OUT_DIR build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java
+# sufficiently many files were generated.
-
+build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java
-# We build with -k, so check that we have generated at least 100K files
+declare -r out="${OUT_DIR:-out}"
-# (the actual number is 180K+)
+declare -r kzip_count=$(find "$out" -name '*.kzip' | wc -l)
-declare -r kzip_count=$(find $OUT_DIR -name '*.kzip' | wc -l)
 (($kzip_count>100000)) || { printf "Too few kzip files were generated: %d\n" $kzip_count; exit 1; }
 
 # Pack
 # TODO(asmundak): this should be done by soong.
 declare -r allkzip="$BUILD_NUMBER.kzip"
-"$OUT_DIR/soong/host/linux-x86/bin/merge_zips" "$DIST_DIR/$allkzip" @<(find $OUT_DIR -name '*.kzip')
+"$out/soong/host/linux-x86/bin/merge_zips" "$DIST_DIR/$allkzip" @<(find "$out" -name '*.kzip')
 

cc/builder.go

@@ -224,12 +224,13 @@ var (
 
 	_ = pctx.SourcePathVariable("cxxExtractor",
 		"prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/cxx_extractor")
+	_ = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json")
 	_ = pctx.VariableFunc("kytheCorpus",
 		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
 	kytheExtract = pctx.StaticRule("kythe",
 		blueprint.RuleParams{
-			Command:     "rm -f $out && KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out $cxxExtractor $cFlags $in ",
+			Command:     "rm -f $out && KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out KYTHE_VNAMES=$kytheVnames $cxxExtractor $cFlags $in ",
-			CommandDeps: []string{"$cxxExtractor"},
+			CommandDeps: []string{"$cxxExtractor", "$kytheVnames"},
 		},
 		"cFlags")
 )

java/builder.go

@@ -64,6 +64,7 @@ var (
 
 	_ = pctx.VariableFunc("kytheCorpus",
 		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
+	_ = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json")
 	// Run it with -add-opens=java.base/java.nio=ALL-UNNAMED to avoid JDK9's warning about
 	// "Illegal reflective access by com.google.protobuf.Utf8$UnsafeProcessor ...
 	// to field java.nio.Buffer.address"
@@ -74,6 +75,7 @@ var (
 				`( [ ! -s $srcJarDir/list -a ! -s $out.rsp ] || ` +
 				`KYTHE_ROOT_DIRECTORY=. KYTHE_OUTPUT_FILE=$out ` +
 				`KYTHE_CORPUS=${kytheCorpus} ` +
+				`KYTHE_VNAMES=${kytheVnames} ` +
 				`${config.SoongJavacWrapper} ${config.JavaCmd} ` +
 				`--add-opens=java.base/java.nio=ALL-UNNAMED ` +
 				`-jar ${config.JavaKytheExtractorJar} ` +
@@ -84,6 +86,7 @@ var (
 			CommandDeps: []string{
 				"${config.JavaCmd}",
 				"${config.JavaKytheExtractorJar}",
+				"${kytheVnames}",
 				"${config.ZipSyncCmd}",
 			},
 			CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},

vnames.json

@@ -0,0 +1,18 @@
+[
+  {
+    "pattern": "out/(.*)",
+    "vname": {
+      "corpus": "CORPUS",
+      "root": "out",
+      "path": "@1@"
+    }
+  },
+  {
+    "pattern": "(.*)",
+    "vname": {
+      "corpus": "CORPUS",
+      "path": "@1@"
+    }
+  }
+]
+