Make bazel build //external/... work by only allowing "approved" existing BUILD files in the Bazel workspace.

Test: bazel build //bionic/... //external/... //frameworks/... //system/... Test: ./build/bazel/scripts/run_presubmits.sh Change-Id: I91865ca87c6535053e7a14d2526ff3ce0991bfea
2021-05-12 21:20:13 -04:00
parent 37ef223972
commit 0096079bbc
3 changed files with 120 additions and 44 deletions
--- a/android/bazel.go
+++ b/android/bazel.go
@@ -126,40 +126,21 @@ const (
 )

 var (
-	// Do not write BUILD files for these directories
-	// NOTE: this is not recursive
-	bp2buildDoNotWriteBuildFileList = []string{
-		// Don't generate these BUILD files - because external BUILD files already exist
-		"external/boringssl",
-		"external/brotli",
-		"external/dagger2",
-		"external/flatbuffers",
-		"external/gflags",
-		"external/google-fruit",
-		"external/grpc-grpc",
-		"external/grpc-grpc/test/core/util",
-		"external/grpc-grpc/test/cpp/common",
-		"external/grpc-grpc/third_party/address_sorting",
-		"external/nanopb-c",
-		"external/nos/host/generic",
-		"external/nos/host/generic/libnos",
-		"external/nos/host/generic/libnos/generator",
-		"external/nos/host/generic/libnos_datagram",
-		"external/nos/host/generic/libnos_transport",
-		"external/nos/host/generic/nugget/proto",
-		"external/perfetto",
-		"external/protobuf",
-		"external/rust/cxx",
-		"external/rust/cxx/demo",
-		"external/ruy",
-		"external/tensorflow",
-		"external/tensorflow/tensorflow/lite",
-		"external/tensorflow/tensorflow/lite/java",
-		"external/tensorflow/tensorflow/lite/kernels",
-		"external/tflite-support",
-		"external/tinyalsa_new",
-		"external/wycheproof",
-		"external/libyuv",
+	// Keep any existing BUILD files (and do not generate new BUILD files) for these directories
+	bp2buildKeepExistingBuildFile = map[string]bool{
+		// This is actually build/bazel/build.BAZEL symlinked to ./BUILD
+		".":/*recrusive = */ false,
+
+		"build/bazel":/* recursive = */ true,
+		"build/pesto":/* recursive = */ true,
+
+		// external/bazelbuild-rules_android/... is needed by mixed builds, otherwise mixed builds analysis fails
+		// e.g. ERROR: Analysis of target '@soong_injection//mixed_builds:buildroot' failed
+		"external/bazelbuild-rules_android":/* recursive = */ true,
+
+		"prebuilts/clang/host/linux-x86":/* recursive = */ false,
+		"prebuilts/sdk":/* recursive = */ false,
+		"prebuilts/sdk/tools":/* recursive = */ false,
 	}

 	// Configure modules in these directories to enable bp2build_available: true or false by default.
@@ -218,6 +199,7 @@ var (
 		// libcxx
 		"libBionicBenchmarksUtils", // cc_library_static, fatal error: 'map' file not found, from libcxx
 		"fmtlib",                   // cc_library_static, fatal error: 'cassert' file not found, from libcxx
+		"fmtlib_ndk",               // cc_library_static, fatal error: 'cassert' file not found
 		"libbase",                  // http://b/186826479, cc_library, fatal error: 'memory' file not found, from libcxx

 		// http://b/186024507: Includes errors because of the system_shared_libs default value.
@@ -229,6 +211,9 @@ var (
 		"note_memtag_heap_async", // http://b/185127353: cc_library_static, error: feature.h not found
 		"note_memtag_heap_sync",  // http://b/185127353: cc_library_static, error: feature.h not found

+		"libjemalloc5",           // cc_library, ld.lld: error: undefined symbol: memset
+		"gwp_asan_crash_handler", // cc_library, ld.lld: error: undefined symbol: memset
+
 		// Tests. Handle later.
 		"libbionic_tests_headers_posix", // http://b/186024507, cc_library_static, sched.h, time.h not found
 		"libjemalloc5_integrationtest",
@@ -254,17 +239,12 @@ var (
 	}

 	// Used for quicker lookups
-	bp2buildDoNotWriteBuildFile = map[string]bool{}
 	bp2buildModuleDoNotConvert  = map[string]bool{}
 	bp2buildCcLibraryStaticOnly = map[string]bool{}
 	mixedBuildsDisabled         = map[string]bool{}
 )

 func init() {
-	for _, moduleName := range bp2buildDoNotWriteBuildFileList {
-		bp2buildDoNotWriteBuildFile[moduleName] = true
-	}
-
 	for _, moduleName := range bp2buildModuleDoNotConvertList {
 		bp2buildModuleDoNotConvert[moduleName] = true
 	}
@@ -282,12 +262,21 @@ func GenerateCcLibraryStaticOnly(ctx BazelConversionPathContext) bool {
 	return bp2buildCcLibraryStaticOnly[ctx.Module().Name()]
 }

-func ShouldWriteBuildFileForDir(dir string) bool {
-	if _, ok := bp2buildDoNotWriteBuildFile[dir]; ok {
-		return false
-	} else {
+func ShouldKeepExistingBuildFileForDir(dir string) bool {
+	if _, ok := bp2buildKeepExistingBuildFile[dir]; ok {
+		// Exact dir match
 		return true
 	}
+	// Check if subtree match
+	for prefix, recursive := range bp2buildKeepExistingBuildFile {
+		if recursive {
+			if strings.HasPrefix(dir, prefix+"/") {
+				return true
+			}
+		}
+	}
+	// Default
+	return false
 }

 // MixedBuildsEnabled checks that a module is ready to be replaced by a
--- a/bp2build/conversion.go
+++ b/bp2build/conversion.go
@@ -59,7 +59,7 @@ func CreateBazelFiles(
 func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile {
 	files := make([]BazelFile, 0, len(buildToTargets))
 	for _, dir := range android.SortedStringKeys(buildToTargets) {
-		if mode == Bp2Build && !android.ShouldWriteBuildFileForDir(dir) {
+		if mode == Bp2Build && android.ShouldKeepExistingBuildFileForDir(dir) {
 			fmt.Printf("[bp2build] Not writing generated BUILD file for dir: '%s'\n", dir)
 			continue
 		}
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -17,6 +17,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -357,6 +358,80 @@ func touch(path string) {
 	}
 }

+// Find BUILD files in the srcDir which...
+//
+// - are not on the allow list (android/bazel.go#ShouldKeepExistingBuildFileForDir())
+//
+// - won't be overwritten by corresponding bp2build generated files
+//
+// And return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
+func getPathsToIgnoredBuildFiles(topDir string, outDir string, generatedRoot string) ([]string, error) {
+	paths := make([]string, 0)
+
+	err := filepath.WalkDir(topDir, func(fFullPath string, fDirEntry fs.DirEntry, err error) error {
+		if err != nil {
+			// Warn about error, but continue trying to walk the directory tree
+			fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fFullPath, err)
+			return nil
+		}
+		if fDirEntry.IsDir() {
+			// Don't ignore entire directories
+			return nil
+		}
+		if !(fDirEntry.Name() == "BUILD" || fDirEntry.Name() == "BUILD.bazel") {
+			// Don't ignore this file - it is not a build file
+			return nil
+		}
+		f := strings.TrimPrefix(fFullPath, topDir+"/")
+		if strings.HasPrefix(f, ".repo/") {
+			// Don't check for files to ignore in the .repo dir (recursively)
+			return fs.SkipDir
+		}
+		if strings.HasPrefix(f, outDir+"/") {
+			// Don't check for files to ignore in the out dir (recursively)
+			return fs.SkipDir
+		}
+		if strings.HasPrefix(f, generatedRoot) {
+			// Don't check for files to ignore in the bp2build dir (recursively)
+			// NOTE: This is usually under outDir
+			return fs.SkipDir
+		}
+		fDir := filepath.Dir(f)
+		if android.ShouldKeepExistingBuildFileForDir(fDir) {
+			// Don't ignore this existing build file
+			return nil
+		}
+		f_bp2build := shared.JoinPath(topDir, generatedRoot, f)
+		if _, err := os.Stat(f_bp2build); err == nil {
+			// If bp2build generated an alternate BUILD file, don't exclude this workspace path
+			// BUILD file clash resolution happens later in the symlink forest creation
+			return nil
+		}
+		fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", f)
+		paths = append(paths, f)
+		return nil
+	})
+
+	return paths, err
+}
+
+// Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work
+func getTemporaryExcludes() []string {
+	excludes := make([]string, 0)
+
+	// FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite symlink expansion error for Bazel
+	excludes = append(excludes, "external/autotest/venv/autotest_lib")
+
+	// FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
+	// It contains several symlinks back to real source dirs, and those source dirs contain BUILD files we want to ignore
+	excludes = append(excludes, "external/google-fruit/extras/bazel_root/third_party/fruit")
+
+	// FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
+	excludes = append(excludes, "frameworks/compile/slang")
+
+	return excludes
+}
+
 // Run Soong in the bp2build mode. This creates a standalone context that registers
 // an alternate pipeline of mutators and singletons specifically for generating
 // Bazel BUILD files instead of Ninja files.
@@ -415,6 +490,18 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 		excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)
 	}

+	// FIXME: Don't hardcode this here
+	topLevelOutDir := "out"
+
+	pathsToIgnoredBuildFiles, err := getPathsToIgnoredBuildFiles(topDir, topLevelOutDir, generatedRoot)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error walking SrcDir: '%s': %s\n", configuration.SrcDir(), err)
+		os.Exit(1)
+	}
+	excludes = append(excludes, pathsToIgnoredBuildFiles...)
+
+	excludes = append(excludes, getTemporaryExcludes()...)
+
 	symlinkForestDeps := bp2build.PlantSymlinkForest(
 		topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes)