If `SOONG_USE_N2=true` is set in the environment, then n2 is used in place of ninja. Some ninja features are not available in n2 at this time, but this enables bringup efforts to happen in parallel. Bug: 352368206 Test: manual Change-Id: I8455cb24eb640a4651782ee76e48a7d3a9932b93
1706 lines
48 KiB
Go
1706 lines
48 KiB
Go
// Copyright 2017 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package build
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"os"
|
|
"os/exec"
|
|
"os/user"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"android/soong/shared"
|
|
"android/soong/ui/metrics"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
smpb "android/soong/ui/metrics/metrics_proto"
|
|
)
|
|
|
|
const (
|
|
envConfigDir = "vendor/google/tools/soong_config"
|
|
jsonSuffix = "json"
|
|
)
|
|
|
|
var (
|
|
rbeRandPrefix int
|
|
googleProdCredsExistCache bool
|
|
)
|
|
|
|
func init() {
|
|
rand.Seed(time.Now().UnixNano())
|
|
rbeRandPrefix = rand.Intn(1000)
|
|
}
|
|
|
|
type Config struct{ *configImpl }
|
|
|
|
type configImpl struct {
|
|
// Some targets that are implemented in soong_build
|
|
// (bp2build, json-module-graph) are not here and have their own bits below.
|
|
arguments []string
|
|
goma bool
|
|
environ *Environment
|
|
distDir string
|
|
buildDateTime string
|
|
logsPrefix string
|
|
|
|
// From the arguments
|
|
parallel int
|
|
keepGoing int
|
|
verbose bool
|
|
checkbuild bool
|
|
dist bool
|
|
jsonModuleGraph bool
|
|
queryview bool
|
|
reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
|
|
soongDocs bool
|
|
skipConfig bool
|
|
skipKati bool
|
|
skipKatiNinja bool
|
|
skipSoong bool
|
|
skipNinja bool
|
|
skipSoongTests bool
|
|
searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
|
|
skipMetricsUpload bool
|
|
buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
|
|
buildFromSourceStub bool
|
|
incrementalBuildActions bool
|
|
ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
|
|
|
|
// From the product config
|
|
katiArgs []string
|
|
ninjaArgs []string
|
|
katiSuffix string
|
|
targetDevice string
|
|
targetDeviceDir string
|
|
sandboxConfig *SandboxConfig
|
|
|
|
// Autodetected
|
|
totalRAM uint64
|
|
|
|
brokenDupRules bool
|
|
brokenUsesNetwork bool
|
|
brokenNinjaEnvVars []string
|
|
brokenMissingOutputs bool
|
|
|
|
pathReplaced bool
|
|
|
|
// Set by multiproduct_kati
|
|
emptyNinjaFile bool
|
|
|
|
metricsUploader string
|
|
|
|
includeTags []string
|
|
sourceRootDirs []string
|
|
|
|
// Data source to write ninja weight list
|
|
ninjaWeightListSource NinjaWeightListSource
|
|
|
|
// This file is a detailed dump of all soong-defined modules for debugging purposes.
|
|
// There's quite a bit of overlap with module-info.json and soong module graph. We
|
|
// could consider merging them.
|
|
moduleDebugFile string
|
|
|
|
// Whether to use n2 instead of ninja. This is controlled with the
|
|
// environment variable SOONG_USE_N2
|
|
useN2 bool
|
|
}
|
|
|
|
type NinjaWeightListSource uint
|
|
|
|
const (
|
|
// ninja doesn't use weight list.
|
|
NOT_USED NinjaWeightListSource = iota
|
|
// ninja uses weight list based on previous builds by ninja log
|
|
NINJA_LOG
|
|
// ninja thinks every task has the same weight.
|
|
EVENLY_DISTRIBUTED
|
|
// ninja uses an external custom weight list
|
|
EXTERNAL_FILE
|
|
// ninja uses a prioritized module list from Soong
|
|
HINT_FROM_SOONG
|
|
// If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead.
|
|
// We can assume it is an incremental build if ninja log exists.
|
|
DEFAULT
|
|
)
|
|
const srcDirFileCheck = "build/soong/root.bp"
|
|
|
|
var buildFiles = []string{"Android.mk", "Android.bp"}
|
|
|
|
type BuildAction uint
|
|
|
|
const (
|
|
// Builds all of the modules and their dependencies of a specified directory, relative to the root
|
|
// directory of the source tree.
|
|
BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
|
|
|
|
// Builds all of the modules and their dependencies of a list of specified directories. All specified
|
|
// directories are relative to the root directory of the source tree.
|
|
BUILD_MODULES_IN_DIRECTORIES
|
|
|
|
// Build a list of specified modules. If none was specified, simply build the whole source tree.
|
|
BUILD_MODULES
|
|
)
|
|
|
|
// checkTopDir validates that the current directory is at the root directory of the source tree.
|
|
func checkTopDir(ctx Context) {
|
|
if _, err := os.Stat(srcDirFileCheck); err != nil {
|
|
if os.IsNotExist(err) {
|
|
ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
|
|
}
|
|
ctx.Fatalln("Error verifying tree state:", err)
|
|
}
|
|
}
|
|
|
|
func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
|
|
if bc == "" {
|
|
return nil
|
|
}
|
|
|
|
configDirs := []string{
|
|
config.OutDir(),
|
|
os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
|
|
envConfigDir,
|
|
}
|
|
for _, dir := range configDirs {
|
|
cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
|
|
envVarsJSON, err := ioutil.ReadFile(cfgFile)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
ctx.Verbosef("Loading config file %v\n", cfgFile)
|
|
var envVars map[string]map[string]string
|
|
if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
|
|
continue
|
|
}
|
|
for k, v := range envVars["env"] {
|
|
if os.Getenv(k) != "" {
|
|
continue
|
|
}
|
|
config.environ.Set(k, v)
|
|
}
|
|
ctx.Verbosef("Finished loading config file %v\n", cfgFile)
|
|
break
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func NewConfig(ctx Context, args ...string) Config {
|
|
ret := &configImpl{
|
|
environ: OsEnvironment(),
|
|
sandboxConfig: &SandboxConfig{},
|
|
ninjaWeightListSource: DEFAULT,
|
|
}
|
|
|
|
// Skip soong tests by default on Linux
|
|
if runtime.GOOS == "linux" {
|
|
ret.skipSoongTests = true
|
|
}
|
|
|
|
// Default matching ninja
|
|
ret.parallel = runtime.NumCPU() + 2
|
|
ret.keepGoing = 1
|
|
|
|
ret.totalRAM = detectTotalRAM(ctx)
|
|
ret.parseArgs(ctx, args)
|
|
|
|
if ret.ninjaWeightListSource == HINT_FROM_SOONG {
|
|
ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always")
|
|
} else if ret.ninjaWeightListSource == DEFAULT {
|
|
defaultNinjaWeightListSource := NINJA_LOG
|
|
if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) {
|
|
ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead")
|
|
defaultNinjaWeightListSource = HINT_FROM_SOONG
|
|
} else {
|
|
ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG")
|
|
}
|
|
ret.ninjaWeightListSource = defaultNinjaWeightListSource
|
|
// soong_build generates ninja hint depending on ninja log existence.
|
|
// Set it "depend" to avoid soong re-run due to env variable change.
|
|
ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend")
|
|
}
|
|
|
|
// Make sure OUT_DIR is set appropriately
|
|
if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
|
|
ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
|
|
} else {
|
|
outDir := "out"
|
|
if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
|
|
if wd, err := os.Getwd(); err != nil {
|
|
ctx.Fatalln("Failed to get working directory:", err)
|
|
} else {
|
|
outDir = filepath.Join(baseDir, filepath.Base(wd))
|
|
}
|
|
}
|
|
ret.environ.Set("OUT_DIR", outDir)
|
|
}
|
|
|
|
// loadEnvConfig needs to know what the OUT_DIR is, so it should
|
|
// be called after we determine the appropriate out directory.
|
|
bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
|
|
|
|
if bc != "" {
|
|
if err := loadEnvConfig(ctx, ret, bc); err != nil {
|
|
ctx.Fatalln("Failed to parse env config files: %v", err)
|
|
}
|
|
if !ret.canSupportRBE() {
|
|
// Explicitly set USE_RBE env variable to false when we cannot run
|
|
// an RBE build to avoid ninja local execution pool issues.
|
|
ret.environ.Set("USE_RBE", "false")
|
|
}
|
|
}
|
|
|
|
if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
|
|
ret.distDir = filepath.Clean(distDir)
|
|
} else {
|
|
ret.distDir = filepath.Join(ret.OutDir(), "dist")
|
|
}
|
|
|
|
if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
|
|
ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
|
|
}
|
|
|
|
if os.Getenv("GENERATE_SOONG_DEBUG") == "true" {
|
|
ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json"))
|
|
}
|
|
|
|
if os.Getenv("SOONG_USE_N2") == "true" {
|
|
ret.useN2 = true
|
|
}
|
|
|
|
ret.environ.Unset(
|
|
// We're already using it
|
|
"USE_SOONG_UI",
|
|
|
|
// We should never use GOROOT/GOPATH from the shell environment
|
|
"GOROOT",
|
|
"GOPATH",
|
|
|
|
// These should only come from Soong, not the environment.
|
|
"CLANG",
|
|
"CLANG_CXX",
|
|
"CCC_CC",
|
|
"CCC_CXX",
|
|
|
|
// Used by the goma compiler wrapper, but should only be set by
|
|
// gomacc
|
|
"GOMACC_PATH",
|
|
|
|
// We handle this above
|
|
"OUT_DIR_COMMON_BASE",
|
|
|
|
// This is handled above too, and set for individual commands later
|
|
"DIST_DIR",
|
|
|
|
// Variables that have caused problems in the past
|
|
"BASH_ENV",
|
|
"CDPATH",
|
|
"DISPLAY",
|
|
"GREP_OPTIONS",
|
|
"JAVAC",
|
|
"LEX",
|
|
"NDK_ROOT",
|
|
"POSIXLY_CORRECT",
|
|
|
|
// Drop make flags
|
|
"MAKEFLAGS",
|
|
"MAKELEVEL",
|
|
"MFLAGS",
|
|
|
|
// Set in envsetup.sh, reset in makefiles
|
|
"ANDROID_JAVA_TOOLCHAIN",
|
|
|
|
// Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
|
|
"ANDROID_BUILD_TOP",
|
|
"ANDROID_HOST_OUT",
|
|
"ANDROID_PRODUCT_OUT",
|
|
"ANDROID_HOST_OUT_TESTCASES",
|
|
"ANDROID_TARGET_OUT_TESTCASES",
|
|
"ANDROID_TOOLCHAIN",
|
|
"ANDROID_TOOLCHAIN_2ND_ARCH",
|
|
"ANDROID_DEV_SCRIPTS",
|
|
"ANDROID_EMULATOR_PREBUILTS",
|
|
"ANDROID_PRE_BUILD_PATHS",
|
|
|
|
// We read it here already, don't let others share in the fun
|
|
"GENERATE_SOONG_DEBUG",
|
|
|
|
// Use config.useN2 instead.
|
|
"SOONG_USE_N2",
|
|
)
|
|
|
|
if ret.UseGoma() || ret.ForceUseGoma() {
|
|
ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
|
|
ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
|
|
}
|
|
|
|
// Tell python not to spam the source tree with .pyc files.
|
|
ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
|
|
|
|
tmpDir := absPath(ctx, ret.TempDir())
|
|
ret.environ.Set("TMPDIR", tmpDir)
|
|
|
|
// Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
|
|
symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
|
|
"llvm-binutils-stable/llvm-symbolizer")
|
|
ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
|
|
|
|
// Precondition: the current directory is the top of the source tree
|
|
checkTopDir(ctx)
|
|
|
|
srcDir := absPath(ctx, ".")
|
|
if strings.ContainsRune(srcDir, ' ') {
|
|
ctx.Println("You are building in a directory whose absolute path contains a space character:")
|
|
ctx.Println()
|
|
ctx.Printf("%q\n", srcDir)
|
|
ctx.Println()
|
|
ctx.Fatalln("Directory names containing spaces are not supported")
|
|
}
|
|
|
|
ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
|
|
|
|
if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
|
|
ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
|
|
ctx.Println()
|
|
ctx.Printf("%q\n", outDir)
|
|
ctx.Println()
|
|
ctx.Fatalln("Directory names containing spaces are not supported")
|
|
}
|
|
|
|
if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
|
|
ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
|
|
ctx.Println()
|
|
ctx.Printf("%q\n", distDir)
|
|
ctx.Println()
|
|
ctx.Fatalln("Directory names containing spaces are not supported")
|
|
}
|
|
|
|
// Configure Java-related variables, including adding it to $PATH
|
|
java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
|
|
java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag())
|
|
javaHome := func() string {
|
|
if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
|
|
return override
|
|
}
|
|
if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
|
|
ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.")
|
|
}
|
|
if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
|
|
ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.")
|
|
}
|
|
if toolchain21, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN"); ok && toolchain21 != "true" {
|
|
ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.")
|
|
}
|
|
return java21Home
|
|
}()
|
|
absJavaHome := absPath(ctx, javaHome)
|
|
|
|
ret.configureLocale(ctx)
|
|
|
|
newPath := []string{filepath.Join(absJavaHome, "bin")}
|
|
if path, ok := ret.environ.Get("PATH"); ok && path != "" {
|
|
newPath = append(newPath, path)
|
|
}
|
|
|
|
ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
|
|
ret.environ.Set("JAVA_HOME", absJavaHome)
|
|
ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
|
|
ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
|
|
ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
|
|
|
|
// b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches
|
|
// to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large
|
|
// zip files produced by soong_zip. Disable zipbomb detection.
|
|
ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE")
|
|
|
|
outDir := ret.OutDir()
|
|
buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
|
|
if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
|
|
ret.buildDateTime = buildDateTime
|
|
} else {
|
|
ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
|
|
}
|
|
|
|
ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
|
|
|
|
if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok {
|
|
username := "unknown"
|
|
if u, err := user.Current(); err == nil {
|
|
username = u.Username
|
|
} else {
|
|
ctx.Println("Failed to get current user:", err)
|
|
}
|
|
ret.environ.Set("BUILD_USERNAME", username)
|
|
}
|
|
|
|
if ret.UseRBE() {
|
|
for k, v := range getRBEVars(ctx, Config{ret}) {
|
|
ret.environ.Set(k, v)
|
|
}
|
|
}
|
|
|
|
c := Config{ret}
|
|
storeConfigMetrics(ctx, c)
|
|
return c
|
|
}
|
|
|
|
// NewBuildActionConfig returns a build configuration based on the build action. The arguments are
|
|
// processed based on the build action and extracts any arguments that belongs to the build action.
|
|
func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
|
|
return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
|
|
}
|
|
|
|
// Prepare for getting make variables. For them to be accurate, we need to have
|
|
// obtained PRODUCT_RELEASE_CONFIG_MAPS.
|
|
//
|
|
// Returns:
|
|
//
|
|
// Whether config should be called again.
|
|
//
|
|
// TODO: when converting product config to a declarative language, make sure
|
|
// that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in
|
|
// that process.
|
|
func SetProductReleaseConfigMaps(ctx Context, config Config) bool {
|
|
ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps")
|
|
defer ctx.EndTrace()
|
|
|
|
if config.SkipConfig() {
|
|
// This duplicates the logic from Build to skip product config
|
|
// if the user has explicitly said to.
|
|
return false
|
|
}
|
|
|
|
releaseConfigVars := []string{
|
|
"PRODUCT_RELEASE_CONFIG_MAPS",
|
|
}
|
|
|
|
origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS")
|
|
// Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment
|
|
// when we run product config to get the rest of the make vars.
|
|
releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "")
|
|
if err != nil {
|
|
ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err)
|
|
}
|
|
productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"]
|
|
os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps)
|
|
return origValue != productReleaseConfigMaps
|
|
}
|
|
|
|
// storeConfigMetrics selects a set of configuration information and store in
|
|
// the metrics system for further analysis.
|
|
func storeConfigMetrics(ctx Context, config Config) {
|
|
if ctx.Metrics == nil {
|
|
return
|
|
}
|
|
|
|
ctx.Metrics.BuildConfig(buildConfig(config))
|
|
|
|
s := &smpb.SystemResourceInfo{
|
|
TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
|
|
AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
|
|
}
|
|
ctx.Metrics.SystemResourceInfo(s)
|
|
}
|
|
|
|
func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
|
|
switch s {
|
|
case NINJA_LOG:
|
|
return smpb.BuildConfig_NINJA_LOG.Enum()
|
|
case EVENLY_DISTRIBUTED:
|
|
return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
|
|
case EXTERNAL_FILE:
|
|
return smpb.BuildConfig_EXTERNAL_FILE.Enum()
|
|
case HINT_FROM_SOONG:
|
|
return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
|
|
default:
|
|
return smpb.BuildConfig_NOT_USED.Enum()
|
|
}
|
|
}
|
|
|
|
func buildConfig(config Config) *smpb.BuildConfig {
|
|
c := &smpb.BuildConfig{
|
|
ForceUseGoma: proto.Bool(config.ForceUseGoma()),
|
|
UseGoma: proto.Bool(config.UseGoma()),
|
|
UseRbe: proto.Bool(config.UseRBE()),
|
|
NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
|
|
}
|
|
c.Targets = append(c.Targets, config.arguments...)
|
|
|
|
return c
|
|
}
|
|
|
|
// getConfigArgs processes the command arguments based on the build action and creates a set of new
|
|
// arguments to be accepted by Config.
|
|
func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
|
|
// The next block of code verifies that the current directory is the root directory of the source
|
|
// tree. It then finds the relative path of dir based on the root directory of the source tree
|
|
// and verify that dir is inside of the source tree.
|
|
checkTopDir(ctx)
|
|
topDir, err := os.Getwd()
|
|
if err != nil {
|
|
ctx.Fatalf("Error retrieving top directory: %v", err)
|
|
}
|
|
dir, err = filepath.EvalSymlinks(dir)
|
|
if err != nil {
|
|
ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
|
|
}
|
|
dir, err = filepath.Abs(dir)
|
|
if err != nil {
|
|
ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
|
|
}
|
|
relDir, err := filepath.Rel(topDir, dir)
|
|
if err != nil {
|
|
ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
|
|
}
|
|
// If there are ".." in the path, it's not in the source tree.
|
|
if strings.Contains(relDir, "..") {
|
|
ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
|
|
}
|
|
|
|
configArgs := args[:]
|
|
|
|
// If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
|
|
// GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
|
|
targetNamePrefix := "MODULES-IN-"
|
|
if inList("GET-INSTALL-PATH", configArgs) {
|
|
targetNamePrefix = "GET-INSTALL-PATH-IN-"
|
|
configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
|
|
}
|
|
|
|
var targets []string
|
|
|
|
switch action {
|
|
case BUILD_MODULES:
|
|
// No additional processing is required when building a list of specific modules or all modules.
|
|
case BUILD_MODULES_IN_A_DIRECTORY:
|
|
// If dir is the root source tree, all the modules are built of the source tree are built so
|
|
// no need to find the build file.
|
|
if topDir == dir {
|
|
break
|
|
}
|
|
|
|
buildFile := findBuildFile(ctx, relDir)
|
|
if buildFile == "" {
|
|
ctx.Fatalf("Build file not found for %s directory", relDir)
|
|
}
|
|
targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
|
|
case BUILD_MODULES_IN_DIRECTORIES:
|
|
newConfigArgs, dirs := splitArgs(configArgs)
|
|
configArgs = newConfigArgs
|
|
targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
|
|
}
|
|
|
|
// Tidy only override all other specified targets.
|
|
tidyOnly := os.Getenv("WITH_TIDY_ONLY")
|
|
if tidyOnly == "true" || tidyOnly == "1" {
|
|
configArgs = append(configArgs, "tidy_only")
|
|
} else {
|
|
configArgs = append(configArgs, targets...)
|
|
}
|
|
|
|
return configArgs
|
|
}
|
|
|
|
// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
|
|
func convertToTarget(dir string, targetNamePrefix string) string {
|
|
return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
|
|
}
|
|
|
|
// hasBuildFile returns true if dir contains an Android build file.
|
|
func hasBuildFile(ctx Context, dir string) bool {
|
|
for _, buildFile := range buildFiles {
|
|
_, err := os.Stat(filepath.Join(dir, buildFile))
|
|
if err == nil {
|
|
return true
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
ctx.Fatalf("Error retrieving the build file stats: %v", err)
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
|
|
// in the current and any sub directory of dir. If a build file is not found, traverse the path
|
|
// up by one directory and repeat again until either a build file is found or reached to the root
|
|
// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
|
|
// string is returned.
|
|
func findBuildFile(ctx Context, dir string) string {
|
|
// If the string is empty or ".", assume it is top directory of the source tree.
|
|
if dir == "" || dir == "." {
|
|
return ""
|
|
}
|
|
|
|
found := false
|
|
for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
|
|
err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found {
|
|
return filepath.SkipDir
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
for _, buildFile := range buildFiles {
|
|
if info.Name() == buildFile {
|
|
found = true
|
|
return filepath.SkipDir
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
ctx.Fatalf("Error finding Android build file: %v", err)
|
|
}
|
|
|
|
if found {
|
|
return filepath.Join(buildDir, "Android.mk")
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
|
|
func splitArgs(args []string) (newArgs []string, dirs []string) {
|
|
specialArgs := map[string]bool{
|
|
"showcommands": true,
|
|
"snod": true,
|
|
"dist": true,
|
|
"checkbuild": true,
|
|
}
|
|
|
|
newArgs = []string{}
|
|
dirs = []string{}
|
|
|
|
for _, arg := range args {
|
|
// It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
|
|
if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
|
|
newArgs = append(newArgs, arg)
|
|
continue
|
|
}
|
|
|
|
if _, ok := specialArgs[arg]; ok {
|
|
newArgs = append(newArgs, arg)
|
|
continue
|
|
}
|
|
|
|
dirs = append(dirs, arg)
|
|
}
|
|
|
|
return newArgs, dirs
|
|
}
|
|
|
|
// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
|
|
// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
|
|
// source root tree where the build action command was invoked. Each directory is validated if the
|
|
// build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
|
|
func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
|
|
for _, dir := range dirs {
|
|
// The directory may have specified specific modules to build. ":" is the separator to separate
|
|
// the directory and the list of modules.
|
|
s := strings.Split(dir, ":")
|
|
l := len(s)
|
|
if l > 2 { // more than one ":" was specified.
|
|
ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
|
|
}
|
|
|
|
dir = filepath.Join(relDir, s[0])
|
|
if _, err := os.Stat(dir); err != nil {
|
|
ctx.Fatalf("couldn't find directory %s", dir)
|
|
}
|
|
|
|
// Verify that if there are any targets specified after ":". Each target is separated by ",".
|
|
var newTargets []string
|
|
if l == 2 && s[1] != "" {
|
|
newTargets = strings.Split(s[1], ",")
|
|
if inList("", newTargets) {
|
|
ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
|
|
}
|
|
}
|
|
|
|
// If there are specified targets to build in dir, an android build file must exist for the one
|
|
// shot build. For the non-targets case, find the appropriate build file and build all the
|
|
// modules in dir (or the closest one in the dir path).
|
|
if len(newTargets) > 0 {
|
|
if !hasBuildFile(ctx, dir) {
|
|
ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
|
|
}
|
|
} else {
|
|
buildFile := findBuildFile(ctx, dir)
|
|
if buildFile == "" {
|
|
ctx.Fatalf("Build file not found for %s directory", dir)
|
|
}
|
|
newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
|
|
}
|
|
|
|
targets = append(targets, newTargets...)
|
|
}
|
|
|
|
return targets
|
|
}
|
|
|
|
func (c *configImpl) parseArgs(ctx Context, args []string) {
|
|
for i := 0; i < len(args); i++ {
|
|
arg := strings.TrimSpace(args[i])
|
|
if arg == "showcommands" {
|
|
c.verbose = true
|
|
} else if arg == "--empty-ninja-file" {
|
|
c.emptyNinjaFile = true
|
|
} else if arg == "--skip-ninja" {
|
|
c.skipNinja = true
|
|
} else if arg == "--skip-make" {
|
|
// TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
|
|
// but it does run a Kati ninja file if the .kati_enabled marker file was created
|
|
// by a previous build.
|
|
c.skipConfig = true
|
|
c.skipKati = true
|
|
} else if arg == "--soong-only" {
|
|
c.skipKati = true
|
|
c.skipKatiNinja = true
|
|
} else if arg == "--config-only" {
|
|
c.skipKati = true
|
|
c.skipKatiNinja = true
|
|
c.skipSoong = true
|
|
} else if arg == "--skip-config" {
|
|
c.skipConfig = true
|
|
} else if arg == "--skip-soong-tests" {
|
|
c.skipSoongTests = true
|
|
} else if arg == "--no-skip-soong-tests" {
|
|
c.skipSoongTests = false
|
|
} else if arg == "--skip-metrics-upload" {
|
|
c.skipMetricsUpload = true
|
|
} else if arg == "--mk-metrics" {
|
|
c.reportMkMetrics = true
|
|
} else if arg == "--search-api-dir" {
|
|
c.searchApiDir = true
|
|
} else if strings.HasPrefix(arg, "--ninja_weight_source=") {
|
|
source := strings.TrimPrefix(arg, "--ninja_weight_source=")
|
|
if source == "ninja_log" {
|
|
c.ninjaWeightListSource = NINJA_LOG
|
|
} else if source == "evenly_distributed" {
|
|
c.ninjaWeightListSource = EVENLY_DISTRIBUTED
|
|
} else if source == "not_used" {
|
|
c.ninjaWeightListSource = NOT_USED
|
|
} else if source == "soong" {
|
|
c.ninjaWeightListSource = HINT_FROM_SOONG
|
|
} else if strings.HasPrefix(source, "file,") {
|
|
c.ninjaWeightListSource = EXTERNAL_FILE
|
|
filePath := strings.TrimPrefix(source, "file,")
|
|
err := validateNinjaWeightList(filePath)
|
|
if err != nil {
|
|
ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
|
|
}
|
|
_, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
|
|
if err != nil {
|
|
ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
|
|
}
|
|
} else {
|
|
ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
|
|
}
|
|
} else if arg == "--build-from-source-stub" {
|
|
c.buildFromSourceStub = true
|
|
} else if arg == "--incremental-build-actions" {
|
|
c.incrementalBuildActions = true
|
|
} else if strings.HasPrefix(arg, "--build-command=") {
|
|
buildCmd := strings.TrimPrefix(arg, "--build-command=")
|
|
// remove quotations
|
|
buildCmd = strings.TrimPrefix(buildCmd, "\"")
|
|
buildCmd = strings.TrimSuffix(buildCmd, "\"")
|
|
ctx.Metrics.SetBuildCommand([]string{buildCmd})
|
|
} else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
|
|
buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
|
|
val, err := strconv.ParseInt(buildTimeStr, 10, 64)
|
|
if err == nil {
|
|
c.buildStartedTime = val
|
|
} else {
|
|
ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
|
|
}
|
|
} else if arg == "--ensure-allowlist-integrity" {
|
|
c.ensureAllowlistIntegrity = true
|
|
} else if len(arg) > 0 && arg[0] == '-' {
|
|
parseArgNum := func(def int) int {
|
|
if len(arg) > 2 {
|
|
p, err := strconv.ParseUint(arg[2:], 10, 31)
|
|
if err != nil {
|
|
ctx.Fatalf("Failed to parse %q: %v", arg, err)
|
|
}
|
|
return int(p)
|
|
} else if i+1 < len(args) {
|
|
p, err := strconv.ParseUint(args[i+1], 10, 31)
|
|
if err == nil {
|
|
i++
|
|
return int(p)
|
|
}
|
|
}
|
|
return def
|
|
}
|
|
|
|
if len(arg) > 1 && arg[1] == 'j' {
|
|
c.parallel = parseArgNum(c.parallel)
|
|
} else if len(arg) > 1 && arg[1] == 'k' {
|
|
c.keepGoing = parseArgNum(0)
|
|
} else {
|
|
ctx.Fatalln("Unknown option:", arg)
|
|
}
|
|
} else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
|
|
if k == "OUT_DIR" {
|
|
ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
|
|
}
|
|
c.environ.Set(k, v)
|
|
} else if arg == "dist" {
|
|
c.dist = true
|
|
} else if arg == "json-module-graph" {
|
|
c.jsonModuleGraph = true
|
|
} else if arg == "queryview" {
|
|
c.queryview = true
|
|
} else if arg == "soong_docs" {
|
|
c.soongDocs = true
|
|
} else {
|
|
if arg == "checkbuild" {
|
|
c.checkbuild = true
|
|
}
|
|
c.arguments = append(c.arguments, arg)
|
|
}
|
|
}
|
|
}
|
|
|
|
func validateNinjaWeightList(weightListFilePath string) (err error) {
|
|
data, err := os.ReadFile(weightListFilePath)
|
|
if err != nil {
|
|
return
|
|
}
|
|
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
|
|
for _, line := range lines {
|
|
fields := strings.Split(line, ",")
|
|
if len(fields) != 2 {
|
|
return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
|
|
}
|
|
_, err = strconv.Atoi(fields[1])
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (c *configImpl) configureLocale(ctx Context) {
|
|
cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
|
|
output, err := cmd.Output()
|
|
|
|
var locales []string
|
|
if err == nil {
|
|
locales = strings.Split(string(output), "\n")
|
|
} else {
|
|
// If we're unable to list the locales, let's assume en_US.UTF-8
|
|
locales = []string{"en_US.UTF-8"}
|
|
ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
|
|
}
|
|
|
|
// gettext uses LANGUAGE, which is passed directly through
|
|
|
|
// For LANG and LC_*, only preserve the evaluated version of
|
|
// LC_MESSAGES
|
|
userLang := ""
|
|
if lc_all, ok := c.environ.Get("LC_ALL"); ok {
|
|
userLang = lc_all
|
|
} else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
|
|
userLang = lc_messages
|
|
} else if lang, ok := c.environ.Get("LANG"); ok {
|
|
userLang = lang
|
|
}
|
|
|
|
c.environ.UnsetWithPrefix("LC_")
|
|
|
|
if userLang != "" {
|
|
c.environ.Set("LC_MESSAGES", userLang)
|
|
}
|
|
|
|
// The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
|
|
// for others)
|
|
if inList("C.UTF-8", locales) {
|
|
c.environ.Set("LANG", "C.UTF-8")
|
|
} else if inList("C.utf8", locales) {
|
|
// These normalize to the same thing
|
|
c.environ.Set("LANG", "C.UTF-8")
|
|
} else if inList("en_US.UTF-8", locales) {
|
|
c.environ.Set("LANG", "en_US.UTF-8")
|
|
} else if inList("en_US.utf8", locales) {
|
|
// These normalize to the same thing
|
|
c.environ.Set("LANG", "en_US.UTF-8")
|
|
} else {
|
|
ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) Environment() *Environment {
|
|
return c.environ
|
|
}
|
|
|
|
func (c *configImpl) Arguments() []string {
|
|
return c.arguments
|
|
}
|
|
|
|
func (c *configImpl) SoongBuildInvocationNeeded() bool {
|
|
if len(c.Arguments()) > 0 {
|
|
// Explicit targets requested that are not special targets like b2pbuild
|
|
// or the JSON module graph
|
|
return true
|
|
}
|
|
|
|
if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() {
|
|
// Command line was empty, the default Ninja target is built
|
|
return true
|
|
}
|
|
|
|
if c.Dist() {
|
|
return true
|
|
}
|
|
|
|
// build.ninja doesn't need to be generated
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) OutDir() string {
|
|
if outDir, ok := c.environ.Get("OUT_DIR"); ok {
|
|
return outDir
|
|
}
|
|
return "out"
|
|
}
|
|
|
|
func (c *configImpl) DistDir() string {
|
|
return c.distDir
|
|
}
|
|
|
|
func (c *configImpl) RealDistDir() string {
|
|
return c.distDir
|
|
}
|
|
|
|
func (c *configImpl) NinjaArgs() []string {
|
|
if c.skipKati {
|
|
return c.arguments
|
|
}
|
|
return c.ninjaArgs
|
|
}
|
|
|
|
func (c *configImpl) SoongOutDir() string {
|
|
return filepath.Join(c.OutDir(), "soong")
|
|
}
|
|
|
|
func (c *configImpl) ApiSurfacesOutDir() string {
|
|
return filepath.Join(c.OutDir(), "api_surfaces")
|
|
}
|
|
|
|
func (c *configImpl) PrebuiltOS() string {
|
|
switch runtime.GOOS {
|
|
case "linux":
|
|
return "linux-x86"
|
|
case "darwin":
|
|
return "darwin-x86"
|
|
default:
|
|
panic("Unknown GOOS")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) HostToolDir() string {
|
|
if c.SkipKatiNinja() {
|
|
return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
|
|
} else {
|
|
return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) NamedGlobFile(name string) string {
|
|
return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) UsedEnvFile(tag string) string {
|
|
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
|
|
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag)
|
|
}
|
|
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
|
|
}
|
|
|
|
func (c *configImpl) SoongDocsHtml() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
|
|
}
|
|
|
|
func (c *configImpl) QueryviewMarkerFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
|
|
}
|
|
|
|
func (c *configImpl) ModuleGraphFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
|
|
}
|
|
|
|
func (c *configImpl) ModuleActionsFile() string {
|
|
return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
|
|
}
|
|
|
|
func (c *configImpl) TempDir() string {
|
|
return shared.TempDirForOutDir(c.SoongOutDir())
|
|
}
|
|
|
|
func (c *configImpl) FileListDir() string {
|
|
return filepath.Join(c.OutDir(), ".module_paths")
|
|
}
|
|
|
|
func (c *configImpl) KatiSuffix() string {
|
|
if c.katiSuffix != "" {
|
|
return c.katiSuffix
|
|
}
|
|
panic("SetKatiSuffix has not been called")
|
|
}
|
|
|
|
// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
|
|
// user is interested in additional checks at the expense of build time.
|
|
func (c *configImpl) Checkbuild() bool {
|
|
return c.checkbuild
|
|
}
|
|
|
|
func (c *configImpl) Dist() bool {
|
|
return c.dist
|
|
}
|
|
|
|
func (c *configImpl) JsonModuleGraph() bool {
|
|
return c.jsonModuleGraph
|
|
}
|
|
|
|
func (c *configImpl) Queryview() bool {
|
|
return c.queryview
|
|
}
|
|
|
|
func (c *configImpl) SoongDocs() bool {
|
|
return c.soongDocs
|
|
}
|
|
|
|
func (c *configImpl) IsVerbose() bool {
|
|
return c.verbose
|
|
}
|
|
|
|
func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
|
|
return c.ninjaWeightListSource
|
|
}
|
|
|
|
func (c *configImpl) SkipKati() bool {
|
|
return c.skipKati
|
|
}
|
|
|
|
func (c *configImpl) SkipKatiNinja() bool {
|
|
return c.skipKatiNinja
|
|
}
|
|
|
|
func (c *configImpl) SkipSoong() bool {
|
|
return c.skipSoong
|
|
}
|
|
|
|
func (c *configImpl) SkipNinja() bool {
|
|
return c.skipNinja
|
|
}
|
|
|
|
func (c *configImpl) SetSkipNinja(v bool) {
|
|
c.skipNinja = v
|
|
}
|
|
|
|
func (c *configImpl) SkipConfig() bool {
|
|
return c.skipConfig
|
|
}
|
|
|
|
func (c *configImpl) BuildFromTextStub() bool {
|
|
return !c.buildFromSourceStub
|
|
}
|
|
|
|
func (c *configImpl) TargetProduct() string {
|
|
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
|
|
return v
|
|
}
|
|
panic("TARGET_PRODUCT is not defined")
|
|
}
|
|
|
|
func (c *configImpl) TargetProductOrErr() (string, error) {
|
|
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
|
|
return v, nil
|
|
}
|
|
return "", fmt.Errorf("TARGET_PRODUCT is not defined")
|
|
}
|
|
|
|
func (c *configImpl) TargetDevice() string {
|
|
return c.targetDevice
|
|
}
|
|
|
|
func (c *configImpl) SetTargetDevice(device string) {
|
|
c.targetDevice = device
|
|
}
|
|
|
|
func (c *configImpl) TargetBuildVariant() string {
|
|
if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
|
|
return v
|
|
}
|
|
panic("TARGET_BUILD_VARIANT is not defined")
|
|
}
|
|
|
|
func (c *configImpl) KatiArgs() []string {
|
|
return c.katiArgs
|
|
}
|
|
|
|
func (c *configImpl) Parallel() int {
|
|
return c.parallel
|
|
}
|
|
|
|
func (c *configImpl) GetSourceRootDirs() []string {
|
|
return c.sourceRootDirs
|
|
}
|
|
|
|
func (c *configImpl) SetSourceRootDirs(i []string) {
|
|
c.sourceRootDirs = i
|
|
}
|
|
|
|
func (c *configImpl) GetLogsPrefix() string {
|
|
return c.logsPrefix
|
|
}
|
|
|
|
func (c *configImpl) SetLogsPrefix(prefix string) {
|
|
c.logsPrefix = prefix
|
|
}
|
|
|
|
func (c *configImpl) HighmemParallel() int {
|
|
if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
|
|
return i
|
|
}
|
|
|
|
const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
|
|
parallel := c.Parallel()
|
|
if c.UseRemoteBuild() {
|
|
// Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
|
|
// is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
|
|
// to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
|
|
// Return 1/16th of the size of the local pool, rounding up.
|
|
return (parallel + 15) / 16
|
|
} else if c.totalRAM == 0 {
|
|
// Couldn't detect the total RAM, don't restrict highmem processes.
|
|
return parallel
|
|
} else if c.totalRAM <= 16*1024*1024*1024 {
|
|
// Less than 16GB of ram, restrict to 1 highmem processes
|
|
return 1
|
|
} else if c.totalRAM <= 32*1024*1024*1024 {
|
|
// Less than 32GB of ram, restrict to 2 highmem processes
|
|
return 2
|
|
} else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
|
|
// If less than 8GB total RAM per process, reduce the number of highmem processes
|
|
return p
|
|
}
|
|
// No restriction on highmem processes
|
|
return parallel
|
|
}
|
|
|
|
func (c *configImpl) TotalRAM() uint64 {
|
|
return c.totalRAM
|
|
}
|
|
|
|
// ForceUseGoma determines whether we should override Goma deprecation
|
|
// and use Goma for the current build or not.
|
|
func (c *configImpl) ForceUseGoma() bool {
|
|
if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) UseGoma() bool {
|
|
if v, ok := c.environ.Get("USE_GOMA"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) StartGoma() bool {
|
|
if !c.UseGoma() {
|
|
return false
|
|
}
|
|
|
|
if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) canSupportRBE() bool {
|
|
// Only supported on linux
|
|
if runtime.GOOS != "linux" {
|
|
return false
|
|
}
|
|
|
|
// Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since
|
|
// its unlikely that we will be able to obtain necessary creds without stubby.
|
|
authType, _ := c.rbeAuth()
|
|
if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) UseRBE() bool {
|
|
// These alternate modes of running Soong do not use RBE / reclient.
|
|
if c.Queryview() || c.JsonModuleGraph() {
|
|
return false
|
|
}
|
|
|
|
if !c.canSupportRBE() {
|
|
return false
|
|
}
|
|
|
|
if v, ok := c.Environment().Get("USE_RBE"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (c *configImpl) StartRBE() bool {
|
|
if !c.UseRBE() {
|
|
return false
|
|
}
|
|
|
|
if v, ok := c.environ.Get("NOSTART_RBE"); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) rbeProxyLogsDir() string {
|
|
for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
return c.rbeTmpDir()
|
|
}
|
|
|
|
func (c *configImpl) rbeDownloadTmpDir() string {
|
|
for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
return c.rbeTmpDir()
|
|
}
|
|
|
|
func (c *configImpl) rbeTmpDir() string {
|
|
buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
|
|
return filepath.Join(buildTmpDir, "rbe")
|
|
}
|
|
|
|
func (c *configImpl) rbeCacheDir() string {
|
|
for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
return shared.JoinPath(c.SoongOutDir(), "rbe")
|
|
}
|
|
|
|
func (c *configImpl) shouldCleanupRBELogsDir() bool {
|
|
// Perform a log directory cleanup only when the log directory
|
|
// is auto created by the build rather than user-specified.
|
|
for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
|
|
if _, ok := c.environ.Get(f); ok {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *configImpl) rbeExecRoot() string {
|
|
for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return wd
|
|
}
|
|
|
|
func (c *configImpl) rbeDir() string {
|
|
if v, ok := c.environ.Get("RBE_DIR"); ok {
|
|
return v
|
|
}
|
|
return "prebuilts/remoteexecution-client/live/"
|
|
}
|
|
|
|
func (c *configImpl) rbeReproxy() string {
|
|
for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
return v
|
|
}
|
|
}
|
|
return filepath.Join(c.rbeDir(), "reproxy")
|
|
}
|
|
|
|
func (c *configImpl) rbeAuth() (string, string) {
|
|
credFlags := []string{
|
|
"use_application_default_credentials",
|
|
"use_gce_credentials",
|
|
"credential_file",
|
|
"use_google_prod_creds",
|
|
}
|
|
for _, cf := range credFlags {
|
|
for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
|
|
if v, ok := c.environ.Get(f); ok {
|
|
v = strings.TrimSpace(v)
|
|
if v != "" && v != "false" && v != "0" {
|
|
return "RBE_" + cf, v
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return "RBE_use_application_default_credentials", "true"
|
|
}
|
|
|
|
func (c *configImpl) rbeSockAddr(dir string) (string, error) {
|
|
// Absolute path socket addresses have a prefix of //. This should
|
|
// be included in the length limit.
|
|
maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - 2
|
|
base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
|
|
|
|
name := filepath.Join(dir, base)
|
|
if len(name) < maxNameLen {
|
|
return name, nil
|
|
}
|
|
|
|
name = filepath.Join("/tmp", base)
|
|
if len(name) < maxNameLen {
|
|
return name, nil
|
|
}
|
|
|
|
return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
|
|
}
|
|
|
|
// IsGooglerEnvironment returns true if the current build is running
|
|
// on a Google developer machine and false otherwise.
|
|
func (c *configImpl) IsGooglerEnvironment() bool {
|
|
cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
|
|
if v, ok := c.environ.Get(cf); ok {
|
|
return v == "googler"
|
|
}
|
|
return false
|
|
}
|
|
|
|
// GoogleProdCredsExist determine whether credentials exist on the
|
|
// Googler machine to use remote execution.
|
|
func (c *configImpl) GoogleProdCredsExist() bool {
|
|
if googleProdCredsExistCache {
|
|
return googleProdCredsExistCache
|
|
}
|
|
if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil {
|
|
return false
|
|
}
|
|
googleProdCredsExistCache = true
|
|
return true
|
|
}
|
|
|
|
// UseRemoteBuild indicates whether to use a remote build acceleration system
|
|
// to speed up the build.
|
|
func (c *configImpl) UseRemoteBuild() bool {
|
|
return c.UseGoma() || c.UseRBE()
|
|
}
|
|
|
|
// StubbyExists checks whether the stubby binary exists on the machine running
|
|
// the build.
|
|
func (c *configImpl) StubbyExists() bool {
|
|
if _, err := exec.LookPath("stubby"); err != nil {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// RemoteParallel controls how many remote jobs (i.e., commands which contain
|
|
// gomacc) are run in parallel. Note the parallelism of all other jobs is
|
|
// still limited by Parallel()
|
|
func (c *configImpl) RemoteParallel() int {
|
|
if !c.UseRemoteBuild() {
|
|
return 0
|
|
}
|
|
if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
|
|
return i
|
|
}
|
|
return 500
|
|
}
|
|
|
|
func (c *configImpl) SetKatiArgs(args []string) {
|
|
c.katiArgs = args
|
|
}
|
|
|
|
func (c *configImpl) SetNinjaArgs(args []string) {
|
|
c.ninjaArgs = args
|
|
}
|
|
|
|
func (c *configImpl) SetKatiSuffix(suffix string) {
|
|
c.katiSuffix = suffix
|
|
}
|
|
|
|
func (c *configImpl) LastKatiSuffixFile() string {
|
|
return filepath.Join(c.OutDir(), "last_kati_suffix")
|
|
}
|
|
|
|
func (c *configImpl) HasKatiSuffix() bool {
|
|
return c.katiSuffix != ""
|
|
}
|
|
|
|
func (c *configImpl) KatiEnvFile() string {
|
|
return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
|
|
}
|
|
|
|
func (c *configImpl) KatiBuildNinjaFile() string {
|
|
return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) KatiPackageNinjaFile() string {
|
|
return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) SoongVarsFile() string {
|
|
targetProduct, err := c.TargetProductOrErr()
|
|
if err != nil {
|
|
return filepath.Join(c.SoongOutDir(), "soong.variables")
|
|
} else {
|
|
return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) SoongExtraVarsFile() string {
|
|
targetProduct, err := c.TargetProductOrErr()
|
|
if err != nil {
|
|
return filepath.Join(c.SoongOutDir(), "soong.extra.variables")
|
|
} else {
|
|
return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".extra.variables")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) SoongNinjaFile() string {
|
|
targetProduct, err := c.TargetProductOrErr()
|
|
if err != nil {
|
|
return filepath.Join(c.SoongOutDir(), "build.ninja")
|
|
} else {
|
|
return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) CombinedNinjaFile() string {
|
|
if c.katiSuffix == "" {
|
|
return filepath.Join(c.OutDir(), "combined.ninja")
|
|
}
|
|
return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
|
|
}
|
|
|
|
func (c *configImpl) SoongAndroidMk() string {
|
|
return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
|
|
}
|
|
|
|
func (c *configImpl) SoongMakeVarsMk() string {
|
|
return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
|
|
}
|
|
|
|
func (c *configImpl) SoongBuildMetrics() string {
|
|
return filepath.Join(c.LogsDir(), "soong_build_metrics.pb")
|
|
}
|
|
|
|
func (c *configImpl) ProductOut() string {
|
|
return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
|
|
}
|
|
|
|
func (c *configImpl) DevicePreviousProductConfig() string {
|
|
return filepath.Join(c.ProductOut(), "previous_build_config.mk")
|
|
}
|
|
|
|
func (c *configImpl) KatiPackageMkDir() string {
|
|
return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
|
|
}
|
|
|
|
func (c *configImpl) hostOutRoot() string {
|
|
return filepath.Join(c.OutDir(), "host")
|
|
}
|
|
|
|
func (c *configImpl) HostOut() string {
|
|
return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
|
|
}
|
|
|
|
// This probably needs to be multi-valued, so not exporting it for now
|
|
func (c *configImpl) hostCrossOut() string {
|
|
if runtime.GOOS == "linux" {
|
|
return filepath.Join(c.hostOutRoot(), "windows-x86")
|
|
} else {
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) HostPrebuiltTag() string {
|
|
if runtime.GOOS == "linux" {
|
|
return "linux-x86"
|
|
} else if runtime.GOOS == "darwin" {
|
|
return "darwin-x86"
|
|
} else {
|
|
panic("Unsupported OS")
|
|
}
|
|
}
|
|
|
|
func (c *configImpl) PrebuiltBuildTool(name string) string {
|
|
if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
|
|
if sanitize := strings.Fields(v); inList("address", sanitize) {
|
|
asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
|
|
if _, err := os.Stat(asan); err == nil {
|
|
return asan
|
|
}
|
|
}
|
|
}
|
|
return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenDupRules(val bool) {
|
|
c.brokenDupRules = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenDupRules() bool {
|
|
return c.brokenDupRules
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
|
|
c.brokenUsesNetwork = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenUsesNetwork() bool {
|
|
return c.brokenUsesNetwork
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
|
|
c.brokenNinjaEnvVars = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
|
|
return c.brokenNinjaEnvVars
|
|
}
|
|
|
|
func (c *configImpl) SetBuildBrokenMissingOutputs(val bool) {
|
|
c.brokenMissingOutputs = val
|
|
}
|
|
|
|
func (c *configImpl) BuildBrokenMissingOutputs() bool {
|
|
return c.brokenMissingOutputs
|
|
}
|
|
|
|
func (c *configImpl) SetTargetDeviceDir(dir string) {
|
|
c.targetDeviceDir = dir
|
|
}
|
|
|
|
func (c *configImpl) TargetDeviceDir() string {
|
|
return c.targetDeviceDir
|
|
}
|
|
|
|
func (c *configImpl) BuildDateTime() string {
|
|
return c.buildDateTime
|
|
}
|
|
|
|
func (c *configImpl) MetricsUploaderApp() string {
|
|
return c.metricsUploader
|
|
}
|
|
|
|
// LogsDir returns the absolute path to the logs directory where build log and
|
|
// metrics files are located. By default, the logs directory is the out
|
|
// directory. If the argument dist is specified, the logs directory
|
|
// is <dist_dir>/logs.
|
|
func (c *configImpl) LogsDir() string {
|
|
dir := c.OutDir()
|
|
if c.Dist() {
|
|
// Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
|
|
dir = filepath.Join(c.RealDistDir(), "logs")
|
|
}
|
|
absDir, err := filepath.Abs(dir)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
|
|
os.Exit(1)
|
|
}
|
|
return absDir
|
|
}
|
|
|
|
// MkFileMetrics returns the file path for make-related metrics.
|
|
func (c *configImpl) MkMetrics() string {
|
|
return filepath.Join(c.LogsDir(), "mk_metrics.pb")
|
|
}
|
|
|
|
func (c *configImpl) SetEmptyNinjaFile(v bool) {
|
|
c.emptyNinjaFile = v
|
|
}
|
|
|
|
func (c *configImpl) EmptyNinjaFile() bool {
|
|
return c.emptyNinjaFile
|
|
}
|
|
|
|
func (c *configImpl) SkipMetricsUpload() bool {
|
|
return c.skipMetricsUpload
|
|
}
|
|
|
|
func (c *configImpl) EnsureAllowlistIntegrity() bool {
|
|
return c.ensureAllowlistIntegrity
|
|
}
|
|
|
|
// Returns a Time object if one was passed via a command-line flag.
|
|
// Otherwise returns the passed default.
|
|
func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
|
|
if c.buildStartedTime == 0 {
|
|
return defaultTime
|
|
}
|
|
return time.UnixMilli(c.buildStartedTime)
|
|
}
|
|
|
|
func GetMetricsUploader(topDir string, env *Environment) string {
|
|
if p, ok := env.Get("METRICS_UPLOADER"); ok {
|
|
metricsUploader := filepath.Join(topDir, p)
|
|
if _, err := os.Stat(metricsUploader); err == nil {
|
|
return metricsUploader
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|