Remove bp2build and bazel from soong_ui

Bug: 315353489
Test: m blueprint_tests
Change-Id: I9df31b18caaae24e3cf2994e56bb90b50523f11e
This commit is contained in:
Colin Cross
2023-12-07 10:31:24 -08:00
parent d788b3e6cb
commit 8d411ff8f8
10 changed files with 41 additions and 913 deletions

View File

@@ -18,7 +18,6 @@ import (
"context" "context"
"flag" "flag"
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
@@ -194,16 +193,13 @@ func main() {
buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb") soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
metricsFiles := []string{ metricsFiles := []string{
buildErrorFile, // build error strings buildErrorFile, // build error strings
rbeMetricsFile, // high level metrics related to remote build execution. rbeMetricsFile, // high level metrics related to remote build execution.
bp2buildMetricsFile, // high level metrics related to bp2build.
soongMetricsFile, // high level metrics related to this build system. soongMetricsFile, // high level metrics related to this build system.
soongBuildMetricsFile, // high level metrics related to soong build(except bp2build) soongBuildMetricsFile, // high level metrics related to soong build
config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
} }
os.MkdirAll(logsDir, 0777) os.MkdirAll(logsDir, 0777)
@@ -293,38 +289,12 @@ func preProductConfigSetup(buildCtx build.Context, config build.Config) {
} }
} }
removeBadTargetRename(buildCtx, config)
// Create a source finder. // Create a source finder.
f := build.NewSourceFinder(buildCtx, config) f := build.NewSourceFinder(buildCtx, config)
defer f.Shutdown() defer f.Shutdown()
build.FindSources(buildCtx, config, f) build.FindSources(buildCtx, config, f)
} }
func removeBadTargetRename(ctx build.Context, config build.Config) {
log := ctx.ContextImpl.Logger
// find bad paths
m, err := filepath.Glob(filepath.Join(config.OutDir(), "bazel", "output", "execroot", "__main__", "bazel-out", "mixed_builds_product-*", "bin", "tools", "metalava", "metalava"))
if err != nil {
log.Fatalf("Glob for invalid file failed %s", err)
}
for _, f := range m {
info, err := os.Stat(f)
if err != nil {
log.Fatalf("Stat of invalid file %q failed %s", f, err)
}
// if it's a directory, leave it, but remove the files
if !info.IsDir() {
err = os.Remove(f)
if err != nil {
log.Fatalf("Remove of invalid file %q failed %s", f, err)
} else {
log.Verbosef("Removed %q", f)
}
}
}
}
func dumpVar(ctx build.Context, config build.Config, args []string) { func dumpVar(ctx build.Context, config build.Config, args []string) {
flags := flag.NewFlagSet("dumpvar", flag.ExitOnError) flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
flags.SetOutput(ctx.Writer) flags.SetOutput(ctx.Writer)
@@ -607,81 +577,6 @@ func getCommand(args []string) (*command, []string, error) {
return nil, nil, fmt.Errorf("Command not found: %q\nDid you mean one of these: %q", args[1], listFlags()) return nil, nil, fmt.Errorf("Command not found: %q\nDid you mean one of these: %q", args[1], listFlags())
} }
// For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary.
func populateExternalDistDir(ctx build.Context, config build.Config) {
// Make sure that internalDistDirPath and externalDistDirPath are both absolute paths, so we can compare them
var err error
var internalDistDirPath string
var externalDistDirPath string
if internalDistDirPath, err = filepath.Abs(config.DistDir()); err != nil {
ctx.Fatalf("Unable to find absolute path of %s: %s", internalDistDirPath, err)
}
if externalDistDirPath, err = filepath.Abs(config.RealDistDir()); err != nil {
ctx.Fatalf("Unable to find absolute path of %s: %s", externalDistDirPath, err)
}
if externalDistDirPath == internalDistDirPath {
return
}
// Make sure the internal DIST_DIR actually exists before trying to read from it
if _, err = os.Stat(internalDistDirPath); os.IsNotExist(err) {
ctx.Println("Skipping Bazel dist dir migration - nothing to do!")
return
}
// Make sure the external DIST_DIR actually exists before trying to write to it
if err = os.MkdirAll(externalDistDirPath, 0755); err != nil {
ctx.Fatalf("Unable to make directory %s: %s", externalDistDirPath, err)
}
ctx.Println("Populating external DIST_DIR...")
populateExternalDistDirHelper(ctx, config, internalDistDirPath, externalDistDirPath)
}
func populateExternalDistDirHelper(ctx build.Context, config build.Config, internalDistDirPath string, externalDistDirPath string) {
files, err := ioutil.ReadDir(internalDistDirPath)
if err != nil {
ctx.Fatalf("Can't read internal distdir %s: %s", internalDistDirPath, err)
}
for _, f := range files {
internalFilePath := filepath.Join(internalDistDirPath, f.Name())
externalFilePath := filepath.Join(externalDistDirPath, f.Name())
if f.IsDir() {
// Moving a directory - check if there is an existing directory to merge with
externalLstat, err := os.Lstat(externalFilePath)
if err != nil {
if !os.IsNotExist(err) {
ctx.Fatalf("Can't lstat external %s: %s", externalDistDirPath, err)
}
// Otherwise, if the error was os.IsNotExist, that's fine and we fall through to the rename at the bottom
} else {
if externalLstat.IsDir() {
// Existing dir - try to merge the directories?
populateExternalDistDirHelper(ctx, config, internalFilePath, externalFilePath)
continue
} else {
// Existing file being replaced with a directory. Delete the existing file...
if err := os.RemoveAll(externalFilePath); err != nil {
ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
}
}
}
} else {
// Moving a file (not a dir) - delete any existing file or directory
if err := os.RemoveAll(externalFilePath); err != nil {
ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
}
}
// The actual move - do a rename instead of a copy in order to save disk space.
if err := os.Rename(internalFilePath, externalFilePath); err != nil {
ctx.Fatalf("Unable to rename %s -> %s due to error %s", internalFilePath, externalFilePath, err)
}
}
}
func setMaxFiles(ctx build.Context) { func setMaxFiles(ctx build.Context) {
var limits syscall.Rlimit var limits syscall.Rlimit

View File

@@ -46,7 +46,6 @@ bootstrap_go_package {
"soong-ui-tracer", "soong-ui-tracer",
], ],
srcs: [ srcs: [
"bazel_metrics.go",
"build.go", "build.go",
"cleanbuild.go", "cleanbuild.go",
"config.go", "config.go",
@@ -75,7 +74,6 @@ bootstrap_go_package {
"proc_sync_test.go", "proc_sync_test.go",
"rbe_test.go", "rbe_test.go",
"staging_snapshot_test.go", "staging_snapshot_test.go",
"upload_test.go",
"util_test.go", "util_test.go",
], ],
darwin: { darwin: {

View File

@@ -1,136 +0,0 @@
// Copyright 2023 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package build
// This file contains functionality to parse bazel profile data into
// a bazel_metrics proto, defined in build/soong/ui/metrics/bazel_metrics_proto
// These metrics are later uploaded in upload.go
import (
"bufio"
"os"
"strconv"
"strings"
"android/soong/shared"
"google.golang.org/protobuf/proto"
bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto"
)
func parseTimingToNanos(str string) int64 {
millisString := removeDecimalPoint(str)
timingMillis, _ := strconv.ParseInt(millisString, 10, 64)
return timingMillis * 1000000
}
func parsePercentageToTenThousandths(str string) int32 {
percentageString := removeDecimalPoint(str)
//remove the % at the end of the string
percentage := strings.ReplaceAll(percentageString, "%", "")
percentagePortion, _ := strconv.ParseInt(percentage, 10, 32)
return int32(percentagePortion)
}
func removeDecimalPoint(numString string) string {
// The format is always 0.425 or 10.425
return strings.ReplaceAll(numString, ".", "")
}
func parseTotal(line string) int64 {
words := strings.Fields(line)
timing := words[3]
return parseTimingToNanos(timing)
}
func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming {
words := strings.Fields(line)
getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) {
// Sample lines include:
// Total launch phase time 0.011 s 2.59%
// Total target pattern evaluation phase time 0.011 s 2.59%
var beginning int
var end int
for ind, word := range words {
if word == "Total" {
beginning = ind + 1
} else if beginning > 0 && word == "phase" {
end = ind
break
}
}
phaseName := strings.Join(words[beginning:end], " ")
// end is now "phase" - advance by 2 for timing and 4 for percentage
percentageString := words[end+4]
timingString := words[end+2]
timing := parseTimingToNanos(timingString)
percentagePortion := parsePercentageToTenThousandths(percentageString)
return phaseName, timing, percentagePortion
}
phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words)
phaseTiming := bazel_metrics_proto.PhaseTiming{}
phaseTiming.DurationNanos = &timing
phaseTiming.PortionOfBuildTime = &portion
phaseTiming.PhaseName = &phaseName
return phaseTiming
}
// This method takes a file created by bazel's --analyze-profile mode and
// writes bazel metrics data to the provided filepath.
func ProcessBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context, config Config) {
if bazelProfileFile == "" {
return
}
readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics {
//serialize the proto, write it
bazelMetrics := bazel_metrics_proto.BazelMetrics{}
file, err := os.ReadFile(filepath)
if err != nil {
ctx.Fatalln("Error reading metrics file\n", err)
}
scanner := bufio.NewScanner(strings.NewReader(string(file)))
scanner.Split(bufio.ScanLines)
var phaseTimings []*bazel_metrics_proto.PhaseTiming
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Total run time") {
total := parseTotal(line)
bazelMetrics.Total = &total
} else if strings.HasPrefix(line, "Total") {
phaseTiming := parsePhaseTiming(line)
phaseTimings = append(phaseTimings, &phaseTiming)
}
}
bazelMetrics.PhaseTimings = phaseTimings
bazelMetrics.BesId = proto.String(config.besId)
return bazelMetrics
}
if _, err := os.Stat(bazelProfileFile); err != nil {
// We can assume bazel didn't run if the profile doesn't exist
return
}
bazelProto := readBazelProto(bazelProfileFile)
bazelProto.ExitCode = proto.Int32(config.bazelExitCode)
shared.Save(&bazelProto, bazelMetricsFile)
}

View File

@@ -135,22 +135,6 @@ const (
RunBuildTests = 1 << iota RunBuildTests = 1 << iota
) )
// checkBazelMode fails the build if there are conflicting arguments for which bazel
// build mode to use.
func checkBazelMode(ctx Context, config Config) {
count := 0
if config.bazelProdMode {
count++
}
if config.bazelStagingMode {
count++
}
if count > 1 {
ctx.Fatalln("Conflicting bazel mode.\n" +
"Do not specify more than one of --bazel-mode and --bazel-mode-staging ")
}
}
// checkProblematicFiles fails the build if existing Android.mk or CleanSpec.mk files are found at the root of the tree. // checkProblematicFiles fails the build if existing Android.mk or CleanSpec.mk files are found at the root of the tree.
func checkProblematicFiles(ctx Context) { func checkProblematicFiles(ctx Context) {
files := []string{"Android.mk", "CleanSpec.mk"} files := []string{"Android.mk", "CleanSpec.mk"}
@@ -262,8 +246,6 @@ func Build(ctx Context, config Config) {
defer waitForDist(ctx) defer waitForDist(ctx)
checkBazelMode(ctx, config)
// checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree. // checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree.
checkProblematicFiles(ctx) checkProblematicFiles(ctx)

View File

@@ -72,7 +72,6 @@ type configImpl struct {
checkbuild bool checkbuild bool
dist bool dist bool
jsonModuleGraph bool jsonModuleGraph bool
bp2build bool
queryview bool queryview bool
reportMkMetrics bool // Collect and report mk2bp migration progress metrics. reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
soongDocs bool soongDocs bool
@@ -88,8 +87,6 @@ type configImpl struct {
buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
buildFromSourceStub bool buildFromSourceStub bool
ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
bazelExitCode int32 // For b runs - necessary for updating NonZeroExit
besId string // For b runs, to identify the BuildEventService logs
// From the product config // From the product config
katiArgs []string katiArgs []string
@@ -108,16 +105,11 @@ type configImpl struct {
pathReplaced bool pathReplaced bool
bazelProdMode bool
bazelStagingMode bool
// Set by multiproduct_kati // Set by multiproduct_kati
emptyNinjaFile bool emptyNinjaFile bool
metricsUploader string metricsUploader string
bazelForceEnabledModules string
includeTags []string includeTags []string
sourceRootDirs []string sourceRootDirs []string
@@ -450,11 +442,6 @@ func NewConfig(ctx Context, args ...string) Config {
} }
} }
bpd := ret.BazelMetricsDir()
if err := os.RemoveAll(bpd); err != nil {
ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
}
c := Config{ret} c := Config{ret}
storeConfigMetrics(ctx, c) storeConfigMetrics(ctx, c)
return c return c
@@ -538,8 +525,6 @@ func buildConfig(config Config) *smpb.BuildConfig {
ForceUseGoma: proto.Bool(config.ForceUseGoma()), ForceUseGoma: proto.Bool(config.ForceUseGoma()),
UseGoma: proto.Bool(config.UseGoma()), UseGoma: proto.Bool(config.UseGoma()),
UseRbe: proto.Bool(config.UseRBE()), UseRbe: proto.Bool(config.UseRBE()),
BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()), NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
} }
c.Targets = append(c.Targets, config.arguments...) c.Targets = append(c.Targets, config.arguments...)
@@ -794,10 +779,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
c.reportMkMetrics = true c.reportMkMetrics = true
} else if arg == "--multitree-build" { } else if arg == "--multitree-build" {
c.multitreeBuild = true c.multitreeBuild = true
} else if arg == "--bazel-mode" {
c.bazelProdMode = true
} else if arg == "--bazel-mode-staging" {
c.bazelStagingMode = true
} else if arg == "--search-api-dir" { } else if arg == "--search-api-dir" {
c.searchApiDir = true c.searchApiDir = true
} else if strings.HasPrefix(arg, "--ninja_weight_source=") { } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
@@ -832,8 +813,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
buildCmd = strings.TrimPrefix(buildCmd, "\"") buildCmd = strings.TrimPrefix(buildCmd, "\"")
buildCmd = strings.TrimSuffix(buildCmd, "\"") buildCmd = strings.TrimSuffix(buildCmd, "\"")
ctx.Metrics.SetBuildCommand([]string{buildCmd}) ctx.Metrics.SetBuildCommand([]string{buildCmd})
} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
} else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") { } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=") buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
val, err := strconv.ParseInt(buildTimeStr, 10, 64) val, err := strconv.ParseInt(buildTimeStr, 10, 64)
@@ -878,8 +857,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
c.dist = true c.dist = true
} else if arg == "json-module-graph" { } else if arg == "json-module-graph" {
c.jsonModuleGraph = true c.jsonModuleGraph = true
} else if arg == "bp2build" {
c.bp2build = true
} else if arg == "queryview" { } else if arg == "queryview" {
c.queryview = true c.queryview = true
} else if arg == "soong_docs" { } else if arg == "soong_docs" {
@@ -976,13 +953,12 @@ func (c *configImpl) SoongBuildInvocationNeeded() bool {
return true return true
} }
if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() { if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() {
// Command line was empty, the default Ninja target is built // Command line was empty, the default Ninja target is built
return true return true
} }
// bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation if c.Dist() {
if c.Dist() && !c.Bp2Build() {
return true return true
} }
@@ -1012,14 +988,6 @@ func (c *configImpl) NinjaArgs() []string {
return c.ninjaArgs return c.ninjaArgs
} }
func (c *configImpl) BazelOutDir() string {
return filepath.Join(c.OutDir(), "bazel")
}
func (c *configImpl) bazelOutputBase() string {
return filepath.Join(c.BazelOutDir(), "output")
}
func (c *configImpl) SoongOutDir() string { func (c *configImpl) SoongOutDir() string {
return filepath.Join(c.OutDir(), "soong") return filepath.Join(c.OutDir(), "soong")
} }
@@ -1058,14 +1026,6 @@ func (c *configImpl) UsedEnvFile(tag string) string {
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
} }
func (c *configImpl) Bp2BuildFilesMarkerFile() string {
return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
}
func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
}
func (c *configImpl) SoongDocsHtml() string { func (c *configImpl) SoongDocsHtml() string {
return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
} }
@@ -1111,10 +1071,6 @@ func (c *configImpl) JsonModuleGraph() bool {
return c.jsonModuleGraph return c.jsonModuleGraph
} }
func (c *configImpl) Bp2Build() bool {
return c.bp2build
}
func (c *configImpl) Queryview() bool { func (c *configImpl) Queryview() bool {
return c.queryview return c.queryview
} }
@@ -1306,7 +1262,7 @@ func (c *configImpl) canSupportRBE() bool {
func (c *configImpl) UseRBE() bool { func (c *configImpl) UseRBE() bool {
// These alternate modes of running Soong do not use RBE / reclient. // These alternate modes of running Soong do not use RBE / reclient.
if c.Bp2Build() || c.Queryview() || c.JsonModuleGraph() { if c.Queryview() || c.JsonModuleGraph() {
return false return false
} }
@@ -1323,10 +1279,6 @@ func (c *configImpl) UseRBE() bool {
return false return false
} }
func (c *configImpl) BazelBuildEnabled() bool {
return c.bazelProdMode || c.bazelStagingMode
}
func (c *configImpl) StartRBE() bool { func (c *configImpl) StartRBE() bool {
if !c.UseRBE() { if !c.UseRBE() {
return false return false
@@ -1679,12 +1631,6 @@ func (c *configImpl) LogsDir() string {
return absDir return absDir
} }
// BazelMetricsDir returns the <logs dir>/bazel_metrics directory
// where the bazel profiles are located.
func (c *configImpl) BazelMetricsDir() string {
return filepath.Join(c.LogsDir(), "bazel_metrics")
}
// MkFileMetrics returns the file path for make-related metrics. // MkFileMetrics returns the file path for make-related metrics.
func (c *configImpl) MkMetrics() string { func (c *configImpl) MkMetrics() string {
return filepath.Join(c.LogsDir(), "mk_metrics.pb") return filepath.Join(c.LogsDir(), "mk_metrics.pb")
@@ -1698,28 +1644,6 @@ func (c *configImpl) EmptyNinjaFile() bool {
return c.emptyNinjaFile return c.emptyNinjaFile
} }
func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
}
func (c *configImpl) IsPersistentBazelEnabled() bool {
return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
}
// GetBazeliskBazelVersion returns the Bazel version to use for this build,
// or the empty string if the current canonical prod Bazel should be used.
// This environment variable should only be set to debug the build system.
// The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
// handle downloading and invoking the correct Bazel binary.
func (c *configImpl) GetBazeliskBazelVersion() string {
value, _ := c.Environment().Get("USE_BAZEL_VERSION")
return value
}
func (c *configImpl) BazelModulesForceEnabledByFlag() string {
return c.bazelForceEnabledModules
}
func (c *configImpl) SkipMetricsUpload() bool { func (c *configImpl) SkipMetricsUpload() bool {
return c.skipMetricsUpload return c.skipMetricsUpload
} }
@@ -1737,10 +1661,6 @@ func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time
return time.UnixMilli(c.buildStartedTime) return time.UnixMilli(c.buildStartedTime)
} }
func (c *configImpl) BazelExitCode() int32 {
return c.bazelExitCode
}
func GetMetricsUploader(topDir string, env *Environment) string { func GetMetricsUploader(topDir string, env *Environment) string {
if p, ok := env.Get("METRICS_UPLOADER"); ok { if p, ok := env.Get("METRICS_UPLOADER"); ok {
metricsUploader := filepath.Join(topDir, p) metricsUploader := filepath.Join(topDir, p)

View File

@@ -1017,9 +1017,6 @@ func TestBuildConfig(t *testing.T) {
name string name string
environ Environment environ Environment
arguments []string arguments []string
useBazel bool
bazelProdMode bool
bazelStagingMode bool
expectedBuildConfig *smpb.BuildConfig expectedBuildConfig *smpb.BuildConfig
}{ }{
{ {
@@ -1029,8 +1026,6 @@ func TestBuildConfig(t *testing.T) {
ForceUseGoma: proto.Bool(false), ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false), UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false), UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(false),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(), NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
}, },
}, },
@@ -1041,8 +1036,6 @@ func TestBuildConfig(t *testing.T) {
ForceUseGoma: proto.Bool(true), ForceUseGoma: proto.Bool(true),
UseGoma: proto.Bool(false), UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false), UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(false),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(), NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
}, },
}, },
@@ -1053,8 +1046,6 @@ func TestBuildConfig(t *testing.T) {
ForceUseGoma: proto.Bool(false), ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(true), UseGoma: proto.Bool(true),
UseRbe: proto.Bool(false), UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(false),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(), NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
}, },
}, },
@@ -1065,109 +1056,18 @@ func TestBuildConfig(t *testing.T) {
ForceUseGoma: proto.Bool(false), ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false), UseGoma: proto.Bool(false),
UseRbe: proto.Bool(true), UseRbe: proto.Bool(true),
BazelMixedBuild: proto.Bool(false),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
},
},
{
name: "disable mixed builds",
environ: Environment{"BUILD_BROKEN_DISABLE_BAZEL=1"},
expectedBuildConfig: &smpb.BuildConfig{
ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(false),
ForceDisableBazelMixedBuild: proto.Bool(true),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
},
},
{
name: "use bazel as ninja",
environ: Environment{},
useBazel: true,
expectedBuildConfig: &smpb.BuildConfig{
ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(false),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
},
},
{
name: "bazel mixed build from prod mode",
environ: Environment{},
bazelProdMode: true,
expectedBuildConfig: &smpb.BuildConfig{
ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(true),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
},
},
{
name: "bazel mixed build from staging mode",
environ: Environment{},
bazelStagingMode: true,
expectedBuildConfig: &smpb.BuildConfig{
ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(true),
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
},
},
{
name: "specified targets",
environ: Environment{},
useBazel: true,
arguments: []string{"droid", "dist"},
expectedBuildConfig: &smpb.BuildConfig{
ForceUseGoma: proto.Bool(false),
UseGoma: proto.Bool(false),
UseRbe: proto.Bool(false),
BazelMixedBuild: proto.Bool(false),
Targets: []string{"droid", "dist"},
ForceDisableBazelMixedBuild: proto.Bool(false),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
},
},
{
name: "all set",
environ: Environment{
"FORCE_USE_GOMA=1",
"USE_GOMA=1",
"USE_RBE=1",
"BUILD_BROKEN_DISABLE_BAZEL=1",
},
useBazel: true,
bazelProdMode: true,
expectedBuildConfig: &smpb.BuildConfig{
ForceUseGoma: proto.Bool(true),
UseGoma: proto.Bool(true),
UseRbe: proto.Bool(true),
BazelMixedBuild: proto.Bool(true),
ForceDisableBazelMixedBuild: proto.Bool(true),
NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(), NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(),
}, },
}, },
} }
ctx := testContext()
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
c := &configImpl{ c := &configImpl{
environ: &tc.environ, environ: &tc.environ,
bazelProdMode: tc.bazelProdMode,
bazelStagingMode: tc.bazelStagingMode,
arguments: tc.arguments, arguments: tc.arguments,
} }
config := Config{c} config := Config{c}
checkBazelMode(ctx, config)
actualBuildConfig := buildConfig(config) actualBuildConfig := buildConfig(config)
if expected := tc.expectedBuildConfig; !proto.Equal(expected, actualBuildConfig) { if expected := tc.expectedBuildConfig; !proto.Equal(expected, actualBuildConfig) {
t.Errorf("Build config mismatch.\n"+ t.Errorf("Build config mismatch.\n"+

View File

@@ -74,10 +74,6 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
"AndroidProducts.mk", "AndroidProducts.mk",
// General Soong build definitions, using the Blueprint syntax. // General Soong build definitions, using the Blueprint syntax.
"Android.bp", "Android.bp",
// Bazel build definitions.
"BUILD.bazel",
// Bazel build definitions.
"BUILD",
// Kati clean definitions. // Kati clean definitions.
"CleanSpec.mk", "CleanSpec.mk",
// Ownership definition. // Ownership definition.
@@ -85,13 +81,11 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
// Test configuration for modules in directories that contain this // Test configuration for modules in directories that contain this
// file. // file.
"TEST_MAPPING", "TEST_MAPPING",
// Bazel top-level file to mark a directory as a Bazel workspace.
"WORKSPACE",
// METADATA file of packages // METADATA file of packages
"METADATA", "METADATA",
}, },
// Bazel Starlark configuration files and all .mk files for product/board configuration. // .mk files for product/board configuration.
IncludeSuffixes: []string{".bzl", ".mk"}, IncludeSuffixes: []string{".mk"},
} }
dumpDir := config.FileListDir() dumpDir := config.FileListDir()
f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard), f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
@@ -111,17 +105,6 @@ func androidBpSearchDirs(config Config) []string {
return dirs return dirs
} }
// Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree.
func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
matches := []string{}
for _, foundName := range entries.FileNames {
if foundName == "BUILD.bazel" || foundName == "BUILD" || foundName == "WORKSPACE" || strings.HasSuffix(foundName, ".bzl") {
matches = append(matches, foundName)
}
}
return entries.DirNames, matches
}
func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
matches := []string{} matches := []string{}
for _, foundName := range entries.FileNames { for _, foundName := range entries.FileNames {
@@ -177,13 +160,6 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
ctx.Fatalf("Could not export product list: %v", err) ctx.Fatalf("Could not export product list: %v", err)
} }
// Recursively look for all Bazel related files.
bazelFiles := f.FindMatching(".", findBazelFiles)
err = dumpListToFile(ctx, config, bazelFiles, filepath.Join(dumpDir, "bazel.list"))
if err != nil {
ctx.Fatalf("Could not export bazel BUILD list: %v", err)
}
// Recursively look for all OWNERS files. // Recursively look for all OWNERS files.
owners := f.FindNamedAt(".", "OWNERS") owners := f.FindNamedAt(".", "OWNERS")
err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list")) err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list"))

View File

@@ -45,8 +45,6 @@ const (
usedEnvFile = "soong.environment.used" usedEnvFile = "soong.environment.used"
soongBuildTag = "build" soongBuildTag = "build"
bp2buildFilesTag = "bp2build_files"
bp2buildWorkspaceTag = "bp2build_workspace"
jsonModuleGraphTag = "modulegraph" jsonModuleGraphTag = "modulegraph"
queryviewTag = "queryview" queryviewTag = "queryview"
soongDocsTag = "soong_docs" soongDocsTag = "soong_docs"
@@ -275,7 +273,6 @@ func bootstrapEpochCleanup(ctx Context, config Config) {
func bootstrapGlobFileList(config Config) []string { func bootstrapGlobFileList(config Config) []string {
return []string{ return []string{
config.NamedGlobFile(getGlobPathName(config)), config.NamedGlobFile(getGlobPathName(config)),
config.NamedGlobFile(bp2buildFilesTag),
config.NamedGlobFile(jsonModuleGraphTag), config.NamedGlobFile(jsonModuleGraphTag),
config.NamedGlobFile(queryviewTag), config.NamedGlobFile(queryviewTag),
config.NamedGlobFile(soongDocsTag), config.NamedGlobFile(soongDocsTag),
@@ -295,18 +292,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
if config.EmptyNinjaFile() { if config.EmptyNinjaFile() {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file") mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
} }
if config.bazelProdMode {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
}
if config.bazelStagingMode {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
}
if config.IsPersistentBazelEnabled() {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
}
if len(config.bazelForceEnabledModules) > 0 {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
}
if config.MultitreeBuild() { if config.MultitreeBuild() {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build") mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build")
} }
@@ -327,24 +312,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
output: config.SoongNinjaFile(), output: config.SoongNinjaFile(),
specificArgs: mainSoongBuildExtraArgs, specificArgs: mainSoongBuildExtraArgs,
}, },
{
name: bp2buildFilesTag,
description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
config: config,
output: config.Bp2BuildFilesMarkerFile(),
specificArgs: append(baseArgs,
"--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
),
},
{
name: bp2buildWorkspaceTag,
description: "Creating Bazel symlink forest",
config: config,
output: config.Bp2BuildWorkspaceMarkerFile(),
specificArgs: append(baseArgs,
"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
),
},
{ {
name: jsonModuleGraphTag, name: jsonModuleGraphTag,
description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()), description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
@@ -410,22 +377,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
pbf.debugPort = delvePort pbf.debugPort = delvePort
} }
pbi := pbf.primaryBuilderInvocation() pbi := pbf.primaryBuilderInvocation()
// Some invocations require adjustment:
switch pbf.name {
case soongBuildTag:
if config.BazelBuildEnabled() {
// Mixed builds call Bazel from soong_build and they therefore need the
// Bazel workspace to be available. Make that so by adding a dependency on
// the bp2build marker file to the action that invokes soong_build .
pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
}
case bp2buildWorkspaceTag:
pbi.Inputs = append(pbi.Inputs,
config.Bp2BuildFilesMarkerFile(),
filepath.Join(config.FileListDir(), "bazel.list"))
case bp2buildFilesTag:
pbi.Inputs = append(pbi.Inputs, filepath.Join(config.FileListDir(), "METADATA.list"))
}
invocations = append(invocations, pbi) invocations = append(invocations, pbi)
} }
@@ -603,16 +554,7 @@ func runSoong(ctx Context, config Config) {
soongBuildEnv := config.Environment().Copy() soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP")) soongBuildEnv.Set("TOP", os.Getenv("TOP"))
// For Bazel mixed builds.
soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
// Bazel's HOME var is set to an output subdirectory which doesn't exist. This
// prevents Bazel from file I/O in the actual user HOME directory.
soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
soongBuildEnv.Set("LOG_DIR", config.LogsDir()) soongBuildEnv.Set("LOG_DIR", config.LogsDir())
soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
// For Soong bootstrapping tests // For Soong bootstrapping tests
if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" { if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
@@ -630,13 +572,9 @@ func runSoong(ctx Context, config Config) {
checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag)) checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag))
if config.BazelBuildEnabled() || config.Bp2Build() {
checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
} else {
// Remove bazel files in the event that bazel is disabled for the build. // Remove bazel files in the event that bazel is disabled for the build.
// These files may have been left over from a previous bazel-enabled build. // These files may have been left over from a previous bazel-enabled build.
cleanBazelFiles(config) cleanBazelFiles(config)
}
if config.JsonModuleGraph() { if config.JsonModuleGraph() {
checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag)) checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
@@ -658,14 +596,6 @@ func runSoong(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "bootstrap") ctx.BeginTrace(metrics.RunSoong, "bootstrap")
defer ctx.EndTrace() defer ctx.EndTrace()
if config.IsPersistentBazelEnabled() {
bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"), config.GetBazeliskBazelVersion())
if err := bazelProxy.Start(); err != nil {
ctx.Fatalf("Failed to create bazel proxy")
}
defer bazelProxy.Close()
}
fifo := filepath.Join(config.OutDir(), ".ninja_fifo") fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo) nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
defer nr.Close() defer nr.Close()
@@ -709,10 +639,6 @@ func runSoong(ctx Context, config Config) {
targets = append(targets, config.ModuleGraphFile()) targets = append(targets, config.ModuleGraphFile())
} }
if config.Bp2Build() {
targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
}
if config.Queryview() { if config.Queryview() {
targets = append(targets, config.QueryviewMarkerFile()) targets = append(targets, config.QueryviewMarkerFile())
} }

View File

@@ -15,47 +15,16 @@
package build package build
import ( import (
"android/soong/ui/metrics"
"android/soong/ui/status"
"bufio" "bufio"
"fmt" "fmt"
"path/filepath" "path/filepath"
"regexp"
"runtime" "runtime"
"sort" "sort"
"strings" "strings"
"sync"
"android/soong/ui/metrics"
"android/soong/ui/status"
) )
var (
// bazel output paths are in __main__/bazel-out/<config-specific-path>/bin
bazelOutputPathRegexOnce sync.Once
bazelOutputPathRegexp *regexp.Regexp
)
func bazelOutputPathPattern(config Config) *regexp.Regexp {
bazelOutputPathRegexOnce.Do(func() {
// Bazel output files are in <Bazel output base>/execroot/__main__/bazel-out/<config>/bin
bazelOutRoot := filepath.Join(regexp.QuoteMeta(config.bazelOutputBase()), "execroot", "__main__", "bazel-out")
bazelOutputPathRegexp = regexp.MustCompile(bazelOutRoot + "/[^/]+/bin")
})
return bazelOutputPathRegexp
}
func ignoreBazelPath(config Config, path string) bool {
bazelRoot := filepath.Join(config.bazelOutputBase(), "execroot")
// Don't check bazel output regexp unless it is Bazel path
if strings.HasPrefix(path, bazelRoot) {
bazelOutputRegexp := bazelOutputPathPattern(config)
// if the file is a bazel path that is _not_ a Bazel generated file output, we rely on Bazel to
// ensure the paths to exist. If it _is_ a Bazel output path, we expect that it should be built
// by Ninja.
return !bazelOutputRegexp.MatchString(path)
}
return false
}
// Checks for files in the out directory that have a rule that depends on them but no rule to // Checks for files in the out directory that have a rule that depends on them but no rule to
// create them. This catches a common set of build failures where a rule to generate a file is // create them. This catches a common set of build failures where a rule to generate a file is
// deleted (either by deleting a module in an Android.mk file, or by modifying the build system // deleted (either by deleting a module in an Android.mk file, or by modifying the build system
@@ -128,9 +97,6 @@ func testForDanglingRules(ctx Context, config Config) {
continue continue
} }
if ignoreBazelPath(config, line) {
continue
}
danglingRules[line] = true danglingRules[line] = true
} }

View File

@@ -1,299 +0,0 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package build
import (
"errors"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"time"
"android/soong/ui/logger"
)
func writeBazelProfileFile(dir string) error {
contents := `
=== PHASE SUMMARY INFORMATION ===
Total launch phase time 1.193 s 15.77%
Total init phase time 1.092 s 14.44%
Total target pattern evaluation phase time 0.580 s 7.67%
Total interleaved loading-and-analysis phase time 3.646 s 48.21%
Total preparation phase time 0.022 s 0.30%
Total execution phase time 0.993 s 13.13%
Total finish phase time 0.036 s 0.48%
---------------------------------------------------------------------
Total run time 7.563 s 100.00%
Critical path (178 ms):
Time Percentage Description
178 ms 100.00% action 'BazelWorkspaceStatusAction stable-status.txt'
`
file := filepath.Join(dir, "bazel_metrics.txt")
return os.WriteFile(file, []byte(contents), 0666)
}
func TestPruneMetricsFiles(t *testing.T) {
rootDir := t.TempDir()
dirs := []string{
filepath.Join(rootDir, "d1"),
filepath.Join(rootDir, "d1", "d2"),
filepath.Join(rootDir, "d1", "d2", "d3"),
}
files := []string{
filepath.Join(rootDir, "d1", "f1"),
filepath.Join(rootDir, "d1", "d2", "f1"),
filepath.Join(rootDir, "d1", "d2", "d3", "f1"),
}
for _, d := range dirs {
if err := os.MkdirAll(d, 0777); err != nil {
t.Fatalf("got %v, expecting nil error for making directory %q", err, d)
}
}
for _, f := range files {
if err := ioutil.WriteFile(f, []byte{}, 0777); err != nil {
t.Fatalf("got %v, expecting nil error on writing file %q", err, f)
}
}
want := []string{
filepath.Join(rootDir, "d1", "f1"),
filepath.Join(rootDir, "d1", "d2", "f1"),
filepath.Join(rootDir, "d1", "d2", "d3", "f1"),
}
got := pruneMetricsFiles([]string{rootDir})
sort.Strings(got)
sort.Strings(want)
if !reflect.DeepEqual(got, want) {
t.Errorf("got %q, want %q after pruning metrics files", got, want)
}
}
func TestUploadMetrics(t *testing.T) {
ctx := testContext()
tests := []struct {
description string
uploader string
createFiles bool
files []string
}{{
description: "no metrics uploader",
}, {
description: "non-existent metrics files no upload",
uploader: "echo",
files: []string{"metrics_file_1", "metrics_file_2", "metrics_file_3, bazel_metrics.pb"},
}, {
description: "trigger upload",
uploader: "echo",
createFiles: true,
files: []string{"metrics_file_1", "metrics_file_2, bazel_metrics.pb"},
}}
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
defer logger.Recover(func(err error) {
t.Fatalf("got unexpected error: %v", err)
})
outDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("failed to create out directory: %v", outDir)
}
defer os.RemoveAll(outDir)
// Supply our own tmpDir to delete the temp dir once the test is done.
orgTmpDir := tmpDir
tmpDir = func(string, string) (string, error) {
retDir := filepath.Join(outDir, "tmp_upload_dir")
if err := os.Mkdir(retDir, 0755); err != nil {
t.Fatalf("failed to create temporary directory %q: %v", retDir, err)
}
return retDir, nil
}
defer func() { tmpDir = orgTmpDir }()
metricsUploadDir := filepath.Join(outDir, ".metrics_uploader")
if err := os.Mkdir(metricsUploadDir, 0755); err != nil {
t.Fatalf("failed to create %q directory for oauth valid check: %v", metricsUploadDir, err)
}
var metricsFiles []string
if tt.createFiles {
for _, f := range tt.files {
filename := filepath.Join(outDir, f)
metricsFiles = append(metricsFiles, filename)
if err := ioutil.WriteFile(filename, []byte("test file"), 0644); err != nil {
t.Fatalf("failed to create a fake metrics file %q for uploading: %v", filename, err)
}
}
}
if err := writeBazelProfileFile(outDir); err != nil {
t.Fatalf("failed to create bazel profile file in dir: %v", outDir)
}
config := Config{&configImpl{
environ: &Environment{
"OUT_DIR=" + outDir,
},
buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10),
metricsUploader: tt.uploader,
}}
UploadMetrics(ctx, config, false, time.Now(), metricsFiles...)
})
}
}
func TestUploadMetricsErrors(t *testing.T) {
ctx := testContext()
tests := []struct {
description string
tmpDir string
tmpDirErr error
expectedErr string
}{{
description: "getTmpDir returned error",
tmpDirErr: errors.New("getTmpDir failed"),
expectedErr: "getTmpDir failed",
}, {
description: "copyFile operation error",
tmpDir: "/fake_dir",
expectedErr: "failed to copy",
}}
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
defer logger.Recover(func(err error) {
got := err.Error()
if !strings.Contains(got, tt.expectedErr) {
t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr)
}
})
outDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("failed to create out directory: %v", outDir)
}
defer os.RemoveAll(outDir)
orgTmpDir := tmpDir
tmpDir = func(string, string) (string, error) {
return tt.tmpDir, tt.tmpDirErr
}
defer func() { tmpDir = orgTmpDir }()
metricsFile := filepath.Join(outDir, "metrics_file_1")
if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil {
t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err)
}
config := Config{&configImpl{
environ: &Environment{
"OUT_DIR=/bad",
},
metricsUploader: "echo",
}}
UploadMetrics(ctx, config, true, time.Now(), metricsFile)
t.Errorf("got nil, expecting %q as a failure", tt.expectedErr)
})
}
}
func TestParsePercentageToTenThousandths(t *testing.T) {
// 2.59% should be returned as 259 - representing 259/10000 of the build
percentage := parsePercentageToTenThousandths("2.59%")
if percentage != 259 {
t.Errorf("Expected percentage to be returned as ten-thousandths. Expected 259, have %d\n", percentage)
}
// Test without a leading digit
percentage = parsePercentageToTenThousandths(".52%")
if percentage != 52 {
t.Errorf("Expected percentage to be returned as ten-thousandths. Expected 52, have %d\n", percentage)
}
}
func TestParseTimingToNanos(t *testing.T) {
// This parses from seconds (with millis precision) and returns nanos
timingNanos := parseTimingToNanos("0.111")
if timingNanos != 111000000 {
t.Errorf("Error parsing timing. Expected 111000, have %d\n", timingNanos)
}
// Test without a leading digit
timingNanos = parseTimingToNanos(".112")
if timingNanos != 112000000 {
t.Errorf("Error parsing timing. Expected 112000, have %d\n", timingNanos)
}
}
func TestParsePhaseTiming(t *testing.T) {
// Sample lines include:
// Total launch phase time 0.011 s 2.59%
// Total target pattern evaluation phase time 0.012 s 4.59%
line1 := "Total launch phase time 0.011 s 2.59%"
timing := parsePhaseTiming(line1)
if timing.GetPhaseName() != "launch" {
t.Errorf("Failed to parse phase name. Expected launch, have %s\n", timing.GetPhaseName())
} else if timing.GetDurationNanos() != 11000000 {
t.Errorf("Failed to parse duration nanos. Expected 11000000, have %d\n", timing.GetDurationNanos())
} else if timing.GetPortionOfBuildTime() != 259 {
t.Errorf("Failed to parse portion of build time. Expected 259, have %d\n", timing.GetPortionOfBuildTime())
}
// Test with a multiword phase name
line2 := "Total target pattern evaluation phase time 0.012 s 4.59%"
timing = parsePhaseTiming(line2)
if timing.GetPhaseName() != "target pattern evaluation" {
t.Errorf("Failed to parse phase name. Expected target pattern evaluation, have %s\n", timing.GetPhaseName())
} else if timing.GetDurationNanos() != 12000000 {
t.Errorf("Failed to parse duration nanos. Expected 12000000, have %d\n", timing.GetDurationNanos())
} else if timing.GetPortionOfBuildTime() != 459 {
t.Errorf("Failed to parse portion of build time. Expected 459, have %d\n", timing.GetPortionOfBuildTime())
}
}
func TestParseTotal(t *testing.T) {
// Total line is in the form of:
// Total run time 7.563 s 100.00%
line := "Total run time 7.563 s 100.00%"
total := parseTotal(line)
// Only the seconds field is parsed, as nanos
if total != 7563000000 {
t.Errorf("Failed to parse total build time. Expected 7563000000, have %d\n", total)
}
}