From b64fc1cab5d9238d02584c137b98bbd3ea249819 Mon Sep 17 00:00:00 2001 From: Jeff Gaston Date: Fri, 4 Aug 2017 12:30:12 -0700 Subject: [PATCH] Run the Finder and make its results available to Kati The Finder runs roughly 200ms faster than findleaves.py in aosp, and runs roughly 400ms faster in internal master. Bug: 64363847 Test: m -j Change-Id: I62db8dacc90871e913576fe2443021fb1749a483 --- cmd/multiproduct_kati/main.go | 5 ++ cmd/soong_ui/main.go | 4 ++ finder/finder.go | 44 +++++++++++---- finder/finder_test.go | 7 ++- ui/build/Android.bp | 2 + ui/build/build.go | 1 + ui/build/config.go | 4 ++ ui/build/finder.go | 102 ++++++++++++++++++++++++++++++++++ 8 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 ui/build/finder.go diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index fb1c89013..b06f3c445 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -233,6 +233,9 @@ func main() { var wg sync.WaitGroup productConfigs := make(chan Product, len(products)) + finder := build.NewSourceFinder(buildCtx, config) + defer finder.Shutdown() + // Run the product config for every product in parallel for _, product := range products { wg.Add(1) @@ -274,6 +277,8 @@ func main() { Thread: trace.NewThread(product), }} + build.FindSources(productCtx, config, finder) + productConfig := build.NewConfig(productCtx) productConfig.Environment().Set("OUT_DIR", productOutDir) productConfig.Lunch(productCtx, product, *buildVariant) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 94d6d5c4b..8a26171e6 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -95,5 +95,9 @@ func main() { } } + f := build.NewSourceFinder(buildCtx, config) + defer f.Shutdown() + build.FindSources(buildCtx, config, f) + build.Build(buildCtx, config, build.BuildAll) } diff --git a/finder/finder.go b/finder/finder.go index f15c8c13a..8f9496d38 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -148,10 +148,11 @@ type Finder struct { filesystem fs.FileSystem // temporary state - threadPool *threadPool - mutex sync.Mutex - fsErrs []fsErr - errlock sync.Mutex + threadPool *threadPool + mutex sync.Mutex + fsErrs []fsErr + errlock sync.Mutex + shutdownWaitgroup sync.WaitGroup // non-temporary state modifiedFlag int32 @@ -183,6 +184,8 @@ func New(cacheParams CacheParams, filesystem fs.FileSystem, nodes: *newPathMap("/"), DbPath: dbPath, + + shutdownWaitgroup: sync.WaitGroup{}, } f.loadFromFilesystem() @@ -195,9 +198,12 @@ func New(cacheParams CacheParams, filesystem fs.FileSystem, // confirm that every path mentioned in the CacheConfig exists for _, path := range cacheParams.RootDirs { + if !filepath.IsAbs(path) { + path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path) + } node := f.nodes.GetNode(filepath.Clean(path), false) if node == nil || node.ModTime == 0 { - return nil, fmt.Errorf("%v does not exist\n", path) + return nil, fmt.Errorf("path %v was specified to be included in the cache but does not exist\n", path) } } @@ -310,20 +316,32 @@ func (f *Finder) FindMatching(rootPath string, filter WalkFunc) []string { return results } -// Shutdown saves the contents of the Finder to its database file +// Shutdown declares that the finder is no longer needed and waits for its cleanup to complete +// Currently, that only entails waiting for the database dump to complete. func (f *Finder) Shutdown() { - f.verbosef("Shutting down\n") + f.waitForDbDump() +} + +// End of public api + +func (f *Finder) goDumpDb() { if f.wasModified() { - err := f.dumpDb() - if err != nil { - f.verbosef("%v\n", err) - } + f.shutdownWaitgroup.Add(1) + go func() { + err := f.dumpDb() + if err != nil { + f.verbosef("%v\n", err) + } + f.shutdownWaitgroup.Done() + }() } else { f.verbosef("Skipping dumping unmodified db\n") } } -// End of public api +func (f *Finder) waitForDbDump() { + f.shutdownWaitgroup.Wait() +} // joinCleanPaths is like filepath.Join but is faster because // joinCleanPaths doesn't have to support paths ending in "/" or containing ".." @@ -353,6 +371,8 @@ func (f *Finder) loadFromFilesystem() { f.startWithoutExternalCache() } + f.goDumpDb() + f.threadPool = nil } diff --git a/finder/finder_test.go b/finder/finder_test.go index 15c3728b1..8d1bbd7fb 100644 --- a/finder/finder_test.go +++ b/finder/finder_test.go @@ -466,12 +466,13 @@ func TestRelativeFilePaths(t *testing.T) { create(t, "/cwd/hi.txt", filesystem) create(t, "/cwd/a/hi.txt", filesystem) create(t, "/cwd/a/a/hi.txt", filesystem) + create(t, "/rel/a/hi.txt", filesystem) finder := newFinder( t, filesystem, CacheParams{ - RootDirs: []string{"/cwd", "/tmp/include"}, + RootDirs: []string{"/cwd", "../rel", "/tmp/include"}, IncludeFiles: []string{"hi.txt"}, }, ) @@ -491,6 +492,10 @@ func TestRelativeFilePaths(t *testing.T) { "a/hi.txt", "a/a/hi.txt"}) + foundPaths = finder.FindNamedAt("/rel", "hi.txt") + assertSameResponse(t, foundPaths, + []string{"/rel/a/hi.txt"}) + foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt") assertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"}) } diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 548baeed4..7640e84dc 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -19,6 +19,7 @@ bootstrap_go_package { "soong-ui-logger", "soong-ui-tracer", "soong-shared", + "soong-finder", ], srcs: [ "build.go", @@ -27,6 +28,7 @@ bootstrap_go_package { "context.go", "environment.go", "exec.go", + "finder.go", "kati.go", "make.go", "ninja.go", diff --git a/ui/build/build.go b/ui/build/build.go index 9650eaa20..076e15e46 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -32,6 +32,7 @@ func SetupOutDir(ctx Context, config Config) { // The ninja_build file is used by our buildbots to understand that the output // can be parsed as ninja output. ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build")) + ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), ".out-dir")) } var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(` diff --git a/ui/build/config.go b/ui/build/config.go index 045f6748c..1c2f73be0 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -280,6 +280,10 @@ func (c *configImpl) TempDir() string { return shared.TempDirForOutDir(c.SoongOutDir()) } +func (c *configImpl) FileListDir() string { + return filepath.Join(c.OutDir(), ".module_paths") +} + func (c *configImpl) KatiSuffix() string { if c.katiSuffix != "" { return c.katiSuffix diff --git a/ui/build/finder.go b/ui/build/finder.go new file mode 100644 index 000000000..05dec3aee --- /dev/null +++ b/ui/build/finder.go @@ -0,0 +1,102 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "android/soong/finder" + "android/soong/fs" + "android/soong/ui/logger" + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +// This file provides an interface to the Finder for use in Soong UI +// This file stores configuration information about which files to find + +// NewSourceFinder returns a new Finder configured to search for source files. +// Callers of NewSourceFinder should call when done +func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { + ctx.BeginTrace("find modules") + defer ctx.EndTrace() + + dir, err := os.Getwd() + if err != nil { + ctx.Fatalf("No working directory for module-finder: %v", err.Error()) + } + cacheParams := finder.CacheParams{ + WorkingDirectory: dir, + RootDirs: []string{"."}, + ExcludeDirs: []string{".git", ".repo"}, + PruneFiles: []string{".out-dir", ".find-ignore"}, + IncludeFiles: []string{"Android.mk", "Android.bp", "Blueprints", "CleanSpec.mk"}, + } + dumpDir := config.FileListDir() + f, err = finder.New(cacheParams, fs.OsFs, logger.New(ioutil.Discard), + filepath.Join(dumpDir, "files.db")) + if err != nil { + ctx.Fatalf("Could not create module-finder: %v", err) + } + return f +} + +// FindSources searches for source files known to and writes them to the filesystem for +// use later. +func FindSources(ctx Context, config Config, f *finder.Finder) { + // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder + // if a caller such as multiproduct_kati wants to share one Finder among several builds + dumpDir := config.FileListDir() + os.MkdirAll(dumpDir, 0777) + + androidMks := f.FindFirstNamedAt(".", "Android.mk") + err := dumpListToFile(androidMks, filepath.Join(dumpDir, "Android.mk.list")) + if err != nil { + ctx.Fatalf("Could not export module list: %v", err) + } + + cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") + dumpListToFile(cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) + if err != nil { + ctx.Fatalf("Could not export module list: %v", err) + } + + isBlueprintFile := func(dir finder.DirEntries) (dirs []string, files []string) { + files = []string{} + for _, file := range dir.FileNames { + if file == "Android.bp" || file == "Blueprints" { + files = append(files, file) + } + } + + return dir.DirNames, files + } + androidBps := f.FindMatching(".", isBlueprintFile) + err = dumpListToFile(androidBps, filepath.Join(dumpDir, "Android.bp.list")) + if err != nil { + ctx.Fatalf("Could not find modules: %v", err) + } +} + +func dumpListToFile(list []string, filePath string) (err error) { + desiredText := strings.Join(list, "\n") + desiredBytes := []byte(desiredText) + actualBytes, readErr := ioutil.ReadFile(filePath) + if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) { + err = ioutil.WriteFile(filePath, desiredBytes, 0777) + } + return err +}