Run the Finder and make its results available to Kati
The Finder runs roughly 200ms faster than findleaves.py in aosp,
and runs roughly 400ms faster in internal master.
Bug: 64363847
Test: m -j
Change-Id: I62db8dacc90871e913576fe2443021fb1749a483
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go
index fb1c890..b06f3c4 100644
--- a/cmd/multiproduct_kati/main.go
+++ b/cmd/multiproduct_kati/main.go
@@ -233,6 +233,9 @@
var wg sync.WaitGroup
productConfigs := make(chan Product, len(products))
+ finder := build.NewSourceFinder(buildCtx, config)
+ defer finder.Shutdown()
+
// Run the product config for every product in parallel
for _, product := range products {
wg.Add(1)
@@ -274,6 +277,8 @@
Thread: trace.NewThread(product),
}}
+ build.FindSources(productCtx, config, finder)
+
productConfig := build.NewConfig(productCtx)
productConfig.Environment().Set("OUT_DIR", productOutDir)
productConfig.Lunch(productCtx, product, *buildVariant)
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 94d6d5c..8a26171 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -95,5 +95,9 @@
}
}
+ f := build.NewSourceFinder(buildCtx, config)
+ defer f.Shutdown()
+ build.FindSources(buildCtx, config, f)
+
build.Build(buildCtx, config, build.BuildAll)
}
diff --git a/finder/finder.go b/finder/finder.go
index f15c8c1..8f9496d 100644
--- a/finder/finder.go
+++ b/finder/finder.go
@@ -148,10 +148,11 @@
filesystem fs.FileSystem
// temporary state
- threadPool *threadPool
- mutex sync.Mutex
- fsErrs []fsErr
- errlock sync.Mutex
+ threadPool *threadPool
+ mutex sync.Mutex
+ fsErrs []fsErr
+ errlock sync.Mutex
+ shutdownWaitgroup sync.WaitGroup
// non-temporary state
modifiedFlag int32
@@ -183,6 +184,8 @@
nodes: *newPathMap("/"),
DbPath: dbPath,
+
+ shutdownWaitgroup: sync.WaitGroup{},
}
f.loadFromFilesystem()
@@ -195,9 +198,12 @@
// confirm that every path mentioned in the CacheConfig exists
for _, path := range cacheParams.RootDirs {
+ if !filepath.IsAbs(path) {
+ path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
+ }
node := f.nodes.GetNode(filepath.Clean(path), false)
if node == nil || node.ModTime == 0 {
- return nil, fmt.Errorf("%v does not exist\n", path)
+ return nil, fmt.Errorf("path %v was specified to be included in the cache but does not exist\n", path)
}
}
@@ -310,20 +316,32 @@
return results
}
-// Shutdown saves the contents of the Finder to its database file
+// Shutdown declares that the finder is no longer needed and waits for its cleanup to complete
+// Currently, that only entails waiting for the database dump to complete.
func (f *Finder) Shutdown() {
- f.verbosef("Shutting down\n")
+ f.waitForDbDump()
+}
+
+// End of public api
+
+func (f *Finder) goDumpDb() {
if f.wasModified() {
- err := f.dumpDb()
- if err != nil {
- f.verbosef("%v\n", err)
- }
+ f.shutdownWaitgroup.Add(1)
+ go func() {
+ err := f.dumpDb()
+ if err != nil {
+ f.verbosef("%v\n", err)
+ }
+ f.shutdownWaitgroup.Done()
+ }()
} else {
f.verbosef("Skipping dumping unmodified db\n")
}
}
-// End of public api
+func (f *Finder) waitForDbDump() {
+ f.shutdownWaitgroup.Wait()
+}
// joinCleanPaths is like filepath.Join but is faster because
// joinCleanPaths doesn't have to support paths ending in "/" or containing ".."
@@ -353,6 +371,8 @@
f.startWithoutExternalCache()
}
+ f.goDumpDb()
+
f.threadPool = nil
}
diff --git a/finder/finder_test.go b/finder/finder_test.go
index 15c3728..8d1bbd7 100644
--- a/finder/finder_test.go
+++ b/finder/finder_test.go
@@ -466,12 +466,13 @@
create(t, "/cwd/hi.txt", filesystem)
create(t, "/cwd/a/hi.txt", filesystem)
create(t, "/cwd/a/a/hi.txt", filesystem)
+ create(t, "/rel/a/hi.txt", filesystem)
finder := newFinder(
t,
filesystem,
CacheParams{
- RootDirs: []string{"/cwd", "/tmp/include"},
+ RootDirs: []string{"/cwd", "../rel", "/tmp/include"},
IncludeFiles: []string{"hi.txt"},
},
)
@@ -491,6 +492,10 @@
"a/hi.txt",
"a/a/hi.txt"})
+ foundPaths = finder.FindNamedAt("/rel", "hi.txt")
+ assertSameResponse(t, foundPaths,
+ []string{"/rel/a/hi.txt"})
+
foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt")
assertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"})
}
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 548baee..7640e84 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -19,6 +19,7 @@
"soong-ui-logger",
"soong-ui-tracer",
"soong-shared",
+ "soong-finder",
],
srcs: [
"build.go",
@@ -27,6 +28,7 @@
"context.go",
"environment.go",
"exec.go",
+ "finder.go",
"kati.go",
"make.go",
"ninja.go",
diff --git a/ui/build/build.go b/ui/build/build.go
index 9650eaa..076e15e 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -32,6 +32,7 @@
// The ninja_build file is used by our buildbots to understand that the output
// can be parsed as ninja output.
ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build"))
+ ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), ".out-dir"))
}
var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(`
diff --git a/ui/build/config.go b/ui/build/config.go
index 045f674..1c2f73b 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -280,6 +280,10 @@
return shared.TempDirForOutDir(c.SoongOutDir())
}
+func (c *configImpl) FileListDir() string {
+ return filepath.Join(c.OutDir(), ".module_paths")
+}
+
func (c *configImpl) KatiSuffix() string {
if c.katiSuffix != "" {
return c.katiSuffix
diff --git a/ui/build/finder.go b/ui/build/finder.go
new file mode 100644
index 0000000..05dec3a
--- /dev/null
+++ b/ui/build/finder.go
@@ -0,0 +1,102 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "android/soong/finder"
+ "android/soong/fs"
+ "android/soong/ui/logger"
+ "bytes"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// This file provides an interface to the Finder for use in Soong UI
+// This file stores configuration information about which files to find
+
+// NewSourceFinder returns a new Finder configured to search for source files.
+// Callers of NewSourceFinder should call <f.Shutdown()> when done
+func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
+ ctx.BeginTrace("find modules")
+ defer ctx.EndTrace()
+
+ dir, err := os.Getwd()
+ if err != nil {
+ ctx.Fatalf("No working directory for module-finder: %v", err.Error())
+ }
+ cacheParams := finder.CacheParams{
+ WorkingDirectory: dir,
+ RootDirs: []string{"."},
+ ExcludeDirs: []string{".git", ".repo"},
+ PruneFiles: []string{".out-dir", ".find-ignore"},
+ IncludeFiles: []string{"Android.mk", "Android.bp", "Blueprints", "CleanSpec.mk"},
+ }
+ dumpDir := config.FileListDir()
+ f, err = finder.New(cacheParams, fs.OsFs, logger.New(ioutil.Discard),
+ filepath.Join(dumpDir, "files.db"))
+ if err != nil {
+ ctx.Fatalf("Could not create module-finder: %v", err)
+ }
+ return f
+}
+
+// FindSources searches for source files known to <f> and writes them to the filesystem for
+// use later.
+func FindSources(ctx Context, config Config, f *finder.Finder) {
+ // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder
+ // if a caller such as multiproduct_kati wants to share one Finder among several builds
+ dumpDir := config.FileListDir()
+ os.MkdirAll(dumpDir, 0777)
+
+ androidMks := f.FindFirstNamedAt(".", "Android.mk")
+ err := dumpListToFile(androidMks, filepath.Join(dumpDir, "Android.mk.list"))
+ if err != nil {
+ ctx.Fatalf("Could not export module list: %v", err)
+ }
+
+ cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
+ dumpListToFile(cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
+ if err != nil {
+ ctx.Fatalf("Could not export module list: %v", err)
+ }
+
+ isBlueprintFile := func(dir finder.DirEntries) (dirs []string, files []string) {
+ files = []string{}
+ for _, file := range dir.FileNames {
+ if file == "Android.bp" || file == "Blueprints" {
+ files = append(files, file)
+ }
+ }
+
+ return dir.DirNames, files
+ }
+ androidBps := f.FindMatching(".", isBlueprintFile)
+ err = dumpListToFile(androidBps, filepath.Join(dumpDir, "Android.bp.list"))
+ if err != nil {
+ ctx.Fatalf("Could not find modules: %v", err)
+ }
+}
+
+func dumpListToFile(list []string, filePath string) (err error) {
+ desiredText := strings.Join(list, "\n")
+ desiredBytes := []byte(desiredText)
+ actualBytes, readErr := ioutil.ReadFile(filePath)
+ if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
+ err = ioutil.WriteFile(filePath, desiredBytes, 0777)
+ }
+ return err
+}