From 6b795dc6a5e6f479adae456b978ba1d2bc761dae Mon Sep 17 00:00:00 2001 From: Sasha Smundak Date: Wed, 18 Aug 2021 16:32:19 -0700 Subject: [PATCH] Add find_files builtin, use it to fix find_and_copy implementation The macro find-and-copy finds all the files in the given source tree that match the given filename patten and create : pair with the same relative path in the destination tree. Bug: 193540681 Test: rbcrun build/make/tests/run.rbc Change-Id: Ic4315ce2fab7a7791ab55dd9eed039205a1c721a --- core/product_config.rbc | 3 +- tests/device.rbc | 2 +- tests/run.rbc | 2 +- tools/rbcrun/README.md | 5 ++++ tools/rbcrun/host.go | 43 +++++++++++++++++++++++++++++ tools/rbcrun/testdata/file_ops.star | 12 ++++++-- 6 files changed, 61 insertions(+), 6 deletions(-) diff --git a/core/product_config.rbc b/core/product_config.rbc index f7ce7aa21b..b6a79cbb1e 100644 --- a/core/product_config.rbc +++ b/core/product_config.rbc @@ -396,7 +396,8 @@ def _file_wildcard_exists(file_pattern): def _find_and_copy(pattern, from_dir, to_dir): """Return a copy list for the files matching the pattern.""" - return ["%s/%s:%s/%s" % (from_dir, f, to_dir, f) for f in rblf_wildcard(pattern, from_dir)] + return ["%s/%s:%s/%s" % ( + from_dir, f, to_dir, f) for f in rblf_find_files(from_dir, pattern, only_files=1)] def _filter_out(pattern, text): """Return all the words from `text' that do not match any word in `pattern'. diff --git a/tests/device.rbc b/tests/device.rbc index b57dbf9311..c91a48773a 100644 --- a/tests/device.rbc +++ b/tests/device.rbc @@ -45,7 +45,7 @@ def init(g, handle): cfg["PRODUCT_COPY_FILES"] += ["device_from:device_to"] _include1_init(g, handle) cfg["PRODUCT_PACKAGES"] += ["dev_after"] - cfg["PRODUCT_COPY_FILES"] += (rblf.find_and_copy("audio_platform_info*.xml", "device/google/redfin/audio", "||VENDOR-PATH-PH||/etc") + + cfg["PRODUCT_COPY_FILES"] += (rblf.find_and_copy("audio_platform_info*.xml", "device/google/redfin", "||VENDOR-PATH-PH||/etc") + ["xyz:/etc/xyz"]) cfg["PRODUCT_COPY_FILES"] += rblf.copy_files("x.xml y.xml", "/etc") rblf.add_soong_config_namespace(g, "NS1") diff --git a/tests/run.rbc b/tests/run.rbc index 35ae19d518..e392e47ccc 100644 --- a/tests/run.rbc +++ b/tests/run.rbc @@ -53,7 +53,7 @@ assert_eq( "PRODUCT_COPY_FILES": [ "part_from:part_to", "device_from:device_to", - "device/google/redfin/audio/audio_platform_info_noextcodec_snd.xml:||VENDOR-PATH-PH||/etc/audio_platform_info_noextcodec_snd.xml", + "device/google/redfin/audio/audio_platform_info_noextcodec_snd.xml:||VENDOR-PATH-PH||/etc/audio/audio_platform_info_noextcodec_snd.xml", "xyz:/etc/xyz", "x.xml:/etc/x.xml", "y.xml:/etc/y.xml", diff --git a/tools/rbcrun/README.md b/tools/rbcrun/README.md index fb58c897aa..ecf8a24a0b 100644 --- a/tools/rbcrun/README.md +++ b/tools/rbcrun/README.md @@ -68,6 +68,11 @@ will have the value of `rblf_cli.FOO` be `"bar"` Returns `True` if *file* exists +#### rblf_find_files(*top*, *file-pattern*, only_files = 0) + +Returns all the paths under *top* whose basename matches *pattern* (which is a shell's glob pattern). If *only_files* is +not zero, only the paths to the regular files are returned. The returned paths are relative to *top*. + #### rblf_wildcard(*glob*, *top* = None) Expands *glob*. If *top* is supplied, expands "*top*/*glob*", then removes diff --git a/tools/rbcrun/host.go b/tools/rbcrun/host.go index b3dd4991f4..7f4f72dac6 100644 --- a/tools/rbcrun/host.go +++ b/tools/rbcrun/host.go @@ -16,6 +16,7 @@ package rbcrun import ( "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -170,6 +171,46 @@ func wildcard(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, return makeStringList(files), nil } +// find(top, pattern, only_files = 0) returns all the paths under 'top' +// whose basename matches 'pattern' (which is a shell's glob pattern). +// If 'only_files' is non-zero, only the paths to the regular files are +// returned. The returned paths are relative to 'top'. +func find(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, + kwargs []starlark.Tuple) (starlark.Value, error) { + var top, pattern string + var onlyFiles int + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "top", &top, "pattern", &pattern, "only_files?", &onlyFiles); err != nil { + return starlark.None, err + } + top = filepath.Clean(top) + pattern = filepath.Clean(pattern) + // Go's filepath.Walk is slow, consider using OS's find + var res []string + err := filepath.WalkDir(top, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if d != nil && d.IsDir() { + return fs.SkipDir + } else { + return nil + } + } + relPath := strings.TrimPrefix(path, top) + if len(relPath) > 0 && relPath[0] == os.PathSeparator { + relPath = relPath[1:] + } + // Do not return top-level dir + if len(relPath) == 0 { + return nil + } + if matched, err := filepath.Match(pattern, d.Name()); err == nil && matched && (onlyFiles == 0 || d.Type().IsRegular()) { + res = append(res, relPath) + } + return nil + }) + return makeStringList(res), err +} + // shell(command) runs OS shell with given command and returns back // its output the same way as Make's $(shell ) function. The end-of-lines // ("\n" or "\r\n") are replaced with " " in the result, and the trailing @@ -226,6 +267,8 @@ func setup(env []string) { "rblf_env": structFromEnv(os.Environ()), // To convert makefile's $(wildcard foo) "rblf_file_exists": starlark.NewBuiltin("rblf_file_exists", fileExists), + // To convert find-copy-subdir and product-copy-files-by pattern + "rblf_find_files": starlark.NewBuiltin("rblf_find_files", find), // To convert makefile's $(filter ...)/$(filter-out) "rblf_regex": starlark.NewBuiltin("rblf_regex", regexMatch), // To convert makefile's $(shell cmd) diff --git a/tools/rbcrun/testdata/file_ops.star b/tools/rbcrun/testdata/file_ops.star index 31631ef462..50e39bfa15 100644 --- a/tools/rbcrun/testdata/file_ops.star +++ b/tools/rbcrun/testdata/file_ops.star @@ -9,11 +9,17 @@ def test(): assert.true(not rblf_file_exists("no_such_file"), "the file no_such_file does not exist") files = rblf_wildcard("*.star") assert.true(myname in files, "expected %s in %s" % (myname, files)) - # RBCDATADIR is set by the caller to the path where this file resides files = rblf_wildcard("*.star", rblf_env.TEST_DATA_DIR) assert.true(myname in files, "expected %s in %s" % (myname, files)) files = rblf_wildcard("*.xxx") assert.true(len(files) == 0, "expansion should be empty but contains %s" % files) - - + mydir = "testdata" + myrelname = "%s/%s" % (mydir, myname) + files = rblf_find_files(rblf_env.TEST_DATA_DIR + "/../", "*") + assert.true(mydir in files and myrelname in files, "expected %s and %s in %s" % (mydir, myrelname, files)) + files = rblf_find_files(rblf_env.TEST_DATA_DIR + "/../", "*", only_files=1) + assert.true(mydir not in files, "did not expect %s in %s" % (mydir, files)) + assert.true(myrelname in files, "expected %s in %s" % (myrelname, files)) + files = rblf_find_files(rblf_env.TEST_DATA_DIR + "/../", "*.star") + assert.true(myrelname in files, "expected %s in %s" % (myrelname, files)) test()