StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Write raw files to disk instead of the ninja file" into main

Browse Source
This commit is contained in:
Colin Cross
2024-01-02 19:44:52 +00:00
committed by Gerrit Code Review
parent 64c9e0ce6e 31a674571e
commit a3759f98a6
12 changed files with 340 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
android/Android.bp
View File

@@ -81,6 +81,7 @@ bootstrap_go_package {
"prebuilt_build_tool.go",
"proto.go",
"provider.go",
"raw_files.go",
"register.go",
"rule_builder.go",
"sandbox.go",

6
android/config.go
View File

@@ -18,6 +18,7 @@ package android
// product variables necessary for soong_build's operation.
import (
"android/soong/shared"
"encoding/json"
"fmt"
"os"
@@ -118,6 +119,11 @@ func (c Config) SoongOutDir() string {
return c.soongOutDir
}
// tempDir returns the path to out/soong/.temp, which is cleared at the beginning of every build.
func (c Config) tempDir() string {
return shared.TempDirForOutDir(c.soongOutDir)
}
func (c Config) OutDir() string {
return c.outDir
}

108
android/defs.go
View File

@@ -15,13 +15,8 @@
package android
import (
"fmt"
"strings"
"testing"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools"
)
var (
@@ -72,8 +67,7 @@ var (
Command: "if ! cmp -s $in $out; then cp $in $out; fi",
Description: "cp if changed $out",
Restat: true,
},
"cpFlags")
})
CpExecutable = pctx.AndroidStaticRule("CpExecutable",
blueprint.RuleParams{
@@ -146,106 +140,6 @@ func BazelCcToolchainVars(config Config) string {
return BazelToolchainVars(config, exportedVars)
}
var (
// echoEscaper escapes a string such that passing it to "echo -e" will produce the input value.
echoEscaper = strings.NewReplacer(
`\`, `\\`, // First escape existing backslashes so they aren't interpreted by `echo -e`.
"\n", `\n`, // Then replace newlines with \n
)
// echoEscaper reverses echoEscaper.
echoUnescaper = strings.NewReplacer(
`\n`, "\n",
`\\`, `\`,
)
// shellUnescaper reverses the replacer in proptools.ShellEscape
shellUnescaper = strings.NewReplacer(`'\''`, `'`)
)
func buildWriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
content = echoEscaper.Replace(content)
content = proptools.NinjaEscape(proptools.ShellEscapeIncludingSpaces(content))
if content == "" {
content = "''"
}
ctx.Build(pctx, BuildParams{
Rule: writeFile,
Output: outputFile,
Description: "write " + outputFile.Base(),
Args: map[string]string{
"content": content,
},
})
}
// WriteFileRule creates a ninja rule to write contents to a file. The contents will be escaped
// so that the file contains exactly the contents passed to the function, plus a trailing newline.
func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
WriteFileRuleVerbatim(ctx, outputFile, content+"\n")
}
// WriteFileRuleVerbatim creates a ninja rule to write contents to a file. The contents will be
// escaped so that the file contains exactly the contents passed to the function.
func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) {
// This is MAX_ARG_STRLEN subtracted with some safety to account for shell escapes
const SHARD_SIZE = 131072 - 10000
if len(content) > SHARD_SIZE {
var chunks WritablePaths
for i, c := range ShardString(content, SHARD_SIZE) {
tempPath := outputFile.ReplaceExtension(ctx, fmt.Sprintf("%s.%d", outputFile.Ext(), i))
buildWriteFileRule(ctx, tempPath, c)
chunks = append(chunks, tempPath)
}
ctx.Build(pctx, BuildParams{
Rule: Cat,
Inputs: chunks.Paths(),
Output: outputFile,
Description: "Merging to " + outputFile.Base(),
})
return
}
buildWriteFileRule(ctx, outputFile, content)
}
// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result
func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) {
intermediate := PathForIntermediates(ctx, "write_executable_file_intermediates").Join(ctx, outputFile.String())
WriteFileRuleVerbatim(ctx, intermediate, content)
ctx.Build(pctx, BuildParams{
Rule: CpExecutable,
Output: outputFile,
Input: intermediate,
})
}
// shellUnescape reverses proptools.ShellEscape
func shellUnescape(s string) string {
// Remove leading and trailing quotes if present
if len(s) >= 2 && s[0] == '\'' {
s = s[1 : len(s)-1]
}
s = shellUnescaper.Replace(s)
return s
}
// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use
// in tests.
func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string {
t.Helper()
if g, w := params.Rule, writeFile; g != w {
t.Errorf("expected params.Rule to be %q, was %q", w, g)
return ""
}
content := params.Args["content"]
content = shellUnescape(content)
content = echoUnescaper.Replace(content)
return content
}
// GlobToListFileRule creates a rule that writes a list of files matching a pattern to a file.
func GlobToListFileRule(ctx ModuleContext, pattern string, excludes []string, file WritablePath) {
bootstrap.GlobFile(ctx.blueprintModuleContext(), pattern, excludes, file.String())

279
android/raw_files.go Normal file
View File

@@ -0,0 +1,279 @@
// Copyright 2023 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"crypto/sha1"
"encoding/hex"
"fmt"
"github.com/google/blueprint"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"github.com/google/blueprint/proptools"
)
// WriteFileRule creates a ninja rule to write contents to a file by immediately writing the
// contents, plus a trailing newline, to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating
// a ninja rule to copy the file into place.
func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
writeFileRule(ctx, outputFile, content, true, false)
}
// WriteFileRuleVerbatim creates a ninja rule to write contents to a file by immediately writing the
// contents to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating a ninja rule to copy the file into place.
func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) {
writeFileRule(ctx, outputFile, content, false, false)
}
// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result
func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) {
writeFileRule(ctx, outputFile, content, false, true)
}
// tempFile provides a testable wrapper around a file in out/soong/.temp. It writes to a temporary file when
// not in tests, but writes to a buffer in memory when used in tests.
type tempFile struct {
// tempFile contains wraps an io.Writer, which will be file if testMode is false, or testBuf if it is true.
io.Writer
file *os.File
testBuf *strings.Builder
}
func newTempFile(ctx BuilderContext, pattern string, testMode bool) *tempFile {
if testMode {
testBuf := &strings.Builder{}
return &tempFile{
Writer: testBuf,
testBuf: testBuf,
}
} else {
f, err := os.CreateTemp(absolutePath(ctx.Config().tempDir()), pattern)
if err != nil {
panic(fmt.Errorf("failed to open temporary raw file: %w", err))
}
return &tempFile{
Writer: f,
file: f,
}
}
}
func (t *tempFile) close() error {
if t.file != nil {
return t.file.Close()
}
return nil
}
func (t *tempFile) name() string {
if t.file != nil {
return t.file.Name()
}
return "temp_file_in_test"
}
func (t *tempFile) rename(to string) {
if t.file != nil {
os.MkdirAll(filepath.Dir(to), 0777)
err := os.Rename(t.file.Name(), to)
if err != nil {
panic(fmt.Errorf("failed to rename %s to %s: %w", t.file.Name(), to, err))
}
}
}
func (t *tempFile) remove() error {
if t.file != nil {
return os.Remove(t.file.Name())
}
return nil
}
func writeContentToTempFileAndHash(ctx BuilderContext, content string, newline bool) (*tempFile, string) {
tempFile := newTempFile(ctx, "raw", ctx.Config().captureBuild)
defer tempFile.close()
hash := sha1.New()
w := io.MultiWriter(tempFile, hash)
_, err := io.WriteString(w, content)
if err == nil && newline {
_, err = io.WriteString(w, "\n")
}
if err != nil {
panic(fmt.Errorf("failed to write to temporary raw file %s: %w", tempFile.name(), err))
}
return tempFile, hex.EncodeToString(hash.Sum(nil))
}
func writeFileRule(ctx BuilderContext, outputFile WritablePath, content string, newline bool, executable bool) {
// Write the contents to a temporary file while computing its hash.
tempFile, hash := writeContentToTempFileAndHash(ctx, content, newline)
// Shard the final location of the raw file into a subdirectory based on the first two characters of the
// hash to avoid making the raw directory too large and slowing down accesses.
relPath := filepath.Join(hash[0:2], hash)
// These files are written during soong_build. If something outside the build deleted them there would be no
// trigger to rerun soong_build, and the build would break with dependencies on missing files. Writing them
// to their final locations would risk having them deleted when cleaning a module, and would also pollute the
// output directory with files for modules that have never been built.
// Instead, the files are written to a separate "raw" directory next to the build.ninja file, and a ninja
// rule is created to copy the files into their final location as needed.
// Obsolete files written by previous runs of soong_build must be cleaned up to avoid continually growing
// disk usage as the hashes of the files change over time. The cleanup must not remove files that were
// created by previous runs of soong_build for other products, as the build.ninja files for those products
// may still exist and still reference those files. The raw files from different products are kept
// separate by appending the Make_suffix to the directory name.
rawPath := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix), relPath)
rawFileInfo := rawFileInfo{
relPath: relPath,
}
if ctx.Config().captureBuild {
// When running tests tempFile won't write to disk, instead store the contents for later retrieval by
// ContentFromFileRuleForTests.
rawFileInfo.contentForTests = tempFile.testBuf.String()
}
rawFileSet := getRawFileSet(ctx.Config())
if _, exists := rawFileSet.LoadOrStore(hash, rawFileInfo); exists {
// If a raw file with this hash has already been created delete the temporary file.
tempFile.remove()
} else {
// If this is the first time this hash has been seen then move it from the temporary directory
// to the raw directory. If the file already exists in the raw directory assume it has the correct
// contents.
absRawPath := absolutePath(rawPath.String())
_, err := os.Stat(absRawPath)
if os.IsNotExist(err) {
tempFile.rename(absRawPath)
} else if err != nil {
panic(fmt.Errorf("failed to stat %q: %w", absRawPath, err))
} else {
tempFile.remove()
}
}
// Emit a rule to copy the file from raw directory to the final requested location in the output tree.
// Restat is used to ensure that two different products that produce identical files copied from their
// own raw directories they don't cause everything downstream to rebuild.
rule := rawFileCopy
if executable {
rule = rawFileCopyExecutable
}
ctx.Build(pctx, BuildParams{
Rule: rule,
Input: rawPath,
Output: outputFile,
Description: "raw " + outputFile.Base(),
})
}
var (
rawFileCopy = pctx.AndroidStaticRule("rawFileCopy",
blueprint.RuleParams{
Command: "if ! cmp -s $in $out; then cp $in $out; fi",
Description: "copy raw file $out",
Restat: true,
})
rawFileCopyExecutable = pctx.AndroidStaticRule("rawFileCopyExecutable",
blueprint.RuleParams{
Command: "if ! cmp -s $in $out; then cp $in $out; fi && chmod +x $out",
Description: "copy raw exectuable file $out",
Restat: true,
})
)
type rawFileInfo struct {
relPath string
contentForTests string
}
var rawFileSetKey OnceKey = NewOnceKey("raw file set")
func getRawFileSet(config Config) *SyncMap[string, rawFileInfo] {
return config.Once(rawFileSetKey, func() any {
return &SyncMap[string, rawFileInfo]{}
}).(*SyncMap[string, rawFileInfo])
}
// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use
// in tests.
func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string {
t.Helper()
if params.Rule != rawFileCopy && params.Rule != rawFileCopyExecutable {
t.Errorf("expected params.Rule to be rawFileCopy or rawFileCopyExecutable, was %q", params.Rule)
return ""
}
key := filepath.Base(params.Input.String())
rawFileSet := getRawFileSet(ctx.Config())
rawFileInfo, _ := rawFileSet.Load(key)
return rawFileInfo.contentForTests
}
func rawFilesSingletonFactory() Singleton {
return &rawFilesSingleton{}
}
type rawFilesSingleton struct{}
func (rawFilesSingleton) GenerateBuildActions(ctx SingletonContext) {
if ctx.Config().captureBuild {
// Nothing to do when running in tests, no temporary files were created.
return
}
rawFileSet := getRawFileSet(ctx.Config())
rawFilesDir := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix)).String()
absRawFilesDir := absolutePath(rawFilesDir)
err := filepath.WalkDir(absRawFilesDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
// Ignore obsolete directories for now.
return nil
}
// Assume the basename of the file is a hash
key := filepath.Base(path)
relPath, err := filepath.Rel(absRawFilesDir, path)
if err != nil {
return err
}
// Check if a file with the same hash was written by this run of soong_build. If the file was not written,
// or if a file with the same hash was written but to a different path in the raw directory, then delete it.
// Checking that the path matches allows changing the structure of the raw directory, for example to increase
// the sharding.
rawFileInfo, written := rawFileSet.Load(key)
if !written || rawFileInfo.relPath != relPath {
os.Remove(path)
}
return nil
})
if err != nil {
panic(fmt.Errorf("failed to clean %q: %w", rawFilesDir, err))
}
}

3
android/register.go
View File

@@ -191,8 +191,9 @@ func collateGloballyRegisteredSingletons() sortableComponents {
// Register makevars after other singletons so they can export values through makevars
singleton{parallel: false, name: "makevars", factory: makeVarsSingletonFunc},
// Register env and ninjadeps last so that they can track all used environment variables and
// Register rawfiles and ninjadeps last so that they can track all used environment variables and
// Ninja file dependencies stored in the config.
singleton{parallel: false, name: "rawfiles", factory: rawFilesSingletonFactory},
singleton{parallel: false, name: "ninjadeps", factory: ninjaDepsSingletonFactory},
)

13
android/rule_builder_test.go
View File

@@ -816,13 +816,13 @@ func TestRuleBuilderHashInputs(t *testing.T) {
func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) {
bp := `
rule_builder_test {
name: "foo_sbox_escaped_ninja",
name: "foo_sbox_escaped",
flags: ["${cmdFlags}"],
sbox: true,
sbox_inputs: true,
}
rule_builder_test {
name: "foo_sbox",
name: "foo_sbox_unescaped",
flags: ["${cmdFlags}"],
sbox: true,
sbox_inputs: true,
@@ -834,15 +834,16 @@ func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) {
FixtureWithRootAndroidBp(bp),
).RunTest(t)
escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped_ninja", "").Rule("writeFile")
escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped", "").Output("sbox.textproto")
AssertStringEquals(t, "expected rule", "android/soong/android.rawFileCopy", escapedNinjaMod.Rule.String())
AssertStringDoesContain(
t,
"",
escapedNinjaMod.BuildParams.Args["content"],
"$${cmdFlags}",
ContentFromFileRuleForTests(t, result.TestContext, escapedNinjaMod),
"${cmdFlags}",
)
unescapedNinjaMod := result.ModuleForTests("foo_sbox", "").Rule("unescapedWriteFile")
unescapedNinjaMod := result.ModuleForTests("foo_sbox_unescaped", "").Rule("unescapedWriteFile")
AssertStringDoesContain(
t,
"",

30
android/util.go
View File

@@ -22,6 +22,7 @@ import (
"runtime"
"sort"
"strings"
"sync"
)
// CopyOf returns a new slice that has the same contents as s.
@@ -597,3 +598,32 @@ func AddToStringSet(set map[string]bool, items []string) {
set[item] = true
}
}
// SyncMap is a wrapper around sync.Map that provides type safety via generics.
type SyncMap[K comparable, V any] struct {
sync.Map
}
// Load returns the value stored in the map for a key, or the zero value if no
// value is present.
// The ok result indicates whether value was found in the map.
func (m *SyncMap[K, V]) Load(key K) (value V, ok bool) {
v, ok := m.Map.Load(key)
if !ok {
return *new(V), false
}
return v.(V), true
}
// Store sets the value for a key.
func (m *SyncMap[K, V]) Store(key K, value V) {
m.Map.Store(key, value)
}
// LoadOrStore returns the existing value for the key if present.
// Otherwise, it stores and returns the given value.
// The loaded result is true if the value was loaded, false if stored.
func (m *SyncMap[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) {
v, loaded := m.Map.LoadOrStore(key, value)
return v.(V), loaded
}

19
java/code_metadata_test.go
View File

@@ -25,12 +25,10 @@ func TestCodeMetadata(t *testing.T) {
}`
result := runCodeMetadataTest(t, android.FixtureExpectsNoErrors, bp)
module := result.ModuleForTests(
"module-name", "",
).Module().(*soongTesting.CodeMetadataModule)
module := result.ModuleForTests("module-name", "")
// Check that the provider has the right contents
data, _ := android.SingletonModuleProvider(result, module, soongTesting.CodeMetadataProviderKey)
data, _ := android.SingletonModuleProvider(result, module.Module(), soongTesting.CodeMetadataProviderKey)
if !strings.HasSuffix(
data.IntermediatePath.String(), "/intermediateCodeMetadata.pb",
) {
@@ -40,13 +38,8 @@ func TestCodeMetadata(t *testing.T) {
)
}
buildParamsSlice := module.BuildParamsForTests()
var metadata = ""
for _, params := range buildParamsSlice {
if params.Rule.String() == "android/soong/android.writeFile" {
metadata = params.Args["content"]
}
}
metadata := android.ContentFromFileRuleForTests(t, result.TestContext,
module.Output(data.IntermediatePath.String()))
metadataList := make([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership, 0, 2)
teamId := "12345"
@@ -63,9 +56,7 @@ func TestCodeMetadata(t *testing.T) {
CodeMetadataMetadata := code_metadata_internal_proto.CodeMetadataInternal{TargetOwnershipList: metadataList}
protoData, _ := proto.Marshal(&CodeMetadataMetadata)
rawData := string(protoData)
formattedData := strings.ReplaceAll(rawData, "\n", "\\n")
expectedMetadata := "'" + formattedData + "\\n'"
expectedMetadata := string(protoData)
if metadata != expectedMetadata {
t.Errorf(

19
java/test_spec_test.go
View File

@@ -29,12 +29,10 @@ func TestTestSpec(t *testing.T) {
}`
result := runTestSpecTest(t, android.FixtureExpectsNoErrors, bp)
module := result.ModuleForTests(
"module-name", "",
).Module().(*soongTesting.TestSpecModule)
module := result.ModuleForTests("module-name", "")
// Check that the provider has the right contents
data, _ := android.SingletonModuleProvider(result, module, soongTesting.TestSpecProviderKey)
data, _ := android.SingletonModuleProvider(result, module.Module(), soongTesting.TestSpecProviderKey)
if !strings.HasSuffix(
data.IntermediatePath.String(), "/intermediateTestSpecMetadata.pb",
) {
@@ -44,13 +42,8 @@ func TestTestSpec(t *testing.T) {
)
}
buildParamsSlice := module.BuildParamsForTests()
var metadata = ""
for _, params := range buildParamsSlice {
if params.Rule.String() == "android/soong/android.writeFile" {
metadata = params.Args["content"]
}
}
metadata := android.ContentFromFileRuleForTests(t, result.TestContext,
module.Output(data.IntermediatePath.String()))
metadataList := make([]*test_spec_proto.TestSpec_OwnershipMetadata, 0, 2)
teamId := "12345"
@@ -70,9 +63,7 @@ func TestTestSpec(t *testing.T) {
}
testSpecMetadata := test_spec_proto.TestSpec{OwnershipMetadataList: metadataList}
protoData, _ := proto.Marshal(&testSpecMetadata)
rawData := string(protoData)
formattedData := strings.ReplaceAll(rawData, "\n", "\\n")
expectedMetadata := "'" + formattedData + "\\n'"
expectedMetadata := string(protoData)
if metadata != expectedMetadata {
t.Errorf(

2
testing/code_metadata.go
View File

@@ -128,7 +128,7 @@ func (module *CodeMetadataModule) GenerateAndroidBuildActions(ctx android.Module
intermediatePath := android.PathForModuleOut(
ctx, "intermediateCodeMetadata.pb",
)
android.WriteFileRule(ctx, intermediatePath, string(protoData))
android.WriteFileRuleVerbatim(ctx, intermediatePath, string(protoData))
android.SetProvider(ctx,
CodeMetadataProviderKey,

2
testing/test_spec.go
View File

@@ -117,7 +117,7 @@ func (module *TestSpecModule) GenerateAndroidBuildActions(ctx android.ModuleCont
if err != nil {
ctx.ModuleErrorf("Error: %s", err.Error())
}
android.WriteFileRule(ctx, intermediatePath, string(protoData))
android.WriteFileRuleVerbatim(ctx, intermediatePath, string(protoData))
android.SetProvider(ctx,
TestSpecProviderKey, TestSpecProviderData{

2
ui/build/test_build.go
View File

@@ -63,6 +63,7 @@ func testForDanglingRules(ctx Context, config Config) {
outDir := config.OutDir()
modulePathsDir := filepath.Join(outDir, ".module_paths")
rawFilesDir := filepath.Join(outDir, "soong", "raw")
variablesFilePath := filepath.Join(outDir, "soong", "soong.variables")
// dexpreopt.config is an input to the soong_docs action, which runs the
@@ -88,6 +89,7 @@ func testForDanglingRules(ctx Context, config Config) {
continue
}
if strings.HasPrefix(line, modulePathsDir) ||
strings.HasPrefix(line, rawFilesDir) ||
line == variablesFilePath ||
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||