From 3b1f83d076cbb741fd9f9b150c7478e52b6f1920 Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Thu, 14 Oct 2021 14:08:38 -0400 Subject: [PATCH] Add x86_host vars to config.bzl Test: USE_BAZEL_ANALYSIS=1 m adbd Test: Manually verified config.bzl contains various x86_host flags after bp2build. Test: Unit tests Change-Id: Ie9201ea2be4cd1c6659bea088a797cedbae37403 --- bp2build/bp2build.go | 7 +- bp2build/conversion.go | 4 +- bp2build/conversion_test.go | 4 +- cc/config/bp2build.go | 144 +++++++++++++++++++++++++++++------- cc/config/bp2build_test.go | 46 ++++++++++-- cc/config/global.go | 2 +- cc/config/x86_linux_host.go | 50 ++++++------- 7 files changed, 191 insertions(+), 66 deletions(-) diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index 45a3cb6fe..b0c3899ba 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -15,11 +15,12 @@ package bp2build import ( - "android/soong/android" - "android/soong/bazel" "fmt" "os" "strings" + + "android/soong/android" + "android/soong/bazel" ) // Codegen is the backend of bp2build. The code generator is responsible for @@ -43,7 +44,7 @@ func Codegen(ctx *CodegenContext) CodegenMetrics { writeFiles(ctx, bp2buildDir, bp2buildFiles) soongInjectionDir := android.PathForOutput(ctx, bazel.SoongInjectionDirName) - writeFiles(ctx, soongInjectionDir, CreateSoongInjectionFiles(res.metrics)) + writeFiles(ctx, soongInjectionDir, CreateSoongInjectionFiles(ctx.Config(), res.metrics)) return res.metrics } diff --git a/bp2build/conversion.go b/bp2build/conversion.go index d34a4ba9b..944cb83a3 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -17,11 +17,11 @@ type BazelFile struct { Contents string } -func CreateSoongInjectionFiles(metrics CodegenMetrics) []BazelFile { +func CreateSoongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []BazelFile { var files []BazelFile files = append(files, newFile("cc_toolchain", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. - files = append(files, newFile("cc_toolchain", "constants.bzl", config.BazelCcToolchainVars())) + files = append(files, newFile("cc_toolchain", "constants.bzl", config.BazelCcToolchainVars(cfg))) files = append(files, newFile("metrics", "converted_modules.txt", strings.Join(metrics.convertedModules, "\n"))) diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index dfa1a9eb4..d09238a2b 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -17,6 +17,8 @@ package bp2build import ( "sort" "testing" + + "android/soong/android" ) type bazelFilepath struct { @@ -80,7 +82,7 @@ func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) { } func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { - files := CreateSoongInjectionFiles(CodegenMetrics{}) + files := CreateSoongInjectionFiles(android.Config{}, CodegenMetrics{}) expectedFilePaths := []bazelFilepath{ { diff --git a/cc/config/bp2build.go b/cc/config/bp2build.go index d19f5ac8e..4797acca1 100644 --- a/cc/config/bp2build.go +++ b/cc/config/bp2build.go @@ -16,9 +16,13 @@ package config import ( "fmt" + "reflect" "regexp" "sort" "strings" + + "android/soong/android" + "github.com/google/blueprint" ) const ( @@ -26,18 +30,27 @@ const ( ) type bazelVarExporter interface { - asBazel(exportedStringVariables, exportedStringListVariables) []bazelConstant + asBazel(android.Config, exportedStringVariables, exportedStringListVariables, exportedConfigDependingVariables) []bazelConstant } // Helpers for exporting cc configuration information to Bazel. var ( - // Map containing toolchain variables that are independent of the + // Maps containing toolchain variables that are independent of the // environment variables of the build. exportedStringListVars = exportedStringListVariables{} exportedStringVars = exportedStringVariables{} exportedStringListDictVars = exportedStringListDictVariables{} + + /// Maps containing variables that are dependent on the build config. + exportedConfigDependingVars = exportedConfigDependingVariables{} ) +type exportedConfigDependingVariables map[string]interface{} + +func (m exportedConfigDependingVariables) Set(k string, v interface{}) { + m[k] = v +} + // Ensure that string s has no invalid characters to be generated into the bzl file. func validateCharacters(s string) string { for _, c := range []string{`\n`, `"`, `\`} { @@ -74,10 +87,14 @@ func printBazelList(items []string, indentLevel int) string { return strings.Join(list, "\n") } -func (m exportedStringVariables) asBazel(stringScope exportedStringVariables, stringListScope exportedStringListVariables) []bazelConstant { +func (m exportedStringVariables) asBazel(config android.Config, + stringVars exportedStringVariables, stringListVars exportedStringListVariables, cfgDepVars exportedConfigDependingVariables) []bazelConstant { ret := make([]bazelConstant, 0, len(m)) for k, variableValue := range m { - expandedVar := expandVar(variableValue, exportedStringVars, exportedStringListVars) + expandedVar, err := expandVar(config, variableValue, stringVars, stringListVars, cfgDepVars) + if err != nil { + panic(fmt.Errorf("error expanding config variable %s: %s", k, err)) + } if len(expandedVar) > 1 { panic(fmt.Errorf("%s expands to more than one string value: %s", variableValue, expandedVar)) } @@ -101,7 +118,9 @@ func (m exportedStringListVariables) Set(k string, v []string) { m[k] = v } -func (m exportedStringListVariables) asBazel(stringScope exportedStringVariables, stringListScope exportedStringListVariables) []bazelConstant { +func (m exportedStringListVariables) asBazel(config android.Config, + stringScope exportedStringVariables, stringListScope exportedStringListVariables, + exportedVars exportedConfigDependingVariables) []bazelConstant { ret := make([]bazelConstant, 0, len(m)) // For each exported variable, recursively expand elements in the variableValue // list to ensure that interpolated variables are expanded according to their values @@ -109,7 +128,11 @@ func (m exportedStringListVariables) asBazel(stringScope exportedStringVariables for k, variableValue := range m { var expandedVars []string for _, v := range variableValue { - expandedVars = append(expandedVars, expandVar(v, stringScope, stringListScope)...) + expandedVar, err := expandVar(config, v, stringScope, stringListScope, exportedVars) + if err != nil { + panic(fmt.Errorf("Error expanding config variable %s=%s: %s", k, v, err)) + } + expandedVars = append(expandedVars, expandedVar...) } // Assign the list as a bzl-private variable; this variable will be exported // out through a constants struct later. @@ -121,6 +144,18 @@ func (m exportedStringListVariables) asBazel(stringScope exportedStringVariables return ret } +// Convenience function to declare a static "source path" variable and export it to Bazel's cc_toolchain. +func exportVariableConfigMethod(name string, method interface{}) blueprint.Variable { + exportedConfigDependingVars.Set(name, method) + return pctx.VariableConfigMethod(name, method) +} + +// Convenience function to declare a static "source path" variable and export it to Bazel's cc_toolchain. +func exportSourcePathVariable(name string, value string) { + pctx.SourcePathVariable(name, value) + exportedStringVars.Set(name, value) +} + // Convenience function to declare a static variable and export it to Bazel's cc_toolchain. func exportStringListStaticVariable(name string, value []string) { pctx.StaticVariable(name, strings.Join(value, " ")) @@ -145,7 +180,8 @@ func printBazelStringListDict(dict map[string][]string) string { } // Since dictionaries are not supported in Ninja, we do not expand variables for dictionaries -func (m exportedStringListDictVariables) asBazel(_ exportedStringVariables, _ exportedStringListVariables) []bazelConstant { +func (m exportedStringListDictVariables) asBazel(_ android.Config, _ exportedStringVariables, + _ exportedStringListVariables, _ exportedConfigDependingVariables) []bazelConstant { ret := make([]bazelConstant, 0, len(m)) for k, dict := range m { ret = append(ret, bazelConstant{ @@ -158,19 +194,20 @@ func (m exportedStringListDictVariables) asBazel(_ exportedStringVariables, _ ex // BazelCcToolchainVars generates bzl file content containing variables for // Bazel's cc_toolchain configuration. -func BazelCcToolchainVars() string { +func BazelCcToolchainVars(config android.Config) string { return bazelToolchainVars( + config, exportedStringListDictVars, exportedStringListVars, exportedStringVars) } -func bazelToolchainVars(vars ...bazelVarExporter) string { +func bazelToolchainVars(config android.Config, vars ...bazelVarExporter) string { ret := "# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT.\n\n" results := []bazelConstant{} for _, v := range vars { - results = append(results, v.asBazel(exportedStringVars, exportedStringListVars)...) + results = append(results, v.asBazel(config, exportedStringVars, exportedStringListVars, exportedConfigDependingVars)...) } sort.Slice(results, func(i, j int) bool { return results[i].variableName < results[j].variableName }) @@ -201,34 +238,35 @@ func bazelToolchainVars(vars ...bazelVarExporter) string { // string slice than to handle a pass-by-referenced map, which would make it // quite complex to track depth-first interpolations. It's also unlikely the // interpolation stacks are deep (n > 1). -func expandVar(toExpand string, stringScope exportedStringVariables, stringListScope exportedStringListVariables) []string { +func expandVar(config android.Config, toExpand string, stringScope exportedStringVariables, + stringListScope exportedStringListVariables, exportedVars exportedConfigDependingVariables) ([]string, error) { // e.g. "${ExternalCflags}" r := regexp.MustCompile(`\${([a-zA-Z0-9_]+)}`) // Internal recursive function. - var expandVarInternal func(string, map[string]bool) []string - expandVarInternal = func(toExpand string, seenVars map[string]bool) []string { - var ret []string - for _, v := range strings.Split(toExpand, " ") { - matches := r.FindStringSubmatch(v) + var expandVarInternal func(string, map[string]bool) (string, error) + expandVarInternal = func(toExpand string, seenVars map[string]bool) (string, error) { + var ret string + remainingString := toExpand + for len(remainingString) > 0 { + matches := r.FindStringSubmatch(remainingString) if len(matches) == 0 { - return []string{v} + return ret + remainingString, nil } - if len(matches) != 2 { - panic(fmt.Errorf( - "Expected to only match 1 subexpression in %s, got %d", - v, - len(matches)-1)) + panic(fmt.Errorf("Expected to only match 1 subexpression in %s, got %d", remainingString, len(matches)-1)) } + matchIndex := strings.Index(remainingString, matches[0]) + ret += remainingString[:matchIndex] + remainingString = remainingString[matchIndex+len(matches[0]):] // Index 1 of FindStringSubmatch contains the subexpression match // (variable name) of the capture group. variable := matches[1] // toExpand contains a variable. if _, ok := seenVars[variable]; ok { - panic(fmt.Errorf( - "Unbounded recursive interpolation of variable: %s", variable)) + return ret, fmt.Errorf( + "Unbounded recursive interpolation of variable: %s", variable) } // A map is passed-by-reference. Create a new map for // this scope to prevent variables seen in one depth-first expansion @@ -239,15 +277,65 @@ func expandVar(toExpand string, stringScope exportedStringVariables, stringListS } newSeenVars[variable] = true if unexpandedVars, ok := stringListScope[variable]; ok { + expandedVars := []string{} for _, unexpandedVar := range unexpandedVars { - ret = append(ret, expandVarInternal(unexpandedVar, newSeenVars)...) + expandedVar, err := expandVarInternal(unexpandedVar, newSeenVars) + if err != nil { + return ret, err + } + expandedVars = append(expandedVars, expandedVar) } + ret += strings.Join(expandedVars, " ") } else if unexpandedVar, ok := stringScope[variable]; ok { - ret = append(ret, expandVarInternal(unexpandedVar, newSeenVars)...) + expandedVar, err := expandVarInternal(unexpandedVar, newSeenVars) + if err != nil { + return ret, err + } + ret += expandedVar + } else if unevaluatedVar, ok := exportedVars[variable]; ok { + evalFunc := reflect.ValueOf(unevaluatedVar) + validateVariableMethod(variable, evalFunc) + evaluatedResult := evalFunc.Call([]reflect.Value{reflect.ValueOf(config)}) + evaluatedValue := evaluatedResult[0].Interface().(string) + expandedVar, err := expandVarInternal(evaluatedValue, newSeenVars) + if err != nil { + return ret, err + } + ret += expandedVar + } else { + return "", fmt.Errorf("Unbound config variable %s", variable) } } - return ret + return ret, nil + } + var ret []string + for _, v := range strings.Split(toExpand, " ") { + val, err := expandVarInternal(v, map[string]bool{}) + if err != nil { + return ret, err + } + ret = append(ret, val) } - return expandVarInternal(toExpand, map[string]bool{}) + return ret, nil +} + +func validateVariableMethod(name string, methodValue reflect.Value) { + methodType := methodValue.Type() + if methodType.Kind() != reflect.Func { + panic(fmt.Errorf("method given for variable %s is not a function", + name)) + } + if n := methodType.NumIn(); n != 1 { + panic(fmt.Errorf("method for variable %s has %d inputs (should be 1)", + name, n)) + } + if n := methodType.NumOut(); n != 1 { + panic(fmt.Errorf("method for variable %s has %d outputs (should be 1)", + name, n)) + } + if kind := methodType.Out(0).Kind(); kind != reflect.String { + panic(fmt.Errorf("method for variable %s does not return a string", + name)) + } } diff --git a/cc/config/bp2build_test.go b/cc/config/bp2build_test.go index 883597ad3..3118df1f8 100644 --- a/cc/config/bp2build_test.go +++ b/cc/config/bp2build_test.go @@ -16,13 +16,21 @@ package config import ( "testing" + + "android/soong/android" ) func TestExpandVars(t *testing.T) { + android_arm64_config := android.TestConfig("out", nil, "", nil) + android_arm64_config.BuildOS = android.Android + android_arm64_config.BuildArch = android.Arm64 + testCases := []struct { description string + config android.Config stringScope exportedStringVariables stringListScope exportedStringListVariables + configVars exportedConfigDependingVariables toExpand string expectedValues []string }{ @@ -57,7 +65,7 @@ func TestExpandVars(t *testing.T) { "bar": []string{"baz", "${qux}"}, }, toExpand: "${foo}", - expectedValues: []string{"baz", "hello"}, + expectedValues: []string{"baz hello"}, }, { description: "double level expansion", @@ -75,7 +83,7 @@ func TestExpandVars(t *testing.T) { "b": []string{"d"}, }, toExpand: "${a}", - expectedValues: []string{"d", "c"}, + expectedValues: []string{"d c"}, }, { description: "double level expansion, with two variables in a string", @@ -85,7 +93,7 @@ func TestExpandVars(t *testing.T) { "c": []string{"e"}, }, toExpand: "${a}", - expectedValues: []string{"d", "e"}, + expectedValues: []string{"d e"}, }, { description: "triple level expansion with two variables in a string", @@ -96,13 +104,38 @@ func TestExpandVars(t *testing.T) { "d": []string{"foo"}, }, toExpand: "${a}", - expectedValues: []string{"foo", "foo", "foo"}, + expectedValues: []string{"foo foo foo"}, + }, + { + description: "expansion with config depending vars", + configVars: exportedConfigDependingVariables{ + "a": func(c android.Config) string { return c.BuildOS.String() }, + "b": func(c android.Config) string { return c.BuildArch.String() }, + }, + config: android_arm64_config, + toExpand: "${a}-${b}", + expectedValues: []string{"android-arm64"}, + }, + { + description: "double level multi type expansion", + stringListScope: exportedStringListVariables{ + "platform": []string{"${os}-${arch}"}, + "const": []string{"const"}, + }, + configVars: exportedConfigDependingVariables{ + "os": func(c android.Config) string { return c.BuildOS.String() }, + "arch": func(c android.Config) string { return c.BuildArch.String() }, + "foo": func(c android.Config) string { return "foo" }, + }, + config: android_arm64_config, + toExpand: "${const}/${platform}/${foo}", + expectedValues: []string{"const/android-arm64/foo"}, }, } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { - output := expandVar(testCase.toExpand, testCase.stringScope, testCase.stringListScope) + output, _ := expandVar(testCase.config, testCase.toExpand, testCase.stringScope, testCase.stringListScope, testCase.configVars) if len(output) != len(testCase.expectedValues) { t.Errorf("Expected %d values, got %d", len(testCase.expectedValues), len(output)) } @@ -119,6 +152,7 @@ func TestExpandVars(t *testing.T) { func TestBazelToolchainVars(t *testing.T) { testCases := []struct { name string + config android.Config vars []bazelVarExporter expectedOut string }{ @@ -248,7 +282,7 @@ constants = struct( for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - out := bazelToolchainVars(tc.vars...) + out := bazelToolchainVars(tc.config, tc.vars...) if out != tc.expectedOut { t.Errorf("Expected \n%s, got \n%s", tc.expectedOut, out) } diff --git a/cc/config/global.go b/cc/config/global.go index 6108d3d84..7abd2fc12 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -404,7 +404,7 @@ func init() { pctx.StaticVariableWithEnvOverride("REAbiLinkerExecStrategy", "RBE_ABI_LINKER_EXEC_STRATEGY", remoteexec.LocalExecStrategy) } -var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) +var HostPrebuiltTag = exportVariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) func ClangPath(ctx android.PathContext, file string) android.SourcePath { type clangToolKey string diff --git a/cc/config/x86_linux_host.go b/cc/config/x86_linux_host.go index ac5d5f74e..43333fada 100644 --- a/cc/config/x86_linux_host.go +++ b/cc/config/x86_linux_host.go @@ -120,40 +120,40 @@ const ( ) func init() { - pctx.StaticVariable("LinuxGccVersion", linuxGccVersion) - pctx.StaticVariable("LinuxGlibcVersion", linuxGlibcVersion) + exportStringStaticVariable("LinuxGccVersion", linuxGccVersion) + exportStringStaticVariable("LinuxGlibcVersion", linuxGlibcVersion) + // Most places use the full GCC version. A few only use up to the first two numbers. if p := strings.Split(linuxGccVersion, "."); len(p) > 2 { - pctx.StaticVariable("ShortLinuxGccVersion", strings.Join(p[:2], ".")) + exportStringStaticVariable("ShortLinuxGccVersion", strings.Join(p[:2], ".")) } else { - pctx.StaticVariable("ShortLinuxGccVersion", linuxGccVersion) + exportStringStaticVariable("ShortLinuxGccVersion", linuxGccVersion) } - pctx.SourcePathVariable("LinuxGccRoot", - "prebuilts/gcc/${HostPrebuiltTag}/host/x86_64-linux-glibc${LinuxGlibcVersion}-${ShortLinuxGccVersion}") + exportSourcePathVariable("LinuxGccRoot", + "prebuilts/gcc/linux-x86/host/x86_64-linux-glibc${LinuxGlibcVersion}-${ShortLinuxGccVersion}") - pctx.StaticVariable("LinuxGccTriple", "x86_64-linux") + exportStringListStaticVariable("LinuxGccTriple", []string{"x86_64-linux"}) - pctx.StaticVariable("LinuxCflags", strings.Join(linuxCflags, " ")) - pctx.StaticVariable("LinuxLdflags", strings.Join(linuxLdflags, " ")) - pctx.StaticVariable("LinuxLldflags", strings.Join(linuxLdflags, " ")) + exportStringListStaticVariable("LinuxCflags", linuxCflags) + exportStringListStaticVariable("LinuxLdflags", linuxLdflags) + exportStringListStaticVariable("LinuxLldflags", linuxLdflags) + exportStringListStaticVariable("LinuxGlibcCflags", linuxGlibcCflags) + exportStringListStaticVariable("LinuxGlibcLdflags", linuxGlibcLdflags) + exportStringListStaticVariable("LinuxGlibcLldflags", linuxGlibcLdflags) + exportStringListStaticVariable("LinuxMuslCflags", linuxMuslCflags) + exportStringListStaticVariable("LinuxMuslLdflags", linuxMuslLdflags) + exportStringListStaticVariable("LinuxMuslLldflags", linuxMuslLdflags) - pctx.StaticVariable("LinuxX86Cflags", strings.Join(linuxX86Cflags, " ")) - pctx.StaticVariable("LinuxX8664Cflags", strings.Join(linuxX8664Cflags, " ")) - pctx.StaticVariable("LinuxX86Ldflags", strings.Join(linuxX86Ldflags, " ")) - pctx.StaticVariable("LinuxX86Lldflags", strings.Join(linuxX86Ldflags, " ")) - pctx.StaticVariable("LinuxX8664Ldflags", strings.Join(linuxX8664Ldflags, " ")) - pctx.StaticVariable("LinuxX8664Lldflags", strings.Join(linuxX8664Ldflags, " ")) + exportStringListStaticVariable("LinuxX86Cflags", linuxX86Cflags) + exportStringListStaticVariable("LinuxX8664Cflags", linuxX8664Cflags) + exportStringListStaticVariable("LinuxX86Ldflags", linuxX86Ldflags) + exportStringListStaticVariable("LinuxX86Lldflags", linuxX86Ldflags) + exportStringListStaticVariable("LinuxX8664Ldflags", linuxX8664Ldflags) + exportStringListStaticVariable("LinuxX8664Lldflags", linuxX8664Ldflags) // Yasm flags - pctx.StaticVariable("LinuxX86YasmFlags", "-f elf32 -m x86") - pctx.StaticVariable("LinuxX8664YasmFlags", "-f elf64 -m amd64") - - pctx.StaticVariable("LinuxGlibcCflags", strings.Join(linuxGlibcCflags, " ")) - pctx.StaticVariable("LinuxGlibcLdflags", strings.Join(linuxGlibcLdflags, " ")) - pctx.StaticVariable("LinuxGlibcLldflags", strings.Join(linuxGlibcLdflags, " ")) - pctx.StaticVariable("LinuxMuslCflags", strings.Join(linuxMuslCflags, " ")) - pctx.StaticVariable("LinuxMuslLdflags", strings.Join(linuxMuslLdflags, " ")) - pctx.StaticVariable("LinuxMuslLldflags", strings.Join(linuxMuslLdflags, " ")) + exportStringListStaticVariable("LinuxX86YasmFlags", []string{"-f elf32 -m x86"}) + exportStringListStaticVariable("LinuxX8664YasmFlags", []string{"-f elf64 -m amd64"}) } type toolchainLinux struct {