Add crunch-flags and build-flag binaries

- crunch-flags automates converting build flags from starlark to protobuf.
- build-flag is used to set, get and trace flag values.

Bug: 328495189
Test: manual
Change-Id: I941a4420a8bdfa2df73d94e52b3f34a6d1ea3278
This commit is contained in:
LaMont Jones
2024-04-11 17:41:15 -07:00
parent 14e2ac68e3
commit b9014c74f4
4 changed files with 652 additions and 0 deletions

View File

@@ -0,0 +1,32 @@
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
blueprint_go_binary {
name: "build-flag",
deps: [
"golang-protobuf-encoding-prototext",
"golang-protobuf-reflect-protoreflect",
"golang-protobuf-runtime-protoimpl",
"soong-cmd-release_config-proto",
"soong-cmd-release_config-lib",
],
srcs: [
"main.go",
],
}
bootstrap_go_package {
name: "soong-cmd-release_config-build_flag",
pkgPath: "android/soong/cmd/release_config/build_flag",
deps: [
"golang-protobuf-encoding-prototext",
"golang-protobuf-reflect-protoreflect",
"golang-protobuf-runtime-protoimpl",
"soong-cmd-release_config-proto",
"soong-cmd-release_config-lib",
],
srcs: [
"main.go",
],
}

View File

@@ -0,0 +1,229 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
rc_lib "android/soong/cmd/release_config/release_config_lib"
rc_proto "android/soong/cmd/release_config/release_config_proto"
"google.golang.org/protobuf/proto"
)
type Flags struct {
// The path to the top of the workspace. Default: ".".
top string
// Pathlist of release config map textproto files.
// If not specified, then the value is (if present):
// - build/release/release_config_map.textproto
// - vendor/google_shared/build/release/release_config_map.textproto
// - vendor/google/release/release_config_map.textproto
//
// Additionally, any maps specified in the environment variable
// `PRODUCT_RELEASE_CONFIG_MAPS` are used.
maps rc_lib.StringList
// Output directory (relative to `top`).
outDir string
// Which $TARGET_RELEASE(s) should we use. Some commands will only
// accept one value, others also accept `--release --all`.
targetReleases rc_lib.StringList
// Disable warning messages
quiet bool
}
type CommandFunc func(*rc_lib.ReleaseConfigs, Flags, string, []string) error
var commandMap map[string]CommandFunc = map[string]CommandFunc{
"get": GetCommand,
"set": SetCommand,
"trace": GetCommand, // Also handled by GetCommand
}
// Find the top of the release config contribution directory.
// Returns the parent of the flag_declarations and flag_values directories.
func GetMapDir(path string) (string, error) {
for p := path; p != "."; p = filepath.Dir(p) {
switch filepath.Base(p) {
case "flag_declarations":
return filepath.Dir(p), nil
case "flag_values":
return filepath.Dir(p), nil
}
}
return "", fmt.Errorf("Could not determine directory from %s", path)
}
func MarshalFlagValue(config *rc_lib.ReleaseConfig, name string) (ret string, err error) {
fa, ok := config.FlagArtifacts[name]
if !ok {
return "", fmt.Errorf("%s not found in %s", name, config.Name)
}
return rc_lib.MarshalValue(fa.Value), nil
}
func GetReleaseArgs(configs *rc_lib.ReleaseConfigs, commonFlags Flags) ([]*rc_lib.ReleaseConfig, error) {
var all bool
relFlags := flag.NewFlagSet("set", flag.ExitOnError)
relFlags.BoolVar(&all, "all", false, "Display all flags")
relFlags.Parse(commonFlags.targetReleases)
var ret []*rc_lib.ReleaseConfig
if all {
for _, config := range configs.ReleaseConfigs {
ret = append(ret, config)
}
return ret, nil
}
for _, arg := range relFlags.Args() {
config, err := configs.GetReleaseConfig(arg)
if err != nil {
return nil, err
}
ret = append(ret, config)
}
return ret, nil
}
func GetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error {
isTrace := cmd == "trace"
var all bool
getFlags := flag.NewFlagSet("set", flag.ExitOnError)
getFlags.BoolVar(&all, "all", false, "Display all flags")
getFlags.Parse(args)
args = getFlags.Args()
releaseConfigList, err := GetReleaseArgs(configs, commonFlags)
if err != nil {
return err
}
if isTrace && len(releaseConfigList) > 1 {
return fmt.Errorf("trace command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " "))
}
if all {
args = []string{}
for _, fa := range configs.FlagArtifacts {
args = append(args, *fa.FlagDeclaration.Name)
}
}
showName := len(releaseConfigList) > 1 || len(args) > 1
for _, config := range releaseConfigList {
var configName string
if len(releaseConfigList) > 1 {
configName = fmt.Sprintf("%s.", config.Name)
}
for _, arg := range args {
val, err := MarshalFlagValue(config, arg)
if err != nil {
return err
}
if showName {
fmt.Printf("%s%s=%s\n", configName, arg, val)
} else {
fmt.Printf("%s\n", val)
}
if isTrace {
for _, trace := range config.FlagArtifacts[arg].Traces {
fmt.Printf(" => \"%s\" in %s\n", rc_lib.MarshalValue(trace.Value), *trace.Source)
}
}
}
}
return nil
}
func SetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error {
var valueDir string
if len(commonFlags.targetReleases) > 1 {
return fmt.Errorf("set command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " "))
}
targetRelease := commonFlags.targetReleases[0]
setFlags := flag.NewFlagSet("set", flag.ExitOnError)
setFlags.StringVar(&valueDir, "dir", "", "Directory in which to place the value")
setFlags.Parse(args)
setArgs := setFlags.Args()
if len(setArgs) != 2 {
return fmt.Errorf("set command expected flag and value, got: %s", strings.Join(setArgs, " "))
}
name := setArgs[0]
value := setArgs[1]
release, err := configs.GetReleaseConfig(targetRelease)
targetRelease = release.Name
if err != nil {
return err
}
flagArtifact, ok := release.FlagArtifacts[name]
if !ok {
return fmt.Errorf("Unknown build flag %s", name)
}
if valueDir == "" {
mapDir, err := GetMapDir(*flagArtifact.Traces[len(flagArtifact.Traces)-1].Source)
if err != nil {
return err
}
valueDir = mapDir
}
flagValue := &rc_proto.FlagValue{
Name: proto.String(name),
Value: rc_lib.UnmarshalValue(value),
}
flagPath := filepath.Join(valueDir, "flag_values", targetRelease, fmt.Sprintf("%s.textproto", name))
return rc_lib.WriteMessage(flagPath, flagValue)
}
func main() {
var err error
var commonFlags Flags
var configs *rc_lib.ReleaseConfigs
outEnv := os.Getenv("OUT_DIR")
if outEnv == "" {
outEnv = "out"
}
// Handle the common arguments
flag.StringVar(&commonFlags.top, "top", ".", "path to top of workspace")
flag.BoolVar(&commonFlags.quiet, "quiet", false, "disable warning messages")
flag.Var(&commonFlags.maps, "map", "path to a release_config_map.textproto. may be repeated")
flag.StringVar(&commonFlags.outDir, "out_dir", rc_lib.GetDefaultOutDir(), "basepath for the output. Multiple formats are created")
flag.Var(&commonFlags.targetReleases, "release", "TARGET_RELEASE for this build")
flag.Parse()
if commonFlags.quiet {
rc_lib.DisableWarnings()
}
if len(commonFlags.targetReleases) == 0 {
commonFlags.targetReleases = rc_lib.StringList{"trunk_staging"}
}
if err = os.Chdir(commonFlags.top); err != nil {
panic(err)
}
// Get the current state of flagging.
relName := commonFlags.targetReleases[0]
if relName == "--all" || relName == "-all" {
// If the users said `--release --all`, grab trunk staging for simplicity.
relName = "trunk_staging"
}
configs, err = rc_lib.ReadReleaseConfigMaps(commonFlags.maps, relName)
if err != nil {
panic(err)
}
if cmd, ok := commandMap[flag.Arg(0)]; ok {
args := flag.Args()
if err = cmd(configs, commonFlags, args[0], args[1:]); err != nil {
panic(err)
}
}
}

View File

@@ -0,0 +1,32 @@
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
blueprint_go_binary {
name: "crunch-flags",
deps: [
"golang-protobuf-encoding-prototext",
"golang-protobuf-reflect-protoreflect",
"golang-protobuf-runtime-protoimpl",
"soong-cmd-release_config-lib",
"soong-cmd-release_config-proto",
],
srcs: [
"main.go",
],
}
bootstrap_go_package {
name: "soong-cmd-release_config-crunch_flags",
pkgPath: "android/soong/cmd/release_config/crunch_flags",
deps: [
"golang-protobuf-encoding-prototext",
"golang-protobuf-reflect-protoreflect",
"golang-protobuf-runtime-protoimpl",
"soong-cmd-release_config-lib",
"soong-cmd-release_config-proto",
],
srcs: [
"main.go",
],
}

View File

@@ -0,0 +1,359 @@
package main
import (
"flag"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
rc_lib "android/soong/cmd/release_config/release_config_lib"
rc_proto "android/soong/cmd/release_config/release_config_proto"
"google.golang.org/protobuf/encoding/prototext"
"google.golang.org/protobuf/proto"
)
// When a flag declaration has an initial value that is a string, the default workflow is PREBUILT.
// If the flag name starts with any of prefixes in manualFlagNamePrefixes, it is MANUAL.
var manualFlagNamePrefixes []string = []string{
"RELEASE_ACONFIG_",
"RELEASE_PLATFORM_",
}
var defaultFlagNamespace string = "android_UNKNOWN"
func RenameNext(name string) string {
if name == "next" {
return "ap3a"
}
return name
}
func WriteFile(path string, message proto.Message) error {
data, err := prototext.MarshalOptions{Multiline: true}.Marshal(message)
if err != nil {
return err
}
err = os.MkdirAll(filepath.Dir(path), 0775)
if err != nil {
return err
}
return os.WriteFile(path, data, 0644)
}
func WalkValueFiles(dir string, Func fs.WalkDirFunc) error {
valPath := filepath.Join(dir, "build_config")
if _, err := os.Stat(valPath); err != nil {
fmt.Printf("%s not found, ignoring.\n", valPath)
return nil
}
return filepath.WalkDir(valPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if strings.HasSuffix(d.Name(), ".scl") && d.Type().IsRegular() {
return Func(path, d, err)
}
return nil
})
}
func ProcessBuildFlags(dir string, namespaceMap map[string]string) error {
var rootAconfigModule string
path := filepath.Join(dir, "build_flags.scl")
if _, err := os.Stat(path); err != nil {
fmt.Printf("%s not found, ignoring.\n", path)
return nil
} else {
fmt.Printf("Processing %s\n", path)
}
commentRegexp, err := regexp.Compile("^[[:space:]]*#(?<comment>.+)")
if err != nil {
return err
}
declRegexp, err := regexp.Compile("^[[:space:]]*flag.\"(?<name>[A-Z_0-9]+)\",[[:space:]]*(?<container>[_A-Z]*),[[:space:]]*(?<value>(\"[^\"]*\"|[^\",)]*))")
if err != nil {
return err
}
declIn, err := os.ReadFile(path)
if err != nil {
return err
}
lines := strings.Split(string(declIn), "\n")
var description string
for _, line := range lines {
if comment := commentRegexp.FindStringSubmatch(commentRegexp.FindString(line)); comment != nil {
// Description is the text from any contiguous series of lines before a `flag()` call.
description += fmt.Sprintf(" %s", strings.TrimSpace(comment[commentRegexp.SubexpIndex("comment")]))
continue
}
matches := declRegexp.FindStringSubmatch(declRegexp.FindString(line))
if matches == nil {
// The line is neither a comment nor a `flag()` call.
// Discard any description we have gathered and process the next line.
description = ""
continue
}
declValue := matches[declRegexp.SubexpIndex("value")]
declName := matches[declRegexp.SubexpIndex("name")]
container := rc_proto.Container(rc_proto.Container_value[matches[declRegexp.SubexpIndex("container")]])
description = strings.TrimSpace(description)
var namespace string
var ok bool
if namespace, ok = namespaceMap[declName]; !ok {
namespace = defaultFlagNamespace
}
flagDeclaration := &rc_proto.FlagDeclaration{
Name: proto.String(declName),
Namespace: proto.String(namespace),
Description: proto.String(description),
Container: &container,
}
description = ""
// Most build flags are `workflow: PREBUILT`.
workflow := rc_proto.Workflow(rc_proto.Workflow_PREBUILT)
switch {
case declName == "RELEASE_ACONFIG_VALUE_SETS":
rootAconfigModule = declValue[1 : len(declValue)-1]
continue
case strings.HasPrefix(declValue, "\""):
// String values mean that the flag workflow is (most likely) either MANUAL or PREBUILT.
declValue = declValue[1 : len(declValue)-1]
flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{declValue}}
for _, prefix := range manualFlagNamePrefixes {
if strings.HasPrefix(declName, prefix) {
workflow = rc_proto.Workflow(rc_proto.Workflow_MANUAL)
break
}
}
case declValue == "False" || declValue == "True":
// Boolean values are LAUNCH flags.
flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{declValue == "True"}}
workflow = rc_proto.Workflow(rc_proto.Workflow_LAUNCH)
case declValue == "None":
// Use PREBUILT workflow with no initial value.
default:
fmt.Printf("%s: Unexpected value %s=%s\n", path, declName, declValue)
}
flagDeclaration.Workflow = &workflow
if flagDeclaration != nil {
declPath := filepath.Join(dir, "flag_declarations", fmt.Sprintf("%s.textproto", declName))
err := WriteFile(declPath, flagDeclaration)
if err != nil {
return err
}
}
}
if rootAconfigModule != "" {
rootProto := &rc_proto.ReleaseConfig{
Name: proto.String("root"),
AconfigValueSets: []string{rootAconfigModule},
}
return WriteFile(filepath.Join(dir, "release_configs", "root.textproto"), rootProto)
}
return nil
}
func ProcessBuildConfigs(dir, name string, paths []string, releaseProto *rc_proto.ReleaseConfig) error {
valRegexp, err := regexp.Compile("[[:space:]]+value.\"(?<name>[A-Z_0-9]+)\",[[:space:]]*(?<value>[^,)]*)")
if err != nil {
return err
}
for _, path := range paths {
fmt.Printf("Processing %s\n", path)
valIn, err := os.ReadFile(path)
if err != nil {
fmt.Printf("%s: error: %v\n", path, err)
return err
}
vals := valRegexp.FindAllString(string(valIn), -1)
for _, val := range vals {
matches := valRegexp.FindStringSubmatch(val)
valValue := matches[valRegexp.SubexpIndex("value")]
valName := matches[valRegexp.SubexpIndex("name")]
flagValue := &rc_proto.FlagValue{
Name: proto.String(valName),
}
switch {
case valName == "RELEASE_ACONFIG_VALUE_SETS":
flagValue = nil
if releaseProto.AconfigValueSets == nil {
releaseProto.AconfigValueSets = []string{}
}
releaseProto.AconfigValueSets = append(releaseProto.AconfigValueSets, valValue[1:len(valValue)-1])
case strings.HasPrefix(valValue, "\""):
valValue = valValue[1 : len(valValue)-1]
flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{valValue}}
case valValue == "None":
// nothing to do here.
case valValue == "True":
flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{true}}
case valValue == "False":
flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{false}}
default:
fmt.Printf("%s: Unexpected value %s=%s\n", path, valName, valValue)
}
if flagValue != nil {
valPath := filepath.Join(dir, "flag_values", RenameNext(name), fmt.Sprintf("%s.textproto", valName))
err := WriteFile(valPath, flagValue)
if err != nil {
return err
}
}
}
}
return err
}
func ProcessReleaseConfigMap(dir string, descriptionMap map[string]string) error {
path := filepath.Join(dir, "release_config_map.mk")
if _, err := os.Stat(path); err != nil {
fmt.Printf("%s not found, ignoring.\n", path)
return nil
} else {
fmt.Printf("Processing %s\n", path)
}
configRegexp, err := regexp.Compile("^..call[[:space:]]+declare-release-config,[[:space:]]+(?<name>[_a-z0-0A-Z]+),[[:space:]]+(?<files>[^,]*)(,[[:space:]]*(?<inherits>.*)|[[:space:]]*)[)]$")
if err != nil {
return err
}
aliasRegexp, err := regexp.Compile("^..call[[:space:]]+alias-release-config,[[:space:]]+(?<name>[_a-z0-9A-Z]+),[[:space:]]+(?<target>[_a-z0-9A-Z]+)")
if err != nil {
return err
}
mapIn, err := os.ReadFile(path)
if err != nil {
return err
}
cleanDir := strings.TrimLeft(dir, "../")
var defaultContainer rc_proto.Container
switch {
case strings.HasPrefix(cleanDir, "build/") || cleanDir == "vendor/google_shared/build":
defaultContainer = rc_proto.Container(rc_proto.Container_ALL)
case cleanDir == "vendor/google/release":
defaultContainer = rc_proto.Container(rc_proto.Container_ALL)
default:
defaultContainer = rc_proto.Container(rc_proto.Container_VENDOR)
}
releaseConfigMap := &rc_proto.ReleaseConfigMap{DefaultContainer: &defaultContainer}
// If we find a description for the directory, include it.
if description, ok := descriptionMap[cleanDir]; ok {
releaseConfigMap.Description = proto.String(description)
}
lines := strings.Split(string(mapIn), "\n")
for _, line := range lines {
alias := aliasRegexp.FindStringSubmatch(aliasRegexp.FindString(line))
if alias != nil {
fmt.Printf("processing alias %s\n", line)
name := alias[aliasRegexp.SubexpIndex("name")]
target := alias[aliasRegexp.SubexpIndex("target")]
if target == "next" {
if RenameNext(target) != name {
return fmt.Errorf("Unexpected name for next (%s)", RenameNext(target))
}
target, name = name, target
}
releaseConfigMap.Aliases = append(releaseConfigMap.Aliases,
&rc_proto.ReleaseAlias{
Name: proto.String(name),
Target: proto.String(target),
})
}
config := configRegexp.FindStringSubmatch(configRegexp.FindString(line))
if config == nil {
continue
}
name := config[configRegexp.SubexpIndex("name")]
releaseConfig := &rc_proto.ReleaseConfig{
Name: proto.String(RenameNext(name)),
}
configFiles := config[configRegexp.SubexpIndex("files")]
files := strings.Split(strings.ReplaceAll(configFiles, "$(local_dir)", dir+"/"), " ")
configInherits := config[configRegexp.SubexpIndex("inherits")]
if len(configInherits) > 0 {
releaseConfig.Inherits = strings.Split(configInherits, " ")
}
err := ProcessBuildConfigs(dir, name, files, releaseConfig)
if err != nil {
return err
}
releasePath := filepath.Join(dir, "release_configs", fmt.Sprintf("%s.textproto", RenameNext(name)))
err = WriteFile(releasePath, releaseConfig)
if err != nil {
return err
}
}
return WriteFile(filepath.Join(dir, "release_config_map.textproto"), releaseConfigMap)
}
func main() {
var err error
var top string
var dirs rc_lib.StringList
var namespacesFile string
var descriptionsFile string
flag.StringVar(&top, "top", ".", "path to top of workspace")
flag.Var(&dirs, "dir", "directory to process, relative to the top of the workspace")
flag.StringVar(&namespacesFile, "namespaces", "", "location of file with 'flag_name namespace' information")
flag.StringVar(&descriptionsFile, "descriptions", "", "location of file with 'directory description' information")
flag.Parse()
if err = os.Chdir(top); err != nil {
panic(err)
}
if len(dirs) == 0 {
dirs = rc_lib.StringList{"build/release", "vendor/google_shared/build/release", "vendor/google/release"}
}
namespaceMap := make(map[string]string)
if namespacesFile != "" {
data, err := os.ReadFile(namespacesFile)
if err != nil {
panic(err)
}
for idx, line := range strings.Split(string(data), "\n") {
fields := strings.Split(line, " ")
if len(fields) > 2 {
panic(fmt.Errorf("line %d: too many fields: %s", idx, line))
}
namespaceMap[fields[0]] = fields[1]
}
}
descriptionMap := make(map[string]string)
descriptionMap["build/release"] = "Published open-source flags and declarations"
if descriptionsFile != "" {
data, err := os.ReadFile(descriptionsFile)
if err != nil {
panic(err)
}
for _, line := range strings.Split(string(data), "\n") {
if strings.TrimSpace(line) != "" {
fields := strings.SplitN(line, " ", 2)
descriptionMap[fields[0]] = fields[1]
}
}
}
for _, dir := range dirs {
err = ProcessBuildFlags(dir, namespaceMap)
if err != nil {
panic(err)
}
err = ProcessReleaseConfigMap(dir, descriptionMap)
if err != nil {
panic(err)
}
}
}