cba2330e84
Flags must only be declared in one place in the tree. This change prevents new ones from being added while we fix the bad flags already present. Bug: 352105274 Test: manual Merged-In: I8c4bf2b2852685e84177500f28fe8908016082b9 Change-Id: I8c4bf2b2852685e84177500f28fe8908016082b9
540 lines
18 KiB
Go
540 lines
18 KiB
Go
// Copyright 2024 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package release_config_lib
|
|
|
|
import (
|
|
"cmp"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
|
|
rc_proto "android/soong/cmd/release_config/release_config_proto"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// A single release_config_map.textproto and its associated data.
|
|
// Used primarily for debugging.
|
|
type ReleaseConfigMap struct {
|
|
// The path to this release_config_map file.
|
|
path string
|
|
|
|
// Data received
|
|
proto rc_proto.ReleaseConfigMap
|
|
|
|
// Map of name:contribution for release config contributions.
|
|
ReleaseConfigContributions map[string]*ReleaseConfigContribution
|
|
|
|
// Flags declared this directory's flag_declarations/*.textproto
|
|
FlagDeclarations []rc_proto.FlagDeclaration
|
|
|
|
// Potential aconfig and build flag contributions in this map directory.
|
|
// This is used to detect errors.
|
|
FlagValueDirs map[string][]string
|
|
}
|
|
|
|
type ReleaseConfigDirMap map[string]int
|
|
|
|
// The generated release configs.
|
|
type ReleaseConfigs struct {
|
|
// Ordered list of release config maps processed.
|
|
ReleaseConfigMaps []*ReleaseConfigMap
|
|
|
|
// Aliases
|
|
Aliases map[string]*string
|
|
|
|
// Dictionary of flag_name:FlagDeclaration, with no overrides applied.
|
|
FlagArtifacts FlagArtifacts
|
|
|
|
// Generated release configs artifact
|
|
Artifact rc_proto.ReleaseConfigsArtifact
|
|
|
|
// Dictionary of name:ReleaseConfig
|
|
// Use `GetReleaseConfigs(name)` to get a release config.
|
|
ReleaseConfigs map[string]*ReleaseConfig
|
|
|
|
// Map of directory to *ReleaseConfigMap
|
|
releaseConfigMapsMap map[string]*ReleaseConfigMap
|
|
|
|
// The files used by all release configs
|
|
FilesUsedMap map[string]bool
|
|
|
|
// The list of config directories used.
|
|
configDirs []string
|
|
|
|
// A map from the config directory to its order in the list of config
|
|
// directories.
|
|
configDirIndexes ReleaseConfigDirMap
|
|
|
|
// True if we should allow a missing primary release config. In this
|
|
// case, we will substitute `trunk_staging` values, but the release
|
|
// config will not be in ALL_RELEASE_CONFIGS_FOR_PRODUCT.
|
|
allowMissing bool
|
|
}
|
|
|
|
func (configs *ReleaseConfigs) WriteInheritanceGraph(outFile string) error {
|
|
data := []string{}
|
|
usedAliases := make(map[string]bool)
|
|
priorStages := make(map[string][]string)
|
|
for _, config := range configs.ReleaseConfigs {
|
|
if config.Name == "root" {
|
|
continue
|
|
}
|
|
var fillColor string
|
|
inherits := []string{}
|
|
for _, inherit := range config.InheritNames {
|
|
if inherit == "root" {
|
|
continue
|
|
}
|
|
data = append(data, fmt.Sprintf(`"%s" -> "%s"`, config.Name, inherit))
|
|
inherits = append(inherits, inherit)
|
|
// If inheriting an alias, add a link from the alias to that release config.
|
|
if name, found := configs.Aliases[inherit]; found {
|
|
if !usedAliases[inherit] {
|
|
usedAliases[inherit] = true
|
|
data = append(data, fmt.Sprintf(`"%s" -> "%s"`, inherit, *name))
|
|
data = append(data,
|
|
fmt.Sprintf(`"%s" [ label="%s\ncurrently: %s" shape=oval ]`,
|
|
inherit, inherit, *name))
|
|
}
|
|
}
|
|
}
|
|
// Add links for all of the advancement progressions.
|
|
for priorStage := range config.PriorStagesMap {
|
|
data = append(data, fmt.Sprintf(`"%s" -> "%s" [ style=dashed color="#81c995" ]`,
|
|
priorStage, config.Name))
|
|
priorStages[config.Name] = append(priorStages[config.Name], priorStage)
|
|
}
|
|
label := config.Name
|
|
if len(inherits) > 0 {
|
|
label += "\\ninherits: " + strings.Join(inherits, " ")
|
|
}
|
|
if len(config.OtherNames) > 0 {
|
|
label += "\\nother names: " + strings.Join(config.OtherNames, " ")
|
|
}
|
|
switch config.Name {
|
|
case *configs.Artifact.ReleaseConfig.Name:
|
|
// The active release config has a light blue fill.
|
|
fillColor = `fillcolor="#d2e3fc" `
|
|
case "trunk", "trunk_staging":
|
|
// Certain workflow stages have a light green fill.
|
|
fillColor = `fillcolor="#ceead6" `
|
|
default:
|
|
// Look for "next" and "*_next", make them light green as well.
|
|
for _, n := range config.OtherNames {
|
|
if n == "next" || strings.HasSuffix(n, "_next") {
|
|
fillColor = `fillcolor="#ceead6" `
|
|
}
|
|
}
|
|
}
|
|
data = append(data,
|
|
fmt.Sprintf(`"%s" [ label="%s" %s]`, config.Name, label, fillColor))
|
|
}
|
|
slices.Sort(data)
|
|
data = append([]string{
|
|
"digraph {",
|
|
"graph [ ratio=.5 ]",
|
|
"node [ shape=box style=filled fillcolor=white colorscheme=svg fontcolor=black ]",
|
|
}, data...)
|
|
data = append(data, "}")
|
|
return os.WriteFile(outFile, []byte(strings.Join(data, "\n")), 0644)
|
|
}
|
|
|
|
// Write the "all_release_configs" artifact.
|
|
//
|
|
// The file will be in "{outDir}/all_release_configs-{product}.{format}"
|
|
//
|
|
// Args:
|
|
//
|
|
// outDir string: directory path. Will be created if not present.
|
|
// product string: TARGET_PRODUCT for the release_configs.
|
|
// format string: one of "json", "pb", or "textproto"
|
|
//
|
|
// Returns:
|
|
//
|
|
// error: Any error encountered.
|
|
func (configs *ReleaseConfigs) WriteArtifact(outDir, product, format string) error {
|
|
return WriteMessage(
|
|
filepath.Join(outDir, fmt.Sprintf("all_release_configs-%s.%s", product, format)),
|
|
&configs.Artifact)
|
|
}
|
|
|
|
func ReleaseConfigsFactory() (c *ReleaseConfigs) {
|
|
configs := ReleaseConfigs{
|
|
Aliases: make(map[string]*string),
|
|
FlagArtifacts: make(map[string]*FlagArtifact),
|
|
ReleaseConfigs: make(map[string]*ReleaseConfig),
|
|
releaseConfigMapsMap: make(map[string]*ReleaseConfigMap),
|
|
configDirs: []string{},
|
|
configDirIndexes: make(ReleaseConfigDirMap),
|
|
FilesUsedMap: make(map[string]bool),
|
|
}
|
|
workflowManual := rc_proto.Workflow(rc_proto.Workflow_MANUAL)
|
|
releaseAconfigValueSets := FlagArtifact{
|
|
FlagDeclaration: &rc_proto.FlagDeclaration{
|
|
Name: proto.String("RELEASE_ACONFIG_VALUE_SETS"),
|
|
Namespace: proto.String("android_UNKNOWN"),
|
|
Description: proto.String("Aconfig value sets assembled by release-config"),
|
|
Workflow: &workflowManual,
|
|
Containers: []string{"system", "system_ext", "product", "vendor"},
|
|
Value: &rc_proto.Value{Val: &rc_proto.Value_UnspecifiedValue{false}},
|
|
},
|
|
DeclarationIndex: -1,
|
|
Traces: []*rc_proto.Tracepoint{},
|
|
}
|
|
configs.FlagArtifacts["RELEASE_ACONFIG_VALUE_SETS"] = &releaseAconfigValueSets
|
|
return &configs
|
|
}
|
|
|
|
func (configs *ReleaseConfigs) GetSortedReleaseConfigs() (ret []*ReleaseConfig) {
|
|
for _, config := range configs.ReleaseConfigs {
|
|
ret = append(ret, config)
|
|
}
|
|
slices.SortFunc(ret, func(a, b *ReleaseConfig) int {
|
|
return cmp.Compare(a.Name, b.Name)
|
|
})
|
|
return ret
|
|
}
|
|
|
|
func ReleaseConfigMapFactory(protoPath string) (m *ReleaseConfigMap) {
|
|
m = &ReleaseConfigMap{
|
|
path: protoPath,
|
|
ReleaseConfigContributions: make(map[string]*ReleaseConfigContribution),
|
|
}
|
|
if protoPath != "" {
|
|
LoadMessage(protoPath, &m.proto)
|
|
}
|
|
return m
|
|
}
|
|
|
|
// Find the top of the release config contribution directory.
|
|
// Returns the parent of the flag_declarations and flag_values directories.
|
|
func (configs *ReleaseConfigs) GetDirIndex(path string) (int, error) {
|
|
for p := path; p != "."; p = filepath.Dir(p) {
|
|
if idx, ok := configs.configDirIndexes[p]; ok {
|
|
return idx, nil
|
|
}
|
|
}
|
|
return -1, fmt.Errorf("Could not determine release config directory from %s", path)
|
|
}
|
|
|
|
// Determine the default directory for writing a flag value.
|
|
//
|
|
// Returns the path of the highest-Indexed one of:
|
|
// - Where the flag is declared
|
|
// - Where the release config is first declared
|
|
// - The last place the value is being written.
|
|
func (configs *ReleaseConfigs) GetFlagValueDirectory(config *ReleaseConfig, flag *FlagArtifact) (string, error) {
|
|
current, err := configs.GetDirIndex(*flag.Traces[len(flag.Traces)-1].Source)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
index := max(flag.DeclarationIndex, config.DeclarationIndex, current)
|
|
return configs.configDirs[index], nil
|
|
}
|
|
|
|
func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex int) error {
|
|
if _, err := os.Stat(path); err != nil {
|
|
return fmt.Errorf("%s does not exist\n", path)
|
|
}
|
|
m := ReleaseConfigMapFactory(path)
|
|
if m.proto.DefaultContainers == nil {
|
|
return fmt.Errorf("Release config map %s lacks default_containers", path)
|
|
}
|
|
for _, container := range m.proto.DefaultContainers {
|
|
if !validContainer(container) {
|
|
return fmt.Errorf("Release config map %s has invalid container %s", path, container)
|
|
}
|
|
}
|
|
configs.FilesUsedMap[path] = true
|
|
dir := filepath.Dir(path)
|
|
// Record any aliases, checking for duplicates.
|
|
for _, alias := range m.proto.Aliases {
|
|
name := *alias.Name
|
|
oldTarget, ok := configs.Aliases[name]
|
|
if ok {
|
|
if *oldTarget != *alias.Target {
|
|
return fmt.Errorf("Conflicting alias declarations: %s vs %s",
|
|
*oldTarget, *alias.Target)
|
|
}
|
|
}
|
|
configs.Aliases[name] = alias.Target
|
|
}
|
|
var err error
|
|
// Temporarily allowlist duplicate flag declaration files to prevent
|
|
// more from entering the tree while we work to clean up the duplicates
|
|
// that already exist.
|
|
dupFlagFile := filepath.Join(dir, "duplicate_allowlist.txt")
|
|
data, err := os.ReadFile(dupFlagFile)
|
|
if err == nil {
|
|
for _, flag := range strings.Split(string(data), "\n") {
|
|
flag = strings.TrimSpace(flag)
|
|
if strings.HasPrefix(flag, "//") || strings.HasPrefix(flag, "#") {
|
|
continue
|
|
}
|
|
DuplicateDeclarationAllowlist[flag] = true
|
|
}
|
|
}
|
|
err = WalkTextprotoFiles(dir, "flag_declarations", func(path string, d fs.DirEntry, err error) error {
|
|
flagDeclaration := FlagDeclarationFactory(path)
|
|
// Container must be specified.
|
|
if flagDeclaration.Containers == nil {
|
|
flagDeclaration.Containers = m.proto.DefaultContainers
|
|
} else {
|
|
for _, container := range flagDeclaration.Containers {
|
|
if !validContainer(container) {
|
|
return fmt.Errorf("Flag declaration %s has invalid container %s", path, container)
|
|
}
|
|
}
|
|
}
|
|
|
|
m.FlagDeclarations = append(m.FlagDeclarations, *flagDeclaration)
|
|
name := *flagDeclaration.Name
|
|
if name == "RELEASE_ACONFIG_VALUE_SETS" {
|
|
return fmt.Errorf("%s: %s is a reserved build flag", path, name)
|
|
}
|
|
if def, ok := configs.FlagArtifacts[name]; !ok {
|
|
configs.FlagArtifacts[name] = &FlagArtifact{FlagDeclaration: flagDeclaration, DeclarationIndex: ConfigDirIndex}
|
|
} else if !proto.Equal(def.FlagDeclaration, flagDeclaration) || !DuplicateDeclarationAllowlist[name] {
|
|
return fmt.Errorf("Duplicate definition of %s in %s", *flagDeclaration.Name, path)
|
|
}
|
|
// Set the initial value in the flag artifact.
|
|
configs.FilesUsedMap[path] = true
|
|
configs.FlagArtifacts[name].UpdateValue(
|
|
FlagValue{path: path, proto: rc_proto.FlagValue{
|
|
Name: proto.String(name), Value: flagDeclaration.Value}})
|
|
if configs.FlagArtifacts[name].Redacted {
|
|
return fmt.Errorf("%s may not be redacted by default.", name)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
subDirs := func(subdir string) (ret []string) {
|
|
if flagVersions, err := os.ReadDir(filepath.Join(dir, subdir)); err == nil {
|
|
for _, e := range flagVersions {
|
|
if e.IsDir() && validReleaseConfigName(e.Name()) {
|
|
ret = append(ret, e.Name())
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
m.FlagValueDirs = map[string][]string{
|
|
"aconfig": subDirs("aconfig"),
|
|
"flag_values": subDirs("flag_values"),
|
|
}
|
|
|
|
err = WalkTextprotoFiles(dir, "release_configs", func(path string, d fs.DirEntry, err error) error {
|
|
releaseConfigContribution := &ReleaseConfigContribution{path: path, DeclarationIndex: ConfigDirIndex}
|
|
LoadMessage(path, &releaseConfigContribution.proto)
|
|
name := *releaseConfigContribution.proto.Name
|
|
if fmt.Sprintf("%s.textproto", name) != filepath.Base(path) {
|
|
return fmt.Errorf("%s incorrectly declares release config %s", path, name)
|
|
}
|
|
if _, ok := configs.ReleaseConfigs[name]; !ok {
|
|
configs.ReleaseConfigs[name] = ReleaseConfigFactory(name, ConfigDirIndex)
|
|
}
|
|
config := configs.ReleaseConfigs[name]
|
|
config.FilesUsedMap[path] = true
|
|
inheritNames := make(map[string]bool)
|
|
for _, inh := range config.InheritNames {
|
|
inheritNames[inh] = true
|
|
}
|
|
// If this contribution says to inherit something we already inherited, we do not want the duplicate.
|
|
for _, cInh := range releaseConfigContribution.proto.Inherits {
|
|
if !inheritNames[cInh] {
|
|
config.InheritNames = append(config.InheritNames, cInh)
|
|
inheritNames[cInh] = true
|
|
}
|
|
}
|
|
|
|
// Only walk flag_values/{RELEASE} for defined releases.
|
|
err2 := WalkTextprotoFiles(dir, filepath.Join("flag_values", name), func(path string, d fs.DirEntry, err error) error {
|
|
flagValue := FlagValueFactory(path)
|
|
if fmt.Sprintf("%s.textproto", *flagValue.proto.Name) != filepath.Base(path) {
|
|
return fmt.Errorf("%s incorrectly sets value for flag %s", path, *flagValue.proto.Name)
|
|
}
|
|
if *flagValue.proto.Name == "RELEASE_ACONFIG_VALUE_SETS" {
|
|
return fmt.Errorf("%s: %s is a reserved build flag", path, *flagValue.proto.Name)
|
|
}
|
|
config.FilesUsedMap[path] = true
|
|
releaseConfigContribution.FlagValues = append(releaseConfigContribution.FlagValues, flagValue)
|
|
return nil
|
|
})
|
|
if err2 != nil {
|
|
return err2
|
|
}
|
|
if releaseConfigContribution.proto.GetAconfigFlagsOnly() {
|
|
config.AconfigFlagsOnly = true
|
|
}
|
|
m.ReleaseConfigContributions[name] = releaseConfigContribution
|
|
config.Contributions = append(config.Contributions, releaseConfigContribution)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
configs.ReleaseConfigMaps = append(configs.ReleaseConfigMaps, m)
|
|
configs.releaseConfigMapsMap[dir] = m
|
|
return nil
|
|
}
|
|
|
|
func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) {
|
|
trace := []string{name}
|
|
for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] {
|
|
name = *target
|
|
trace = append(trace, name)
|
|
}
|
|
if config, ok := configs.ReleaseConfigs[name]; ok {
|
|
return config, nil
|
|
}
|
|
if configs.allowMissing {
|
|
if config, ok := configs.ReleaseConfigs["trunk_staging"]; ok {
|
|
return config, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("Missing config %s. Trace=%v", name, trace)
|
|
}
|
|
|
|
func (configs *ReleaseConfigs) GetAllReleaseNames() []string {
|
|
var allReleaseNames []string
|
|
for _, v := range configs.ReleaseConfigs {
|
|
allReleaseNames = append(allReleaseNames, v.Name)
|
|
allReleaseNames = append(allReleaseNames, v.OtherNames...)
|
|
}
|
|
slices.Sort(allReleaseNames)
|
|
return allReleaseNames
|
|
}
|
|
|
|
func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) error {
|
|
otherNames := make(map[string][]string)
|
|
for aliasName, aliasTarget := range configs.Aliases {
|
|
if _, ok := configs.ReleaseConfigs[aliasName]; ok {
|
|
return fmt.Errorf("Alias %s is a declared release config", aliasName)
|
|
}
|
|
if _, ok := configs.ReleaseConfigs[*aliasTarget]; !ok {
|
|
if _, ok2 := configs.Aliases[*aliasTarget]; !ok2 {
|
|
return fmt.Errorf("Alias %s points to non-existing config %s", aliasName, *aliasTarget)
|
|
}
|
|
}
|
|
otherNames[*aliasTarget] = append(otherNames[*aliasTarget], aliasName)
|
|
}
|
|
for name, aliases := range otherNames {
|
|
configs.ReleaseConfigs[name].OtherNames = aliases
|
|
}
|
|
|
|
sortedReleaseConfigs := configs.GetSortedReleaseConfigs()
|
|
for _, c := range sortedReleaseConfigs {
|
|
err := c.GenerateReleaseConfig(configs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Look for ignored flagging values. Gather the entire list to make it easier to fix them.
|
|
errors := []string{}
|
|
for _, contrib := range configs.ReleaseConfigMaps {
|
|
dirName := filepath.Dir(contrib.path)
|
|
for k, names := range contrib.FlagValueDirs {
|
|
for _, rcName := range names {
|
|
if config, err := configs.GetReleaseConfig(rcName); err == nil {
|
|
rcPath := filepath.Join(dirName, "release_configs", fmt.Sprintf("%s.textproto", config.Name))
|
|
if _, err := os.Stat(rcPath); err != nil {
|
|
errors = append(errors, fmt.Sprintf("%s exists but %s does not contribute to %s",
|
|
filepath.Join(dirName, k, rcName), dirName, config.Name))
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
if len(errors) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(errors, "\n"))
|
|
}
|
|
|
|
releaseConfig, err := configs.GetReleaseConfig(targetRelease)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
orc := []*rc_proto.ReleaseConfigArtifact{}
|
|
for _, c := range sortedReleaseConfigs {
|
|
if c.Name != releaseConfig.Name {
|
|
orc = append(orc, c.ReleaseConfigArtifact)
|
|
}
|
|
}
|
|
|
|
configs.Artifact = rc_proto.ReleaseConfigsArtifact{
|
|
ReleaseConfig: releaseConfig.ReleaseConfigArtifact,
|
|
OtherReleaseConfigs: orc,
|
|
ReleaseConfigMapsMap: func() map[string]*rc_proto.ReleaseConfigMap {
|
|
ret := make(map[string]*rc_proto.ReleaseConfigMap)
|
|
for k, v := range configs.releaseConfigMapsMap {
|
|
ret[k] = &v.proto
|
|
}
|
|
return ret
|
|
}(),
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ReadReleaseConfigMaps(releaseConfigMapPaths StringList, targetRelease string, useBuildVar, allowMissing bool) (*ReleaseConfigs, error) {
|
|
var err error
|
|
|
|
if len(releaseConfigMapPaths) == 0 {
|
|
releaseConfigMapPaths, err = GetDefaultMapPaths(useBuildVar)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(releaseConfigMapPaths) == 0 {
|
|
return nil, fmt.Errorf("No maps found")
|
|
}
|
|
if !useBuildVar {
|
|
warnf("No --map argument provided. Using: --map %s\n", strings.Join(releaseConfigMapPaths, " --map "))
|
|
}
|
|
}
|
|
|
|
configs := ReleaseConfigsFactory()
|
|
configs.allowMissing = allowMissing
|
|
mapsRead := make(map[string]bool)
|
|
var idx int
|
|
for _, releaseConfigMapPath := range releaseConfigMapPaths {
|
|
// Maintain an ordered list of release config directories.
|
|
configDir := filepath.Dir(releaseConfigMapPath)
|
|
if mapsRead[configDir] {
|
|
continue
|
|
}
|
|
mapsRead[configDir] = true
|
|
configs.configDirIndexes[configDir] = idx
|
|
configs.configDirs = append(configs.configDirs, configDir)
|
|
// Force the path to be the textproto path, so that both the scl and textproto formats can coexist.
|
|
releaseConfigMapPath = filepath.Join(configDir, "release_config_map.textproto")
|
|
err = configs.LoadReleaseConfigMap(releaseConfigMapPath, idx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
idx += 1
|
|
}
|
|
|
|
// Now that we have all of the release config maps, can meld them and generate the artifacts.
|
|
err = configs.GenerateReleaseConfigs(targetRelease)
|
|
return configs, err
|
|
}
|