Merge changes Ibb6c4aed,I54af1654

* changes:
  AndroidProducts.mk file reader
  Soong variables reader

mk2rbc/android_products.go

@@ -0,0 +1,110 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	mkparser "android/soong/androidmk/parser"
+)
+
+// Implements mkparser.Scope, to be used by mkparser.Value.Value()
+type localDirEval struct {
+	localDir  string
+	hasErrors bool
+}
+
+func (l *localDirEval) Get(name string) string {
+	if name == "LOCAL_DIR" {
+		return l.localDir
+	}
+	l.hasErrors = true
+	return fmt.Sprintf("$(%s)", name)
+}
+
+func (l *localDirEval) Set(_, _ string) {
+}
+
+func (l *localDirEval) Call(_ string, _ []string) []string {
+	l.hasErrors = true
+	return []string{"$(call ...)"}
+}
+
+func (l *localDirEval) SetFunc(_ string, _ func([]string) []string) {
+}
+
+// UpdateProductConfigMap builds product configuration map.
+// The product configuration map maps a product name (i.e., the value of the
+// TARGET_PRODUCT variable) to the top-level configuration file.
+// In the Android's Make-based build machinery, the equivalent of the
+// product configuration map is $(PRODUCT_MAKEFILES), which is the list
+// of <product>:<configuration makefile> pairs (if <product>: is missing,
+// <product> is the basename of the configuration makefile).
+// UpdateProductConfigMap emulates this build logic by processing the
+// assignments to PRODUCT_MAKEFILES in the file passed to it.
+func UpdateProductConfigMap(configMap map[string]string, configMakefile string) error {
+	contents, err := ioutil.ReadFile(configMakefile)
+	if err != nil {
+		return err
+	}
+	parser := mkparser.NewParser(configMakefile, bytes.NewBuffer(contents))
+	nodes, errs := parser.Parse()
+	if len(errs) > 0 {
+		for _, e := range errs {
+			fmt.Fprintln(os.Stderr, "ERROR:", e)
+		}
+		return fmt.Errorf("cannot parse %s", configMakefile)
+	}
+
+	ldEval := &localDirEval{localDir: filepath.Dir(configMakefile)}
+
+	for _, node := range nodes {
+		// We are interested in assignments to 'PRODUCT_MAKEFILES'
+		asgn, ok := node.(*mkparser.Assignment)
+		if !ok {
+			continue
+		}
+		if !(asgn.Name.Const() && asgn.Name.Strings[0] == "PRODUCT_MAKEFILES") {
+			continue
+		}
+
+		// Resolve the references to $(LOCAL_DIR) in $(PRODUCT_MAKEFILES).
+		ldEval.hasErrors = false
+		value := asgn.Value.Value(ldEval)
+		if ldEval.hasErrors {
+			return fmt.Errorf("cannot evaluate %s", asgn.Value.Dump())
+		}
+		// Each item is either <product>:<configuration makefile>, or
+		// just <configuration makefile>
+		for _, token := range strings.Fields(value) {
+			var product, config_path string
+			if n := strings.Index(token, ":"); n >= 0 {
+				product = token[0:n]
+				config_path = token[n+1:]
+			} else {
+				config_path = token
+				product = filepath.Base(config_path)
+				product = strings.TrimSuffix(product, filepath.Ext(product))
+			}
+			configMap[product] = config_path
+		}
+	}
+	return nil
+}

mk2rbc/android_products_test.go

@@ -0,0 +1,38 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+func TestProductsMakefile(t *testing.T) {
+	testDir := getTestDirectory()
+	abspath := func(relPath string) string { return filepath.Join(testDir, relPath) }
+	actualProducts := make(map[string]string)
+	if err := UpdateProductConfigMap(actualProducts, abspath("android_products.mk.test")); err != nil {
+		t.Fatal(err)
+	}
+	expectedProducts := map[string]string{
+		"aosp_cf_x86_tv": abspath("vsoc_x86/tv/device.mk"),
+		"aosp_tv_arm":    abspath("aosp_tv_arm.mk"),
+		"aosp_tv_arm64":  abspath("aosp_tv_arm64.mk"),
+	}
+	if !reflect.DeepEqual(actualProducts, expectedProducts) {
+		t.Errorf("\nExpected: %v\n  Actual: %v", expectedProducts, actualProducts)
+	}
+}

mk2rbc/soong_variables.go

@@ -0,0 +1,151 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+
+	mkparser "android/soong/androidmk/parser"
+)
+
+type context struct {
+	includeFileScope mkparser.Scope
+	registrar        variableRegistrar
+}
+
+// Scans the makefile Soong uses to generate soong.variables file,
+// collecting variable names and types from the lines that look like this:
+//    $(call add_json_XXX,  <...>,             $(VAR))
+//
+func FindSoongVariables(mkFile string, includeFileScope mkparser.Scope, registrar variableRegistrar) error {
+	ctx := context{includeFileScope, registrar}
+	return ctx.doFind(mkFile)
+}
+
+func (ctx *context) doFind(mkFile string) error {
+	mkContents, err := ioutil.ReadFile(mkFile)
+	if err != nil {
+		return err
+	}
+	parser := mkparser.NewParser(mkFile, bytes.NewBuffer(mkContents))
+	nodes, errs := parser.Parse()
+	if len(errs) > 0 {
+		for _, e := range errs {
+			fmt.Fprintln(os.Stderr, "ERROR:", e)
+		}
+		return fmt.Errorf("cannot parse %s", mkFile)
+	}
+	for _, node := range nodes {
+		switch t := node.(type) {
+		case *mkparser.Variable:
+			ctx.handleVariable(t)
+		case *mkparser.Directive:
+			ctx.handleInclude(t)
+		}
+	}
+	return nil
+}
+
+func (ctx context) NewSoongVariable(name, typeString string) {
+	var valueType starlarkType
+	switch typeString {
+	case "bool":
+		valueType = starlarkTypeBool
+	case "csv":
+		// Only PLATFORM_VERSION_ALL_CODENAMES, and it's a list
+		valueType = starlarkTypeList
+	case "list":
+		valueType = starlarkTypeList
+	case "str":
+		valueType = starlarkTypeString
+	case "val":
+		// Only PLATFORM_SDK_VERSION uses this, and it's integer
+		valueType = starlarkTypeInt
+	default:
+		panic(fmt.Errorf("unknown Soong variable type %s", typeString))
+	}
+
+	ctx.registrar.NewVariable(name, VarClassSoong, valueType)
+}
+
+func (ctx context) handleInclude(t *mkparser.Directive) {
+	if t.Name != "include" && t.Name != "-include" {
+		return
+	}
+	includedPath := t.Args.Value(ctx.includeFileScope)
+	err := ctx.doFind(includedPath)
+	if err != nil && t.Name == "include" {
+		fmt.Fprintf(os.Stderr, "cannot include %s: %s", includedPath, err)
+	}
+}
+
+var callFuncRex = regexp.MustCompile("^call +add_json_(str|val|bool|csv|list) *,")
+
+func (ctx context) handleVariable(t *mkparser.Variable) {
+	// From the variable reference looking as follows:
+	//  $(call json_add_TYPE,arg1,$(VAR))
+	// we infer that the type of $(VAR) is TYPE
+	// VAR can be a simple variable name, or another call
+	// (e.g., $(call invert_bool, $(X)), from which we can infer
+	// that the type of X is bool
+	if prefix, v, ok := prefixedVariable(t.Name); ok && strings.HasPrefix(prefix, "call add_json") {
+		if match := callFuncRex.FindStringSubmatch(prefix); match != nil {
+			ctx.inferSoongVariableType(match[1], v)
+			// NOTE(asmundak): sometimes arg1 (the name of the Soong variable defined
+			// in this statement) may indicate that there is a Make counterpart. E.g, from
+			//     $(call add_json_bool, DisablePreopt, $(call invert_bool,$(ENABLE_PREOPT)))
+			// it may be inferred that there is a Make boolean variable DISABLE_PREOPT.
+			// Unfortunately, Soong variable names have no 1:1 correspondence to Make variables,
+			// for instance,
+			//       $(call add_json_list, PatternsOnSystemOther, $(SYSTEM_OTHER_ODEX_FILTER))
+			// does not mean that there is PATTERNS_ON_SYSTEM_OTHER
+			// Our main interest lies in finding the variables whose values are lists, and
+			// so far there are none that can be found this way, so it is not important.
+		} else {
+			panic(fmt.Errorf("cannot match the call: %s", prefix))
+		}
+	}
+}
+
+var (
+	callInvertBoolRex = regexp.MustCompile("^call +invert_bool *, *$")
+	callFilterBoolRex = regexp.MustCompile("^(filter|filter-out) +(true|false), *$")
+)
+
+func (ctx context) inferSoongVariableType(vType string, n *mkparser.MakeString) {
+	if n.Const() {
+		ctx.NewSoongVariable(n.Strings[0], vType)
+		return
+	}
+	if prefix, v, ok := prefixedVariable(n); ok {
+		if callInvertBoolRex.MatchString(prefix) || callFilterBoolRex.MatchString(prefix) {
+			// It is $(call invert_bool, $(VAR)) or $(filter[-out] [false|true],$(VAR))
+			ctx.inferSoongVariableType("bool", v)
+		}
+	}
+}
+
+// If MakeString is foo$(BAR), returns 'foo', BAR(as *MakeString) and true
+func prefixedVariable(s *mkparser.MakeString) (string, *mkparser.MakeString, bool) {
+	if len(s.Strings) != 2 || s.Strings[1] != "" {
+		return "", nil, false
+	}
+	return s.Strings[0], s.Variables[0].Name, true
+}

mk2rbc/soong_variables_test.go

@@ -0,0 +1,51 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+type dirResolverForTest struct {
+	ScopeBase
+}
+
+func (t dirResolverForTest) Get(name string) string {
+	if name != "BUILD_SYSTEM" {
+		return fmt.Sprintf("$(%s)", name)
+	}
+	return getTestDirectory()
+}
+
+func TestSoongVariables(t *testing.T) {
+	testFile := filepath.Join(getTestDirectory(), "soong_variables.mk.test")
+	var actual testVariables
+	if err := FindSoongVariables(testFile, dirResolverForTest{}, &actual); err != nil {
+		t.Fatal(err)
+	}
+	expected := testVariables{[]testVar{
+		{"BUILD_ID", VarClassSoong, starlarkTypeString},
+		{"PLATFORM_SDK_VERSION", VarClassSoong, starlarkTypeInt},
+		{"DEVICE_PACKAGE_OVERLAYS", VarClassSoong, starlarkTypeList},
+		{"ENABLE_CFI", VarClassSoong, starlarkTypeBool},
+		{"ENABLE_PREOPT", VarClassSoong, starlarkTypeBool},
+	}}
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("\nExpected: %v\n  Actual: %v", expected, actual)
+	}
+}

mk2rbc/test/android_products.mk.test

@@ -0,0 +1,4 @@
+PRODUCT_MAKEFILES := \
+    $(LOCAL_DIR)/aosp_tv_arm.mk \
+    $(LOCAL_DIR)/aosp_tv_arm64.mk \
+    aosp_cf_x86_tv:$(LOCAL_DIR)/vsoc_x86/tv/device.mk

mk2rbc/test/soong_included.mk.test

@@ -0,0 +1 @@
+  $(call add_json_bool, DisablePreopt,                           $(call invert_bool,$(ENABLE_PREOPT)))

mk2rbc/test/soong_variables.mk.test

@@ -0,0 +1,5 @@
+$(call add_json_str,  BuildId,                           $(BUILD_ID))
+$(call add_json_val,  Platform_sdk_version,              $(PLATFORM_SDK_VERSION))
+$(call add_json_list, DeviceResourceOverlays,            $(DEVICE_PACKAGE_OVERLAYS))
+$(call add_json_bool, EnableCFI,                         $(call invert_bool,$(filter false,$(ENABLE_CFI))))
+include $(BUILD_SYSTEM)/soong_included.mk.test