Merge "Handle foreach expressions in mk2rbc"

mk2rbc/expr.go

@@ -31,6 +31,12 @@ type starlarkExpr interface {
 	// Emit the code to copy the expression, otherwise we will end up
 	// with source and target pointing to the same list.
 	emitListVarCopy(gctx *generationContext)
+	// Return the expression, calling the transformer func for
+	// every expression in the tree. If the transformer func returns non-nil,
+	// its result is used in place of the expression it was called with in the
+	// resulting expression. The resulting starlarkExpr will contain as many
+	// of the same objects from the original expression as possible.
+	transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr
 }
 
 func maybeString(expr starlarkExpr) (string, bool) {
@@ -62,6 +68,14 @@ func (s *stringLiteralExpr) emitListVarCopy(gctx *generationContext) {
 	s.emit(gctx)
 }
 
+func (s *stringLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if replacement := transformer(s); replacement != nil {
+		return replacement
+	} else {
+		return s
+	}
+}
+
 // Integer literal
 type intLiteralExpr struct {
 	literal int
@@ -85,6 +99,14 @@ func (s *intLiteralExpr) emitListVarCopy(gctx *generationContext) {
 	s.emit(gctx)
 }
 
+func (s *intLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if replacement := transformer(s); replacement != nil {
+		return replacement
+	} else {
+		return s
+	}
+}
+
 // Boolean literal
 type boolLiteralExpr struct {
 	literal bool
@@ -110,6 +132,14 @@ func (b *boolLiteralExpr) emitListVarCopy(gctx *generationContext) {
 	b.emit(gctx)
 }
 
+func (b *boolLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if replacement := transformer(b); replacement != nil {
+		return replacement
+	} else {
+		return b
+	}
+}
+
 // interpolateExpr represents Starlark's interpolation operator <string> % list
 // we break <string> into a list of chunks, i.e., "first%second%third" % (X, Y)
 // will have chunks = ["first", "second", "third"] and args = [X, Y]
@@ -190,6 +220,19 @@ func (xi *interpolateExpr) emitListVarCopy(gctx *generationContext) {
 	xi.emit(gctx)
 }
 
+func (xi *interpolateExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	argsCopy := make([]starlarkExpr, len(xi.args))
+	for i, arg := range xi.args {
+		argsCopy[i] = arg.transform(transformer)
+	}
+	xi.args = argsCopy
+	if replacement := transformer(xi); replacement != nil {
+		return replacement
+	} else {
+		return xi
+	}
+}
+
 type variableRefExpr struct {
 	ref       variable
 	isDefined bool
@@ -220,6 +263,14 @@ func (v *variableRefExpr) emitListVarCopy(gctx *generationContext) {
 	}
 }
 
+func (v *variableRefExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if replacement := transformer(v); replacement != nil {
+		return replacement
+	} else {
+		return v
+	}
+}
+
 type toStringExpr struct {
 	expr starlarkExpr
 }
@@ -265,6 +316,15 @@ func (s *toStringExpr) emitListVarCopy(gctx *generationContext) {
 	s.emit(gctx)
 }
 
+func (s *toStringExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	s.expr = s.expr.transform(transformer)
+	if replacement := transformer(s); replacement != nil {
+		return replacement
+	} else {
+		return s
+	}
+}
+
 type notExpr struct {
 	expr starlarkExpr
 }
@@ -291,6 +351,15 @@ func (n *notExpr) emitListVarCopy(gctx *generationContext) {
 	n.emit(gctx)
 }
 
+func (n *notExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	n.expr = n.expr.transform(transformer)
+	if replacement := transformer(n); replacement != nil {
+		return replacement
+	} else {
+		return n
+	}
+}
+
 type eqExpr struct {
 	left, right starlarkExpr
 	isEq        bool // if false, it's !=
@@ -360,6 +429,16 @@ func (eq *eqExpr) emitListVarCopy(gctx *generationContext) {
 	eq.emit(gctx)
 }
 
+func (eq *eqExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	eq.left = eq.left.transform(transformer)
+	eq.right = eq.right.transform(transformer)
+	if replacement := transformer(eq); replacement != nil {
+		return replacement
+	} else {
+		return eq
+	}
+}
+
 // variableDefinedExpr corresponds to Make's ifdef VAR
 type variableDefinedExpr struct {
 	v variable
@@ -388,6 +467,11 @@ func (v *variableDefinedExpr) emitListVarCopy(gctx *generationContext) {
 	v.emit(gctx)
 }
 
+func (v *variableDefinedExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	// TODO: VariableDefinedExpr isn't really an expression?
+	return v
+}
+
 type listExpr struct {
 	items []starlarkExpr
 }
@@ -442,6 +526,19 @@ func (l *listExpr) emitListVarCopy(gctx *generationContext) {
 	l.emit(gctx)
 }
 
+func (l *listExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	itemsCopy := make([]starlarkExpr, len(l.items))
+	for i, item := range l.items {
+		itemsCopy[i] = item.transform(transformer)
+	}
+	l.items = itemsCopy
+	if replacement := transformer(l); replacement != nil {
+		return replacement
+	} else {
+		return l
+	}
+}
+
 func newStringListExpr(items []string) *listExpr {
 	v := listExpr{}
 	for _, item := range items {
@@ -505,6 +602,19 @@ func (c *concatExpr) emitListVarCopy(gctx *generationContext) {
 	c.emit(gctx)
 }
 
+func (c *concatExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	itemsCopy := make([]starlarkExpr, len(c.items))
+	for i, item := range c.items {
+		itemsCopy[i] = item.transform(transformer)
+	}
+	c.items = itemsCopy
+	if replacement := transformer(c); replacement != nil {
+		return replacement
+	} else {
+		return c
+	}
+}
+
 // inExpr generates <expr> [not] in <list>
 type inExpr struct {
 	expr  starlarkExpr
@@ -543,23 +653,33 @@ func (i *inExpr) emitListVarCopy(gctx *generationContext) {
 	i.emit(gctx)
 }
 
+func (i *inExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	i.expr = i.expr.transform(transformer)
+	i.list = i.list.transform(transformer)
+	if replacement := transformer(i); replacement != nil {
+		return replacement
+	} else {
+		return i
+	}
+}
+
 type indexExpr struct {
 	array starlarkExpr
 	index starlarkExpr
 }
 
-func (ix indexExpr) emit(gctx *generationContext) {
+func (ix *indexExpr) emit(gctx *generationContext) {
 	ix.array.emit(gctx)
 	gctx.write("[")
 	ix.index.emit(gctx)
 	gctx.write("]")
 }
 
-func (ix indexExpr) typ() starlarkType {
+func (ix *indexExpr) typ() starlarkType {
 	return starlarkTypeString
 }
 
-func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+func (ix *indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
 	newArray, isSameArray := ix.array.eval(valueMap)
 	newIndex, isSameIndex := ix.index.eval(valueMap)
 	if same = isSameArray && isSameIndex; same {
@@ -570,10 +690,20 @@ func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, sa
 	return
 }
 
-func (ix indexExpr) emitListVarCopy(gctx *generationContext) {
+func (ix *indexExpr) emitListVarCopy(gctx *generationContext) {
 	ix.emit(gctx)
 }
 
+func (ix *indexExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	ix.array = ix.array.transform(transformer)
+	ix.index = ix.index.transform(transformer)
+	if replacement := transformer(ix); replacement != nil {
+		return replacement
+	} else {
+		return ix
+	}
+}
+
 type callExpr struct {
 	object     starlarkExpr // nil if static call
 	name       string
@@ -642,6 +772,21 @@ func (cx *callExpr) emitListVarCopy(gctx *generationContext) {
 	cx.emit(gctx)
 }
 
+func (cx *callExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if cx.object != nil {
+		cx.object = cx.object.transform(transformer)
+	}
+	argsCopy := make([]starlarkExpr, len(cx.args))
+	for i, arg := range cx.args {
+		argsCopy[i] = arg.transform(transformer)
+	}
+	if replacement := transformer(cx); replacement != nil {
+		return replacement
+	} else {
+		return cx
+	}
+}
+
 type ifExpr struct {
 	condition starlarkExpr
 	ifTrue    starlarkExpr
@@ -691,6 +836,92 @@ func (i *ifExpr) emitListVarCopy(gctx *generationContext) {
 	i.emit(gctx)
 }
 
+func (i *ifExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	i.condition = i.condition.transform(transformer)
+	i.ifTrue = i.ifTrue.transform(transformer)
+	i.ifFalse = i.ifFalse.transform(transformer)
+	if replacement := transformer(i); replacement != nil {
+		return replacement
+	} else {
+		return i
+	}
+}
+
+type identifierExpr struct {
+	name string
+}
+
+func (i *identifierExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	return i, true
+}
+
+func (i *identifierExpr) emit(gctx *generationContext) {
+	gctx.write(i.name)
+}
+
+func (i *identifierExpr) typ() starlarkType {
+	return starlarkTypeUnknown
+}
+
+func (i *identifierExpr) emitListVarCopy(gctx *generationContext) {
+	i.emit(gctx)
+}
+
+func (i *identifierExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if replacement := transformer(i); replacement != nil {
+		return replacement
+	} else {
+		return i
+	}
+}
+
+type foreachExpr struct {
+	varName string
+	list    starlarkExpr
+	action  starlarkExpr
+}
+
+func (f *foreachExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	list, listSame := f.list.eval(valueMap)
+	action, actionSame := f.action.eval(valueMap)
+	same = listSame && actionSame
+	if same {
+		return f, same
+	} else {
+		return &foreachExpr{
+			varName: f.varName,
+			list:    list,
+			action:  action,
+		}, same
+	}
+}
+
+func (f *foreachExpr) emit(gctx *generationContext) {
+	gctx.write("[")
+	f.action.emit(gctx)
+	gctx.write(" for " + f.varName + " in ")
+	f.list.emit(gctx)
+	gctx.write("]")
+}
+
+func (f *foreachExpr) typ() starlarkType {
+	return starlarkTypeList
+}
+
+func (f *foreachExpr) emitListVarCopy(gctx *generationContext) {
+	f.emit(gctx)
+}
+
+func (f *foreachExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	f.list = f.list.transform(transformer)
+	f.action = f.action.transform(transformer)
+	if replacement := transformer(f); replacement != nil {
+		return replacement
+	} else {
+		return f
+	}
+}
+
 type badExpr struct {
 	errorLocation ErrorLocation
 	message       string
@@ -714,6 +945,14 @@ func (_ *badExpr) emitListVarCopy(_ *generationContext) {
 	panic("implement me")
 }
 
+func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
+	if replacement := transformer(b); replacement != nil {
+		return replacement
+	} else {
+		return b
+	}
+}
+
 func maybeConvertToStringList(expr starlarkExpr) starlarkExpr {
 	if xString, ok := expr.(*stringLiteralExpr); ok {
 		return newStringListExpr(strings.Fields(xString.literal))

mk2rbc/mk2rbc.go

@@ -111,6 +111,7 @@ var knownFunctions = map[string]struct {
 	"filter":                              {baseName + ".filter", starlarkTypeList, hiddenArgNone},
 	"filter-out":                          {baseName + ".filter_out", starlarkTypeList, hiddenArgNone},
 	"firstword":                           {"!firstword", starlarkTypeString, hiddenArgNone},
+	"foreach":                             {"!foreach", starlarkTypeList, hiddenArgNone},
 	"get-vendor-board-platforms":          {"!get-vendor-board-platforms", starlarkTypeList, hiddenArgNone}, // internal macro, used by is-board-platform, etc.
 	"if":                                  {"!if", starlarkTypeUnknown, hiddenArgNone},
 	"info":                                {baseName + ".mkinfo", starlarkTypeVoid, hiddenArgNone},
@@ -147,14 +148,10 @@ var knownFunctions = map[string]struct {
 	"warning":    {baseName + ".mkwarning", starlarkTypeVoid, hiddenArgNone},
 	"word":       {baseName + "!word", starlarkTypeString, hiddenArgNone},
 	"wildcard":   {baseName + ".expand_wildcard", starlarkTypeList, hiddenArgNone},
+	"words":      {baseName + ".words", starlarkTypeList, hiddenArgNone},
 }
 
-var builtinFuncRex = regexp.MustCompile(
+var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
-	"^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" +
-		"|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" +
-		"|if|info|join|lastword|notdir|or|origin|patsubst|realpath" +
-		"|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" +
-		"|wildcard)")
 
 // Conversion request parameters
 type Request struct {
@@ -1412,6 +1409,8 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt
 	switch expr.name {
 	case "if":
 		return ctx.parseIfFunc(node, args)
+	case "foreach":
+		return ctx.parseForeachFunc(node, args)
 	case "word":
 		return ctx.parseWordFunc(node, args)
 	case "firstword", "lastword":
@@ -1496,6 +1495,38 @@ func (ctx *parseContext) parseIfFunc(node mkparser.Node, args *mkparser.MakeStri
 	}
 }
 
+func (ctx *parseContext) parseForeachFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
+	words := args.Split(",")
+	if len(words) != 3 {
+		return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
+	}
+	if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
+		return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier")
+	}
+	loopVarName := words[0].Strings[0]
+	list := ctx.parseMakeString(node, words[1])
+	action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr {
+		if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
+			return &identifierExpr{loopVarName}
+		}
+		return nil
+	})
+
+	if list.typ() != starlarkTypeList {
+		list = &callExpr{
+			name:       "words",
+			returnType: knownFunctions["words"].returnType,
+			args:       []starlarkExpr{list},
+		}
+	}
+
+	return &foreachExpr{
+		varName: loopVarName,
+		list:    list,
+		action:  action,
+	}
+}
+
 func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
 	words := args.Split(",")
 	if len(words) != 2 {
@@ -1517,7 +1548,7 @@ func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeSt
 	if array.typ() != starlarkTypeList {
 		array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
 	}
-	return indexExpr{array, &intLiteralExpr{int(index - 1)}}
+	return &indexExpr{array, &intLiteralExpr{int(index - 1)}}
 }
 
 func (ctx *parseContext) parseFirstOrLastwordFunc(node mkparser.Node, name string, args *mkparser.MakeString) starlarkExpr {

mk2rbc/mk2rbc_test.go

@@ -1156,6 +1156,28 @@ def init(g, handle):
   g["SOURCES"] = "foo.c bar.c"
   g["OBJECTS"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"])
   g["OBJECTS2"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"])
+`,
+	},
+	{
+		desc:   "foreach expressions",
+		mkname: "product.mk",
+		in: `
+BOOT_KERNEL_MODULES := foo.ko bar.ko
+BOOT_KERNEL_MODULES_FILTER := $(foreach m,$(BOOT_KERNEL_MODULES),%/$(m))
+BOOT_KERNEL_MODULES_LIST := foo.ko
+BOOT_KERNEL_MODULES_LIST += bar.ko
+BOOT_KERNEL_MODULES_FILTER_2 := $(foreach m,$(BOOT_KERNEL_MODULES_LIST),%/$(m))
+
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  g["BOOT_KERNEL_MODULES"] = "foo.ko bar.ko"
+  g["BOOT_KERNEL_MODULES_FILTER"] = ["%%/%s" % m for m in rblf.words(g["BOOT_KERNEL_MODULES"])]
+  g["BOOT_KERNEL_MODULES_LIST"] = ["foo.ko"]
+  g["BOOT_KERNEL_MODULES_LIST"] += ["bar.ko"]
+  g["BOOT_KERNEL_MODULES_FILTER_2"] = ["%%/%s" % m for m in g["BOOT_KERNEL_MODULES_LIST"]]
 `,
 	},
 }