From b0d32ab960517c1031c6841f31a7f3f5c44c0b33 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 9 Dec 2021 14:00:59 -0800 Subject: [PATCH] Handle foreach expressions in mk2rbc Bug: 201700692 Test: go test Change-Id: Ia23494a63567a1fe2d4bb797a2d4dd5925b3431d --- mk2rbc/expr.go | 247 +++++++++++++++++++++++++++++++++++++++++- mk2rbc/mk2rbc.go | 45 ++++++-- mk2rbc/mk2rbc_test.go | 22 ++++ 3 files changed, 303 insertions(+), 11 deletions(-) diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go index 81b31c736..ca48bd982 100644 --- a/mk2rbc/expr.go +++ b/mk2rbc/expr.go @@ -31,6 +31,12 @@ type starlarkExpr interface { // Emit the code to copy the expression, otherwise we will end up // with source and target pointing to the same list. emitListVarCopy(gctx *generationContext) + // Return the expression, calling the transformer func for + // every expression in the tree. If the transformer func returns non-nil, + // its result is used in place of the expression it was called with in the + // resulting expression. The resulting starlarkExpr will contain as many + // of the same objects from the original expression as possible. + transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr } func maybeString(expr starlarkExpr) (string, bool) { @@ -62,6 +68,14 @@ func (s *stringLiteralExpr) emitListVarCopy(gctx *generationContext) { s.emit(gctx) } +func (s *stringLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if replacement := transformer(s); replacement != nil { + return replacement + } else { + return s + } +} + // Integer literal type intLiteralExpr struct { literal int @@ -85,6 +99,14 @@ func (s *intLiteralExpr) emitListVarCopy(gctx *generationContext) { s.emit(gctx) } +func (s *intLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if replacement := transformer(s); replacement != nil { + return replacement + } else { + return s + } +} + // Boolean literal type boolLiteralExpr struct { literal bool @@ -110,6 +132,14 @@ func (b *boolLiteralExpr) emitListVarCopy(gctx *generationContext) { b.emit(gctx) } +func (b *boolLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if replacement := transformer(b); replacement != nil { + return replacement + } else { + return b + } +} + // interpolateExpr represents Starlark's interpolation operator % list // we break into a list of chunks, i.e., "first%second%third" % (X, Y) // will have chunks = ["first", "second", "third"] and args = [X, Y] @@ -190,6 +220,19 @@ func (xi *interpolateExpr) emitListVarCopy(gctx *generationContext) { xi.emit(gctx) } +func (xi *interpolateExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + argsCopy := make([]starlarkExpr, len(xi.args)) + for i, arg := range xi.args { + argsCopy[i] = arg.transform(transformer) + } + xi.args = argsCopy + if replacement := transformer(xi); replacement != nil { + return replacement + } else { + return xi + } +} + type variableRefExpr struct { ref variable isDefined bool @@ -220,6 +263,14 @@ func (v *variableRefExpr) emitListVarCopy(gctx *generationContext) { } } +func (v *variableRefExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if replacement := transformer(v); replacement != nil { + return replacement + } else { + return v + } +} + type toStringExpr struct { expr starlarkExpr } @@ -265,6 +316,15 @@ func (s *toStringExpr) emitListVarCopy(gctx *generationContext) { s.emit(gctx) } +func (s *toStringExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + s.expr = s.expr.transform(transformer) + if replacement := transformer(s); replacement != nil { + return replacement + } else { + return s + } +} + type notExpr struct { expr starlarkExpr } @@ -291,6 +351,15 @@ func (n *notExpr) emitListVarCopy(gctx *generationContext) { n.emit(gctx) } +func (n *notExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + n.expr = n.expr.transform(transformer) + if replacement := transformer(n); replacement != nil { + return replacement + } else { + return n + } +} + type eqExpr struct { left, right starlarkExpr isEq bool // if false, it's != @@ -360,6 +429,16 @@ func (eq *eqExpr) emitListVarCopy(gctx *generationContext) { eq.emit(gctx) } +func (eq *eqExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + eq.left = eq.left.transform(transformer) + eq.right = eq.right.transform(transformer) + if replacement := transformer(eq); replacement != nil { + return replacement + } else { + return eq + } +} + // variableDefinedExpr corresponds to Make's ifdef VAR type variableDefinedExpr struct { v variable @@ -388,6 +467,11 @@ func (v *variableDefinedExpr) emitListVarCopy(gctx *generationContext) { v.emit(gctx) } +func (v *variableDefinedExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + // TODO: VariableDefinedExpr isn't really an expression? + return v +} + type listExpr struct { items []starlarkExpr } @@ -442,6 +526,19 @@ func (l *listExpr) emitListVarCopy(gctx *generationContext) { l.emit(gctx) } +func (l *listExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + itemsCopy := make([]starlarkExpr, len(l.items)) + for i, item := range l.items { + itemsCopy[i] = item.transform(transformer) + } + l.items = itemsCopy + if replacement := transformer(l); replacement != nil { + return replacement + } else { + return l + } +} + func newStringListExpr(items []string) *listExpr { v := listExpr{} for _, item := range items { @@ -505,6 +602,19 @@ func (c *concatExpr) emitListVarCopy(gctx *generationContext) { c.emit(gctx) } +func (c *concatExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + itemsCopy := make([]starlarkExpr, len(c.items)) + for i, item := range c.items { + itemsCopy[i] = item.transform(transformer) + } + c.items = itemsCopy + if replacement := transformer(c); replacement != nil { + return replacement + } else { + return c + } +} + // inExpr generates [not] in type inExpr struct { expr starlarkExpr @@ -543,23 +653,33 @@ func (i *inExpr) emitListVarCopy(gctx *generationContext) { i.emit(gctx) } +func (i *inExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + i.expr = i.expr.transform(transformer) + i.list = i.list.transform(transformer) + if replacement := transformer(i); replacement != nil { + return replacement + } else { + return i + } +} + type indexExpr struct { array starlarkExpr index starlarkExpr } -func (ix indexExpr) emit(gctx *generationContext) { +func (ix *indexExpr) emit(gctx *generationContext) { ix.array.emit(gctx) gctx.write("[") ix.index.emit(gctx) gctx.write("]") } -func (ix indexExpr) typ() starlarkType { +func (ix *indexExpr) typ() starlarkType { return starlarkTypeString } -func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { +func (ix *indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { newArray, isSameArray := ix.array.eval(valueMap) newIndex, isSameIndex := ix.index.eval(valueMap) if same = isSameArray && isSameIndex; same { @@ -570,10 +690,20 @@ func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, sa return } -func (ix indexExpr) emitListVarCopy(gctx *generationContext) { +func (ix *indexExpr) emitListVarCopy(gctx *generationContext) { ix.emit(gctx) } +func (ix *indexExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + ix.array = ix.array.transform(transformer) + ix.index = ix.index.transform(transformer) + if replacement := transformer(ix); replacement != nil { + return replacement + } else { + return ix + } +} + type callExpr struct { object starlarkExpr // nil if static call name string @@ -642,6 +772,21 @@ func (cx *callExpr) emitListVarCopy(gctx *generationContext) { cx.emit(gctx) } +func (cx *callExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if cx.object != nil { + cx.object = cx.object.transform(transformer) + } + argsCopy := make([]starlarkExpr, len(cx.args)) + for i, arg := range cx.args { + argsCopy[i] = arg.transform(transformer) + } + if replacement := transformer(cx); replacement != nil { + return replacement + } else { + return cx + } +} + type ifExpr struct { condition starlarkExpr ifTrue starlarkExpr @@ -691,6 +836,92 @@ func (i *ifExpr) emitListVarCopy(gctx *generationContext) { i.emit(gctx) } +func (i *ifExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + i.condition = i.condition.transform(transformer) + i.ifTrue = i.ifTrue.transform(transformer) + i.ifFalse = i.ifFalse.transform(transformer) + if replacement := transformer(i); replacement != nil { + return replacement + } else { + return i + } +} + +type identifierExpr struct { + name string +} + +func (i *identifierExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + return i, true +} + +func (i *identifierExpr) emit(gctx *generationContext) { + gctx.write(i.name) +} + +func (i *identifierExpr) typ() starlarkType { + return starlarkTypeUnknown +} + +func (i *identifierExpr) emitListVarCopy(gctx *generationContext) { + i.emit(gctx) +} + +func (i *identifierExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if replacement := transformer(i); replacement != nil { + return replacement + } else { + return i + } +} + +type foreachExpr struct { + varName string + list starlarkExpr + action starlarkExpr +} + +func (f *foreachExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) { + list, listSame := f.list.eval(valueMap) + action, actionSame := f.action.eval(valueMap) + same = listSame && actionSame + if same { + return f, same + } else { + return &foreachExpr{ + varName: f.varName, + list: list, + action: action, + }, same + } +} + +func (f *foreachExpr) emit(gctx *generationContext) { + gctx.write("[") + f.action.emit(gctx) + gctx.write(" for " + f.varName + " in ") + f.list.emit(gctx) + gctx.write("]") +} + +func (f *foreachExpr) typ() starlarkType { + return starlarkTypeList +} + +func (f *foreachExpr) emitListVarCopy(gctx *generationContext) { + f.emit(gctx) +} + +func (f *foreachExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + f.list = f.list.transform(transformer) + f.action = f.action.transform(transformer) + if replacement := transformer(f); replacement != nil { + return replacement + } else { + return f + } +} + type badExpr struct { errorLocation ErrorLocation message string @@ -714,6 +945,14 @@ func (_ *badExpr) emitListVarCopy(_ *generationContext) { panic("implement me") } +func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { + if replacement := transformer(b); replacement != nil { + return replacement + } else { + return b + } +} + func maybeConvertToStringList(expr starlarkExpr) starlarkExpr { if xString, ok := expr.(*stringLiteralExpr); ok { return newStringListExpr(strings.Fields(xString.literal)) diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index d5ff18105..11d3982da 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -111,6 +111,7 @@ var knownFunctions = map[string]struct { "filter": {baseName + ".filter", starlarkTypeList, hiddenArgNone}, "filter-out": {baseName + ".filter_out", starlarkTypeList, hiddenArgNone}, "firstword": {"!firstword", starlarkTypeString, hiddenArgNone}, + "foreach": {"!foreach", starlarkTypeList, hiddenArgNone}, "get-vendor-board-platforms": {"!get-vendor-board-platforms", starlarkTypeList, hiddenArgNone}, // internal macro, used by is-board-platform, etc. "if": {"!if", starlarkTypeUnknown, hiddenArgNone}, "info": {baseName + ".mkinfo", starlarkTypeVoid, hiddenArgNone}, @@ -147,14 +148,10 @@ var knownFunctions = map[string]struct { "warning": {baseName + ".mkwarning", starlarkTypeVoid, hiddenArgNone}, "word": {baseName + "!word", starlarkTypeString, hiddenArgNone}, "wildcard": {baseName + ".expand_wildcard", starlarkTypeList, hiddenArgNone}, + "words": {baseName + ".words", starlarkTypeList, hiddenArgNone}, } -var builtinFuncRex = regexp.MustCompile( - "^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" + - "|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" + - "|if|info|join|lastword|notdir|or|origin|patsubst|realpath" + - "|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" + - "|wildcard)") +var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") // Conversion request parameters type Request struct { @@ -1399,6 +1396,8 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt switch expr.name { case "if": return ctx.parseIfFunc(node, args) + case "foreach": + return ctx.parseForeachFunc(node, args) case "word": return ctx.parseWordFunc(node, args) case "firstword", "lastword": @@ -1483,6 +1482,38 @@ func (ctx *parseContext) parseIfFunc(node mkparser.Node, args *mkparser.MakeStri } } +func (ctx *parseContext) parseForeachFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr { + words := args.Split(",") + if len(words) != 3 { + return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words))) + } + if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) { + return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier") + } + loopVarName := words[0].Strings[0] + list := ctx.parseMakeString(node, words[1]) + action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr { + if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName { + return &identifierExpr{loopVarName} + } + return nil + }) + + if list.typ() != starlarkTypeList { + list = &callExpr{ + name: "words", + returnType: knownFunctions["words"].returnType, + args: []starlarkExpr{list}, + } + } + + return &foreachExpr{ + varName: loopVarName, + list: list, + action: action, + } +} + func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr { words := args.Split(",") if len(words) != 2 { @@ -1504,7 +1535,7 @@ func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeSt if array.typ() != starlarkTypeList { array = &callExpr{object: array, name: "split", returnType: starlarkTypeList} } - return indexExpr{array, &intLiteralExpr{int(index - 1)}} + return &indexExpr{array, &intLiteralExpr{int(index - 1)}} } func (ctx *parseContext) parseFirstOrLastwordFunc(node mkparser.Node, name string, args *mkparser.MakeString) starlarkExpr { diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index 78444c9c6..e1bb2bbee 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -1129,6 +1129,28 @@ def init(g, handle): g["SOURCES"] = "foo.c bar.c" g["OBJECTS"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"]) g["OBJECTS2"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"]) +`, + }, + { + desc: "foreach expressions", + mkname: "product.mk", + in: ` +BOOT_KERNEL_MODULES := foo.ko bar.ko +BOOT_KERNEL_MODULES_FILTER := $(foreach m,$(BOOT_KERNEL_MODULES),%/$(m)) +BOOT_KERNEL_MODULES_LIST := foo.ko +BOOT_KERNEL_MODULES_LIST += bar.ko +BOOT_KERNEL_MODULES_FILTER_2 := $(foreach m,$(BOOT_KERNEL_MODULES_LIST),%/$(m)) + +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["BOOT_KERNEL_MODULES"] = "foo.ko bar.ko" + g["BOOT_KERNEL_MODULES_FILTER"] = ["%%/%s" % m for m in rblf.words(g["BOOT_KERNEL_MODULES"])] + g["BOOT_KERNEL_MODULES_LIST"] = ["foo.ko"] + g["BOOT_KERNEL_MODULES_LIST"] += ["bar.ko"] + g["BOOT_KERNEL_MODULES_FILTER_2"] = ["%%/%s" % m for m in g["BOOT_KERNEL_MODULES_LIST"]] `, }, }