StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle foreach expressions in mk2rbc

Browse Source 
Bug: 201700692
Test: go test
Change-Id: Ia23494a63567a1fe2d4bb797a2d4dd5925b3431d
This commit is contained in:
Cole Faust
2021-12-09 14:00:59 -08:00
parent 789e5026b4
commit b0d32ab960
3 changed files with 303 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
mk2rbc/expr.go
View File

@@ -31,6 +31,12 @@ type starlarkExpr interface {
// Emit the code to copy the expression, otherwise we will end up
// with source and target pointing to the same list.
emitListVarCopy(gctx *generationContext)
// Return the expression, calling the transformer func for
// every expression in the tree. If the transformer func returns non-nil,
// its result is used in place of the expression it was called with in the
// resulting expression. The resulting starlarkExpr will contain as many
// of the same objects from the original expression as possible.
transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr
}
func maybeString(expr starlarkExpr) (string, bool) {
@@ -62,6 +68,14 @@ func (s *stringLiteralExpr) emitListVarCopy(gctx *generationContext) {
s.emit(gctx)
}
func (s *stringLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if replacement := transformer(s); replacement != nil {
return replacement
} else {
return s
}
}
// Integer literal
type intLiteralExpr struct {
literal int
@@ -85,6 +99,14 @@ func (s *intLiteralExpr) emitListVarCopy(gctx *generationContext) {
s.emit(gctx)
}
func (s *intLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if replacement := transformer(s); replacement != nil {
return replacement
} else {
return s
}
}
// Boolean literal
type boolLiteralExpr struct {
literal bool
@@ -110,6 +132,14 @@ func (b *boolLiteralExpr) emitListVarCopy(gctx *generationContext) {
b.emit(gctx)
}
func (b *boolLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if replacement := transformer(b); replacement != nil {
return replacement
} else {
return b
}
}
// interpolateExpr represents Starlark's interpolation operator <string> % list
// we break <string> into a list of chunks, i.e., "first%second%third" % (X, Y)
// will have chunks = ["first", "second", "third"] and args = [X, Y]
@@ -190,6 +220,19 @@ func (xi *interpolateExpr) emitListVarCopy(gctx *generationContext) {
xi.emit(gctx)
}
func (xi *interpolateExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
argsCopy := make([]starlarkExpr, len(xi.args))
for i, arg := range xi.args {
argsCopy[i] = arg.transform(transformer)
}
xi.args = argsCopy
if replacement := transformer(xi); replacement != nil {
return replacement
} else {
return xi
}
}
type variableRefExpr struct {
ref variable
isDefined bool
@@ -220,6 +263,14 @@ func (v *variableRefExpr) emitListVarCopy(gctx *generationContext) {
}
}
func (v *variableRefExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if replacement := transformer(v); replacement != nil {
return replacement
} else {
return v
}
}
type toStringExpr struct {
expr starlarkExpr
}
@@ -265,6 +316,15 @@ func (s *toStringExpr) emitListVarCopy(gctx *generationContext) {
s.emit(gctx)
}
func (s *toStringExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
s.expr = s.expr.transform(transformer)
if replacement := transformer(s); replacement != nil {
return replacement
} else {
return s
}
}
type notExpr struct {
expr starlarkExpr
}
@@ -291,6 +351,15 @@ func (n *notExpr) emitListVarCopy(gctx *generationContext) {
n.emit(gctx)
}
func (n *notExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
n.expr = n.expr.transform(transformer)
if replacement := transformer(n); replacement != nil {
return replacement
} else {
return n
}
}
type eqExpr struct {
left, right starlarkExpr
isEq bool // if false, it's !=
@@ -360,6 +429,16 @@ func (eq *eqExpr) emitListVarCopy(gctx *generationContext) {
eq.emit(gctx)
}
func (eq *eqExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
eq.left = eq.left.transform(transformer)
eq.right = eq.right.transform(transformer)
if replacement := transformer(eq); replacement != nil {
return replacement
} else {
return eq
}
}
// variableDefinedExpr corresponds to Make's ifdef VAR
type variableDefinedExpr struct {
v variable
@@ -388,6 +467,11 @@ func (v *variableDefinedExpr) emitListVarCopy(gctx *generationContext) {
v.emit(gctx)
}
func (v *variableDefinedExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
// TODO: VariableDefinedExpr isn't really an expression?
return v
}
type listExpr struct {
items []starlarkExpr
}
@@ -442,6 +526,19 @@ func (l *listExpr) emitListVarCopy(gctx *generationContext) {
l.emit(gctx)
}
func (l *listExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
itemsCopy := make([]starlarkExpr, len(l.items))
for i, item := range l.items {
itemsCopy[i] = item.transform(transformer)
}
l.items = itemsCopy
if replacement := transformer(l); replacement != nil {
return replacement
} else {
return l
}
}
func newStringListExpr(items []string) *listExpr {
v := listExpr{}
for _, item := range items {
@@ -505,6 +602,19 @@ func (c *concatExpr) emitListVarCopy(gctx *generationContext) {
c.emit(gctx)
}
func (c *concatExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
itemsCopy := make([]starlarkExpr, len(c.items))
for i, item := range c.items {
itemsCopy[i] = item.transform(transformer)
}
c.items = itemsCopy
if replacement := transformer(c); replacement != nil {
return replacement
} else {
return c
}
}
// inExpr generates <expr> [not] in <list>
type inExpr struct {
expr starlarkExpr
@@ -543,23 +653,33 @@ func (i *inExpr) emitListVarCopy(gctx *generationContext) {
i.emit(gctx)
}
func (i *inExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
i.expr = i.expr.transform(transformer)
i.list = i.list.transform(transformer)
if replacement := transformer(i); replacement != nil {
return replacement
} else {
return i
}
}
type indexExpr struct {
array starlarkExpr
index starlarkExpr
}
func (ix indexExpr) emit(gctx *generationContext) {
func (ix *indexExpr) emit(gctx *generationContext) {
ix.array.emit(gctx)
gctx.write("[")
ix.index.emit(gctx)
gctx.write("]")
}
func (ix indexExpr) typ() starlarkType {
func (ix *indexExpr) typ() starlarkType {
return starlarkTypeString
}
func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
func (ix *indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
newArray, isSameArray := ix.array.eval(valueMap)
newIndex, isSameIndex := ix.index.eval(valueMap)
if same = isSameArray && isSameIndex; same {
@@ -570,10 +690,20 @@ func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, sa
return
}
func (ix indexExpr) emitListVarCopy(gctx *generationContext) {
func (ix *indexExpr) emitListVarCopy(gctx *generationContext) {
ix.emit(gctx)
}
func (ix *indexExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
ix.array = ix.array.transform(transformer)
ix.index = ix.index.transform(transformer)
if replacement := transformer(ix); replacement != nil {
return replacement
} else {
return ix
}
}
type callExpr struct {
object starlarkExpr // nil if static call
name string
@@ -642,6 +772,21 @@ func (cx *callExpr) emitListVarCopy(gctx *generationContext) {
cx.emit(gctx)
}
func (cx *callExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if cx.object != nil {
cx.object = cx.object.transform(transformer)
}
argsCopy := make([]starlarkExpr, len(cx.args))
for i, arg := range cx.args {
argsCopy[i] = arg.transform(transformer)
}
if replacement := transformer(cx); replacement != nil {
return replacement
} else {
return cx
}
}
type ifExpr struct {
condition starlarkExpr
ifTrue starlarkExpr
@@ -691,6 +836,92 @@ func (i *ifExpr) emitListVarCopy(gctx *generationContext) {
i.emit(gctx)
}
func (i *ifExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
i.condition = i.condition.transform(transformer)
i.ifTrue = i.ifTrue.transform(transformer)
i.ifFalse = i.ifFalse.transform(transformer)
if replacement := transformer(i); replacement != nil {
return replacement
} else {
return i
}
}
type identifierExpr struct {
name string
}
func (i *identifierExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
return i, true
}
func (i *identifierExpr) emit(gctx *generationContext) {
gctx.write(i.name)
}
func (i *identifierExpr) typ() starlarkType {
return starlarkTypeUnknown
}
func (i *identifierExpr) emitListVarCopy(gctx *generationContext) {
i.emit(gctx)
}
func (i *identifierExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if replacement := transformer(i); replacement != nil {
return replacement
} else {
return i
}
}
type foreachExpr struct {
varName string
list starlarkExpr
action starlarkExpr
}
func (f *foreachExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
list, listSame := f.list.eval(valueMap)
action, actionSame := f.action.eval(valueMap)
same = listSame && actionSame
if same {
return f, same
} else {
return &foreachExpr{
varName: f.varName,
list: list,
action: action,
}, same
}
}
func (f *foreachExpr) emit(gctx *generationContext) {
gctx.write("[")
f.action.emit(gctx)
gctx.write(" for " + f.varName + " in ")
f.list.emit(gctx)
gctx.write("]")
}
func (f *foreachExpr) typ() starlarkType {
return starlarkTypeList
}
func (f *foreachExpr) emitListVarCopy(gctx *generationContext) {
f.emit(gctx)
}
func (f *foreachExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
f.list = f.list.transform(transformer)
f.action = f.action.transform(transformer)
if replacement := transformer(f); replacement != nil {
return replacement
} else {
return f
}
}
type badExpr struct {
errorLocation ErrorLocation
message string
@@ -714,6 +945,14 @@ func (_ *badExpr) emitListVarCopy(_ *generationContext) {
panic("implement me")
}
func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
if replacement := transformer(b); replacement != nil {
return replacement
} else {
return b
}
}
func maybeConvertToStringList(expr starlarkExpr) starlarkExpr {
if xString, ok := expr.(*stringLiteralExpr); ok {
return newStringListExpr(strings.Fields(xString.literal))

45
mk2rbc/mk2rbc.go
View File

@@ -111,6 +111,7 @@ var knownFunctions = map[string]struct {
"filter": {baseName + ".filter", starlarkTypeList, hiddenArgNone},
"filter-out": {baseName + ".filter_out", starlarkTypeList, hiddenArgNone},
"firstword": {"!firstword", starlarkTypeString, hiddenArgNone},
"foreach": {"!foreach", starlarkTypeList, hiddenArgNone},
"get-vendor-board-platforms": {"!get-vendor-board-platforms", starlarkTypeList, hiddenArgNone}, // internal macro, used by is-board-platform, etc.
"if": {"!if", starlarkTypeUnknown, hiddenArgNone},
"info": {baseName + ".mkinfo", starlarkTypeVoid, hiddenArgNone},
@@ -147,14 +148,10 @@ var knownFunctions = map[string]struct {
"warning": {baseName + ".mkwarning", starlarkTypeVoid, hiddenArgNone},
"word": {baseName + "!word", starlarkTypeString, hiddenArgNone},
"wildcard": {baseName + ".expand_wildcard", starlarkTypeList, hiddenArgNone},
"words": {baseName + ".words", starlarkTypeList, hiddenArgNone},
}
var builtinFuncRex = regexp.MustCompile(
"^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" +
"|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" +
"|if|info|join|lastword|notdir|or|origin|patsubst|realpath" +
"|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" +
"|wildcard)")
var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
// Conversion request parameters
type Request struct {
@@ -1399,6 +1396,8 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt
switch expr.name {
case "if":
return ctx.parseIfFunc(node, args)
case "foreach":
return ctx.parseForeachFunc(node, args)
case "word":
return ctx.parseWordFunc(node, args)
case "firstword", "lastword":
@@ -1483,6 +1482,38 @@ func (ctx *parseContext) parseIfFunc(node mkparser.Node, args *mkparser.MakeStri
}
}
func (ctx *parseContext) parseForeachFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
words := args.Split(",")
if len(words) != 3 {
return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
}
if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier")
}
loopVarName := words[0].Strings[0]
list := ctx.parseMakeString(node, words[1])
action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr {
if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
return &identifierExpr{loopVarName}
}
return nil
})
if list.typ() != starlarkTypeList {
list = &callExpr{
name: "words",
returnType: knownFunctions["words"].returnType,
args: []starlarkExpr{list},
}
}
return &foreachExpr{
varName: loopVarName,
list: list,
action: action,
}
}
func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
words := args.Split(",")
if len(words) != 2 {
@@ -1504,7 +1535,7 @@ func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeSt
if array.typ() != starlarkTypeList {
array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
}
return indexExpr{array, &intLiteralExpr{int(index - 1)}}
return &indexExpr{array, &intLiteralExpr{int(index - 1)}}
}
func (ctx *parseContext) parseFirstOrLastwordFunc(node mkparser.Node, name string, args *mkparser.MakeString) starlarkExpr {

22
mk2rbc/mk2rbc_test.go
View File

@@ -1129,6 +1129,28 @@ def init(g, handle):
g["SOURCES"] = "foo.c bar.c"
g["OBJECTS"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"])
g["OBJECTS2"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"])
`,
},
{
desc: "foreach expressions",
mkname: "product.mk",
in: `
BOOT_KERNEL_MODULES := foo.ko bar.ko
BOOT_KERNEL_MODULES_FILTER := $(foreach m,$(BOOT_KERNEL_MODULES),%/$(m))
BOOT_KERNEL_MODULES_LIST := foo.ko
BOOT_KERNEL_MODULES_LIST += bar.ko
BOOT_KERNEL_MODULES_FILTER_2 := $(foreach m,$(BOOT_KERNEL_MODULES_LIST),%/$(m))
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
def init(g, handle):
cfg = rblf.cfg(handle)
g["BOOT_KERNEL_MODULES"] = "foo.ko bar.ko"
g["BOOT_KERNEL_MODULES_FILTER"] = ["%%/%s" % m for m in rblf.words(g["BOOT_KERNEL_MODULES"])]
g["BOOT_KERNEL_MODULES_LIST"] = ["foo.ko"]
g["BOOT_KERNEL_MODULES_LIST"] += ["bar.ko"]
g["BOOT_KERNEL_MODULES_FILTER_2"] = ["%%/%s" % m for m in g["BOOT_KERNEL_MODULES_LIST"]]
`,
},
}