parseExpr less allocation
diff --git a/eval.go b/eval.go
index 1fc0baf..417f945 100644
--- a/eval.go
+++ b/eval.go
@@ -167,7 +167,7 @@
if assign != nil {
if ast.term == ';' {
- nexpr, _, err := parseExpr(ast.afterTerm, nil)
+ nexpr, _, err := parseExpr(ast.afterTerm, nil, false)
if err != nil {
panic(fmt.Errorf("parse %s:%d %v", ev.filename, ev.lineno, err))
}
@@ -325,7 +325,7 @@
ev.lineno = ast.lineno
Logf("%s:%d include %q", ev.filename, ev.lineno, ast.expr)
- v, _, err := parseExpr([]byte(ast.expr), nil)
+ v, _, err := parseExpr([]byte(ast.expr), nil, false)
if err != nil {
panic(err)
}
@@ -413,7 +413,7 @@
ev.filename = ast.filename
ev.lineno = ast.lineno
- v, _, err := parseExpr(ast.expr, nil)
+ v, _, err := parseExpr(ast.expr, nil, false)
if err != nil {
panic(err)
}
diff --git a/expr.go b/expr.go
index f5e4e11..6c63216 100644
--- a/expr.go
+++ b/expr.go
@@ -26,6 +26,7 @@
var (
errEndOfInput = errors.New("parse: unexpected end of input")
+ errNotLiteral = errors.New("valueNum: not literal")
bufFree = sync.Pool{
New: func() interface{} { return new(buffer) },
@@ -63,7 +64,6 @@
}
// literal is literal value.
-// TODO(ukai): always use []byte?
type literal string
func (s literal) String() string { return string(s) }
@@ -79,7 +79,6 @@
}
// tmpval is temporary value.
-// TODO(ukai): Values() returns []Value? (word list?)
type tmpval []byte
func (t tmpval) String() string { return string(t) }
@@ -249,14 +248,51 @@
v.subst.Dump(w)
}
+func str(buf []byte, alloc bool) Value {
+ if alloc {
+ return literal(string(buf))
+ }
+ return tmpval(buf)
+}
+
+func appendStr(expr Expr, buf []byte, alloc bool) Expr {
+ if len(buf) == 0 {
+ return expr
+ }
+ if len(expr) == 0 {
+ return Expr{str(buf, alloc)}
+ }
+ switch v := expr[len(expr)-1].(type) {
+ case literal:
+ v += literal(string(buf))
+ expr[len(expr)-1] = v
+ return expr
+ case tmpval:
+ v = append(v, buf...)
+ expr[len(expr)-1] = v
+ return expr
+ }
+ return append(expr, str(buf, alloc))
+}
+
+func valueNum(v Value) (int, error) {
+ switch v := v.(type) {
+ case literal, tmpval:
+ n, err := strconv.ParseInt(v.String(), 10, 64)
+ return int(n), err
+ }
+ return 0, errNotLiteral
+}
+
// parseExpr parses expression in `in` until it finds any byte in term.
// if term is nil, it will parse to end of input.
// if term is not nil, and it reaches to end of input, return errEndOfInput.
// it returns parsed value, and parsed length `n`, so in[n-1] is any byte of
// term, and in[n:] is next input.
-func parseExpr(in, term []byte) (Value, int, error) {
+// if alloc is true, text will be literal (allocate string).
+// otherwise, text will be tmpval on in.
+func parseExpr(in, term []byte, alloc bool) (Value, int, error) {
var expr Expr
- buf := make([]byte, 0, len(in))
b := 0
i := 0
var saveParen byte
@@ -273,28 +309,20 @@
break Loop
}
if in[i+1] == '$' {
- buf = append(buf, in[b:i+1]...)
+ expr = appendStr(expr, in[b:i+1], alloc)
i += 2
b = i
continue
}
if bytes.IndexByte(term, in[i+1]) >= 0 {
- buf = append(buf, in[b:i]...)
- if len(buf) > 0 {
- expr = append(expr, literal(string(buf)))
- buf = buf[:0]
- }
+ expr = appendStr(expr, in[b:i], alloc)
expr = append(expr, varref{varname: literal("")})
i++
b = i
break Loop
}
- buf = append(buf, in[b:i]...)
- if len(buf) > 0 {
- expr = append(expr, literal(string(buf)))
- buf = buf[:0]
- }
- v, n, err := parseDollar(in[i:])
+ expr = appendStr(expr, in[b:i], alloc)
+ v, n, err := parseDollar(in[i:], alloc)
if err != nil {
return nil, 0, err
}
@@ -321,10 +349,7 @@
}
i++
}
- buf = append(buf, in[b:i]...)
- if len(buf) > 0 {
- expr = append(expr, literal(string(buf)))
- }
+ expr = appendStr(expr, in[b:i], alloc)
if i == len(in) && term != nil {
return expr, i, errEndOfInput
}
@@ -347,7 +372,7 @@
// $(expr)
// $x
// it returns parsed value and parsed length.
-func parseDollar(in []byte) (Value, int, error) {
+func parseDollar(in []byte, alloc bool) (Value, int, error) {
if len(in) <= 1 {
return nil, 0, errors.New("empty expr")
}
@@ -363,14 +388,14 @@
if in[1] >= '0' && in[1] <= '9' {
return paramref(in[1] - '0'), 2, nil
}
- return varref{varname: literal(string(in[1]))}, 2, nil
+ return varref{varname: str(in[1:2], alloc)}, 2, nil
}
term := []byte{paren, ':', ' '}
var varname Expr
i := 2
Again:
for {
- e, n, err := parseExpr(in[i:], term)
+ e, n, err := parseExpr(in[i:], term, alloc)
if err != nil {
return nil, 0, err
}
@@ -380,40 +405,40 @@
case paren:
// ${expr}
vname := compactExpr(varname)
- if vname, ok := vname.(literal); ok {
- n, err := strconv.ParseInt(string(vname), 10, 64)
- if err == nil {
- // ${n}
- return paramref(n), i + 1, nil
- }
+ n, err := valueNum(vname)
+ if err == nil {
+ // ${n}
+ return paramref(n), i + 1, nil
}
return varref{varname: vname}, i + 1, nil
case ' ':
// ${e ...}
- if token, ok := e.(literal); ok {
- funcName := string(token)
+ switch token := e.(type) {
+ case literal, tmpval:
+ funcName := intern(token.String())
if f, ok := funcMap[funcName]; ok {
- return parseFunc(f(), in, i+1, term[:1], funcName)
+ return parseFunc(f(), in, i+1, term[:1], funcName, alloc)
}
}
term = term[:2] // drop ' '
continue Again
case ':':
// ${varname:...}
+ colon := in[i:i+1]
term = term[:2]
term[1] = '=' // term={paren, '='}.
- e, n, err := parseExpr(in[i+1:], term)
+ e, n, err := parseExpr(in[i+1:], term, alloc)
if err != nil {
return nil, 0, err
}
i += 1 + n
if in[i] == paren {
- varname = append(varname, literal(string(":")), e)
+ varname = appendStr(varname, colon, alloc)
return varref{varname: varname}, i + 1, nil
}
// ${varname:xx=...}
pat := e
- subst, n, err := parseExpr(in[i+1:], term[:1])
+ subst, n, err := parseExpr(in[i+1:], term[:1], alloc)
if err != nil {
return nil, 0, err
}
@@ -485,6 +510,7 @@
}
// concatLine concatinates line with "\\\n" in function expression.
+// TODO(ukai): less alloc?
func concatLine(v Value) Value {
switch v := v.(type) {
case literal:
@@ -523,8 +549,8 @@
// parseFunc parses function arguments from in[s:] for f.
// in[0] is '$' and in[s] is space just after func name.
// in[:n] will be "${func args...}"
-func parseFunc(f Func, in []byte, s int, term []byte, funcName string) (Value, int, error) {
- f.AddArg(literal(string(in[1 : s-1])))
+func parseFunc(f Func, in []byte, s int, term []byte, funcName string, alloc bool) (Value, int, error) {
+ f.AddArg(str(in[1:s-1], alloc))
arity := f.Arity()
term = append(term, ',')
i := skipSpaces(in[s:], term)
@@ -538,7 +564,7 @@
// final arguments.
term = term[:1] // drop ','
}
- v, n, err := parseExpr(in[i:], term)
+ v, n, err := parseExpr(in[i:], term, alloc)
if err != nil {
return nil, 0, err
}
@@ -597,6 +623,15 @@
func (m matchVarref) Serialize() SerializableVar { panic("not implemented") }
func (m matchVarref) Dump(w io.Writer) { panic("not implemented") }
+func matchValue(expr, pat Value) bool {
+ switch pat := pat.(type) {
+ case literal:
+ return literal(expr.String()) == pat
+ }
+ // TODO: other type match?
+ return false
+}
+
func matchExpr(expr, pat Expr) ([]Value, bool) {
if len(expr) != len(pat) {
return nil, false
@@ -612,7 +647,7 @@
}
return nil, false
}
- if expr[i] != pat[i] {
+ if !matchValue(expr[i], pat[i]) {
return nil, false
}
}
diff --git a/expr_test.go b/expr_test.go
index 7c9ac49..dbf715a 100644
--- a/expr_test.go
+++ b/expr_test.go
@@ -281,7 +281,7 @@
},
},
} {
- val, _, err := parseExpr([]byte(tc.in), nil)
+ val, _, err := parseExpr([]byte(tc.in), nil, true)
if tc.isErr {
if err == nil {
t.Errorf(`parseExpr(%q)=_, _, nil; want error`, tc.in)
diff --git a/func.go b/func.go
index 1ce5f38..a7ec835 100644
--- a/func.go
+++ b/func.go
@@ -979,13 +979,13 @@
case ":=":
// TODO(ukai): compute parsed expr in Compact when f.rhs is
// literal? e.g. literal("$(foo)") => varref{literal("foo")}.
- expr, _, err := parseExpr(rhs, nil)
+ expr, _, err := parseExpr(rhs, nil, false)
if err != nil {
panic(fmt.Sprintf("eval assign error: %q: %v", f.String(), err))
}
- abuf.Reset()
- expr.Eval(&abuf, ev)
- rvalue = SimpleVar{value: tmpval(abuf.Bytes()), origin: "file"}
+ var vbuf bytes.Buffer
+ expr.Eval(&vbuf, ev)
+ rvalue = SimpleVar{value: tmpval(vbuf.Bytes()), origin: "file"}
case "=":
rvalue = RecursiveVar{expr: tmpval(rhs), origin: "file"}
case "+=":
diff --git a/parser.go b/parser.go
index 603b812..fed8eac 100644
--- a/parser.go
+++ b/parser.go
@@ -170,11 +170,11 @@
}
func newAssignAST(p *parser, lhsBytes []byte, rhsBytes []byte, op string) *AssignAST {
- lhs, _, err := parseExpr(lhsBytes, nil)
+ lhs, _, err := parseExpr(lhsBytes, nil, true)
if err != nil {
panic(err)
}
- rhs, _, err := parseExpr(rhsBytes, nil)
+ rhs, _, err := parseExpr(rhsBytes, nil, true)
if err != nil {
panic(err)
}
@@ -225,7 +225,7 @@
term = '='
}
- v, _, err := parseExpr(expr, nil)
+ v, _, err := parseExpr(expr, nil, true)
if err != nil {
panic(fmt.Errorf("parse %s:%d %v", p.mk.filename, p.lineno, err))
}
@@ -252,7 +252,7 @@
}
func (p *parser) parseIfdef(line []byte, oplen int) AST {
- lhs, _, err := parseExpr(trimLeftSpaceBytes(line[oplen+1:]), nil)
+ lhs, _, err := parseExpr(trimLeftSpaceBytes(line[oplen+1:]), nil, true)
if err != nil {
panic(fmt.Errorf("ifdef parse %s:%d %v", p.mk.filename, p.lineno, err))
}
@@ -300,14 +300,14 @@
s = s[1 : len(s)-1]
term := []byte{','}
in := []byte(s)
- v, n, err := parseExpr(in, term)
+ v, n, err := parseExpr(in, term, false)
if err != nil {
return "", "", false
}
lhs := v.String()
n++
n += skipSpaces(in[n:], nil)
- v, n, err = parseExpr(in[n:], nil)
+ v, n, err = parseExpr(in[n:], nil, false)
if err != nil {
return "", "", false
}
@@ -328,11 +328,11 @@
Error(p.mk.filename, p.lineno, `*** invalid syntax in conditional.`)
}
- lhs, _, err := parseExpr([]byte(lhsBytes), nil)
+ lhs, _, err := parseExpr([]byte(lhsBytes), nil, true)
if err != nil {
panic(fmt.Errorf("parse ifeq lhs %s:%d %v", p.mk.filename, p.lineno, err))
}
- rhs, _, err := parseExpr([]byte(rhsBytes), nil)
+ rhs, _, err := parseExpr([]byte(rhsBytes), nil, true)
if err != nil {
panic(fmt.Errorf("parse ifeq rhs %s:%d %v", p.mk.filename, p.lineno, err))
}
diff --git a/rule_parser_test.go b/rule_parser_test.go
index 4adf912..5dc8f24 100644
--- a/rule_parser_test.go
+++ b/rule_parser_test.go
@@ -19,14 +19,6 @@
"testing"
)
-func parseExprForTest(e string) Value {
- v, _, err := parseExpr([]byte(e), nil)
- if err != nil {
- panic(err)
- }
- return v
-}
-
func TestRuleParser(t *testing.T) {
for _, tc := range []struct {
in string
@@ -109,8 +101,8 @@
outputs: []string{"foo"},
},
assign: &AssignAST{
- lhs: parseExprForTest("CFLAGS"),
- rhs: parseExprForTest("-g"),
+ lhs: literal("CFLAGS"),
+ rhs: literal("-g"),
op: "=",
},
},
@@ -120,8 +112,8 @@
outputs: []string{"foo"},
},
assign: &AssignAST{
- lhs: parseExprForTest("CFLAGS"),
- rhs: parseExprForTest("-g"),
+ lhs: literal("CFLAGS"),
+ rhs: literal("-g"),
op: "=",
},
},
@@ -131,8 +123,8 @@
outputs: []string{"foo"},
},
assign: &AssignAST{
- lhs: parseExprForTest("CFLAGS"),
- rhs: parseExprForTest("-g"),
+ lhs: literal("CFLAGS"),
+ rhs: literal("-g"),
op: ":=",
},
},
@@ -142,8 +134,8 @@
outputPatterns: []pattern{pattern{suffix: ".o"}},
},
assign: &AssignAST{
- lhs: parseExprForTest("CFLAGS"),
- rhs: parseExprForTest("-g"),
+ lhs: literal("CFLAGS"),
+ rhs: literal("-g"),
op: ":=",
},
},
diff --git a/var.go b/var.go
index 986bd77..630968a 100644
--- a/var.go
+++ b/var.go
@@ -105,7 +105,7 @@
}
func (v SimpleVar) Append(ev *Evaluator, s string) Var {
- val, _, err := parseExpr([]byte(s), nil)
+ val, _, err := parseExpr([]byte(s), nil, false)
if err != nil {
panic(err)
}
@@ -157,7 +157,7 @@
} else {
expr = Expr{v.expr, literal(" ")}
}
- sv, _, err := parseExpr([]byte(s), nil)
+ sv, _, err := parseExpr([]byte(s), nil, true)
if err != nil {
panic(err)
}
@@ -175,7 +175,7 @@
buf.WriteString(v.expr.String())
buf.WriteByte(' ')
buf.WriteString(val.String())
- e, _, err := parseExpr(buf.Bytes(), nil)
+ e, _, err := parseExpr(buf.Bytes(), nil, true)
if err != nil {
panic(err)
}
diff --git a/worker.go b/worker.go
index bd3df11..d9ccd9e 100644
--- a/worker.go
+++ b/worker.go
@@ -126,7 +126,7 @@
return []runner{r}
}
// TODO(ukai): parse once more earlier?
- expr, _, err := parseExpr([]byte(r.cmd), nil)
+ expr, _, err := parseExpr([]byte(r.cmd), nil, false)
if err != nil {
panic(fmt.Errorf("parse cmd %q: %v", r.cmd, err))
}