diff --git a/ast/compound.go b/ast/compound.go index 31b5fc2..e50d0d0 100644 --- a/ast/compound.go +++ b/ast/compound.go @@ -6,6 +6,6 @@ type Compound struct { ExpressionNode Token token.Token Left ExpressionNode - Operator string + Operator token.Type Right ExpressionNode } diff --git a/ast/identifier.go b/ast/identifier.go index d63d240..84cd693 100644 --- a/ast/identifier.go +++ b/ast/identifier.go @@ -2,9 +2,32 @@ package ast import "ghostlang.org/x/ghost/token" +// Library binding states for an identifier. +// +// Resolving an identifier means consulting the library module and function +// registries before the scope chain, which costs two map lookups keyed by +// string. Ordinary variables miss both every time they are read, and in a loop +// that hashing dominated identifier evaluation. +// +// The optimizer classifies each identifier once, before evaluation begins, so +// the evaluator can skip the registries for names that cannot be library +// globals. The zero value is deliberately "unknown": an AST that was never +// optimized keeps the original behavior of always consulting the registries. +// +// This is written during optimization and only read during evaluation, which +// matters because Ghost code can be evaluated concurrently - http.handle() +// callbacks run per request on their own goroutines. +const ( + LibraryBindingUnknown uint8 = iota // not analyzed; consult the registries + LibraryBindingLocal // cannot be a library global + LibraryBindingGlobal // names a library module or function +) + type Identifier struct { - ExpressionNode AssignmentNode Token token.Token Value string + + // LibraryBinding is set by the optimizer; see the constants above. + LibraryBinding uint8 } diff --git a/ast/infix.go b/ast/infix.go index d1a9dc8..203ea71 100644 --- a/ast/infix.go +++ b/ast/infix.go @@ -6,6 +6,6 @@ type Infix struct { ExpressionNode Token token.Token Left ExpressionNode - Operator string + Operator token.Type Right ExpressionNode } diff --git a/ast/postfix.go b/ast/postfix.go index de304f4..b966dbf 100644 --- a/ast/postfix.go +++ b/ast/postfix.go @@ -5,5 +5,5 @@ import "ghostlang.org/x/ghost/token" type Postfix struct { ExpressionNode Token token.Token - Operator string + Operator token.Type } diff --git a/ast/prefix.go b/ast/prefix.go index 1d6a4b3..3226b0f 100644 --- a/ast/prefix.go +++ b/ast/prefix.go @@ -5,6 +5,6 @@ import "ghostlang.org/x/ghost/token" type Prefix struct { ExpressionNode Token token.Token - Operator string + Operator token.Type Right ExpressionNode } diff --git a/evaluator/benchmark_test.go b/evaluator/benchmark_test.go new file mode 100644 index 0000000..8b87e03 --- /dev/null +++ b/evaluator/benchmark_test.go @@ -0,0 +1,158 @@ +package evaluator + +import ( + "testing" + + "ghostlang.org/x/ghost/library/modules" + "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/optimizer" + "ghostlang.org/x/ghost/parser" + "ghostlang.org/x/ghost/scanner" +) + +// Benchmark programs exercise the hot paths of the tree-walking evaluator: +// function call overhead, variable lookup through the scope chain, arithmetic +// allocation, and method dispatch. +var benchmarks = []struct { + name string + source string +}{ + { + "Fib", + `function fib(n) { + if (n <= 1) { return n } + return fib(n - 1) + fib(n - 2) + } + fib(18)`, + }, + { + "LoopArithmetic", + `total = 0 + for (i = 0; i < 50000; i = i + 1) { + total = total + i * 2 - 1 + } + total`, + }, + { + "WhileLoop", + `i = 0 + total = 0 + while (i < 50000) { + total = total + i + i = i + 1 + } + total`, + }, + { + "FunctionCalls", + `function add(a, b) { return a + b } + total = 0 + for (i = 0; i < 20000; i = i + 1) { + total = add(total, i) + } + total`, + }, + { + "NestedScopeLookup", + `outer = 1 + function level1() { + function level2() { + function level3() { + total = 0 + for (i = 0; i < 10000; i = i + 1) { + total = total + outer + } + return total + } + return level3() + } + return level2() + } + level1()`, + }, + { + "ConstantExpressions", + `total = 0 + for (i = 0; i < 20000; i = i + 1) { + total = total + (2 * 3 + 4 * 5 - 6) + } + total`, + }, + { + "ListOperations", + `items = [] + for (i = 0; i < 10000; i = i + 1) { + items.push(i) + } + total = 0 + for (i = 0; i < 10000; i = i + 1) { + total = total + items[i] + } + total`, + }, + { + "StringConcat", + `s = "" + for (i = 0; i < 5000; i = i + 1) { + s = s + "x" + } + s.length()`, + }, + { + "ClassMethods", + `class Counter { + count = 0 + function increment() { + this.count = this.count + 1 + return this.count + } + } + c = Counter.new() + for (i = 0; i < 10000; i = i + 1) { + c.increment() + } + c.count`, + }, + { + "MapOperations", + `m = {"a": 1, "b": 2, "c": 3} + total = 0 + for (i = 0; i < 10000; i = i + 1) { + total = total + m["a"] + m["b"] + m["c"] + } + total`, + }, +} + +func BenchmarkEvaluate(b *testing.B) { + object.RegisterEvaluator(Evaluate) + modules.RegisterEvaluator(Evaluate) + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + scope := &object.Scope{Environment: object.NewEnvironment()} + s := scanner.New(bm.source, "bench.ghost") + p := parser.New(s) + program := p.Parse() + + if len(p.Errors()) != 0 { + b.Fatalf("parse errors: %v", p.Errors()) + } + + // Mirror the real pipeline in ghost.Execute, which optimizes + // the program before evaluating it. + program = optimizer.Optimize(program) + + result := Evaluate(program, scope) + + if object.IsError(result) { + b.Fatalf("runtime error: %s", result.(*object.Error).Message) + } + } + }) + } +} diff --git a/evaluator/boolean.go b/evaluator/boolean.go index 0532cbc..fbf8941 100644 --- a/evaluator/boolean.go +++ b/evaluator/boolean.go @@ -3,6 +3,7 @@ package evaluator import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/token" ) func evaluateBoolean(node *ast.Boolean, scope *object.Scope) object.Object { @@ -14,13 +15,13 @@ func evaluateBooleanInfix(node *ast.Infix, left object.Object, right object.Obje rightValue := right.(*object.Boolean).Value switch node.Operator { - case "and": + case token.AND: return toBooleanValue(leftValue && rightValue) - case "or": + case token.OR: return toBooleanValue(leftValue || rightValue) - case "==": + case token.EQUALEQUAL: return toBooleanValue(leftValue == rightValue) - case "!=": + case token.BANGEQUAL: return toBooleanValue(leftValue != rightValue) } diff --git a/evaluator/compound.go b/evaluator/compound.go index 6d125c2..71bf512 100644 --- a/evaluator/compound.go +++ b/evaluator/compound.go @@ -3,13 +3,29 @@ package evaluator import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/token" ) +// compoundOperators maps each compound assignment operator to the binary +// operator it applies, e.g. `+=` performs `+`. +var compoundOperators = map[token.Type]token.Type{ + token.PLUSEQUAL: token.PLUS, + token.MINUSEQUAL: token.MINUS, + token.STAREQUAL: token.STAR, + token.SLASHEQUAL: token.SLASH, +} + func evaluateCompound(node *ast.Compound, scope *object.Scope) object.Object { + operator, ok := compoundOperators[node.Operator] + + if !ok { + return newError("%d:%d:%s: runtime error: unknown operator: %s", node.Token.Line, node.Token.Column, node.Token.File, node.Operator) + } + infix := &ast.Infix{ Token: node.Token, Left: node.Left, - Operator: node.Operator[:len(node.Operator)-1], + Operator: operator, Right: node.Right, } diff --git a/evaluator/evaluator_test.go b/evaluator/evaluator_test.go index 5786a92..3b02203 100644 --- a/evaluator/evaluator_test.go +++ b/evaluator/evaluator_test.go @@ -5,6 +5,7 @@ import ( "ghostlang.org/x/ghost/library/modules" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/optimizer" "ghostlang.org/x/ghost/parser" "ghostlang.org/x/ghost/scanner" ) @@ -509,6 +510,137 @@ func TestThisOutsideClass(t *testing.T) { // ============================================================================= // Helper functions +func TestBangOperator(t *testing.T) { + tests := []struct { + input string + expected bool + }{ + {"!true", false}, + {"!false", true}, + {"!null", true}, + {"!!true", true}, + {"!!false", false}, + + // Comparisons and library functions produce boolean objects that are + // not the shared TRUE/FALSE singletons. The bang operator must compare + // them by value, not by pointer identity. + {`!("a" != "a")`, true}, + {`!("a" == "a")`, false}, + {`!("a" == "b")`, true}, + {"!(1 != 1)", true}, + {"!(1 == 1)", false}, + {`!("abc".startsWith("x"))`, true}, + {`!("abc".startsWith("a"))`, false}, + + // Non-boolean, non-null operands remain falsy under bang. + {"!5", false}, + {`!"abc"`, false}, + } + + for _, tt := range tests { + result := evaluate(tt.input) + + boolean, ok := result.(*object.Boolean) + + if !ok { + t.Fatalf("evaluate(%q) is not object.Boolean. got=%T (%+v)", tt.input, result, result) + } + + if boolean.Value != tt.expected { + t.Errorf("evaluate(%q) is not %t. got=%t", tt.input, tt.expected, boolean.Value) + } + } +} + +// TestOptimizerPreservesSemantics evaluates each program twice, once on the raw +// AST and once on the optimized AST, and requires the two to agree. Constant +// folding is only correct if it is invisible, including for the expressions it +// deliberately refuses to fold because they raise runtime errors. +func TestOptimizerPreservesSemantics(t *testing.T) { + programs := []string{ + // Arithmetic, including the integer/float promotion rules. + "1 + 2", "10 - 4", "6 * 7", "7 % 3", "-5", "-(3 * 4)", + "2 * 3 + 4 * 5 - 6", "1.5 + 2.5", "1 + 2.5", "6 / 3", "7 / 2", + "2147483647 * 2147483647", "-9223372036854775807 - 1", + + // Comparisons and booleans. + "1 < 2", "2 <= 2", "3 > 4", "1 == 1", "1 != 1", "1.0 == 1", + `"a" == "a"`, `"a" < "b"`, `"a" != "b"`, + "true and false", "true or false", "true == true", "false != true", + "!true", "!false", "!null", "!(1 == 2)", `!("a" != "a")`, + + // Strings. + `"hello" + " " + "world"`, `"" + "a"`, + + // Expressions the optimizer must leave alone: each raises a runtime + // error whose message and position must survive. + "1 / 0", "1 % 0", "1.0 / 0.0", "1 + true", `1 + "a"`, "-true", + "true + false", `"a" - "b"`, + + // Ranges build list objects and must not be folded. + "1 .. 5", "(1 + 1) .. (2 + 3)", + + // Folding inside larger constructs. + `total = 0 + for (i = 0; i < 2 * 5; i = i + 1) { total = total + (3 * 4) } + total`, + `function f(a = 2 * 3) { return a + (1 + 1) } f()`, + `if (1 + 1 == 2) { "yes" } else { "no" }`, + `x = 5 while (x > 5 - 5) { x = x - 1 } x`, + `[1 + 1, 2 * 2, "a" + "b"]`, + `m = {"k": 3 * 3} m["k"]`, + `(1 == 1) ? "t" : "f"`, + + // Library globals are classified by the optimizer, so confirm modules, + // functions, and their precedence over scope bindings all survive it. + `math.pi > 3`, + `math.floor(3.7)`, + `type(5)`, + `type("a")`, + `type([1])`, + `math.abs(-4)`, + `x = math.pi x > 3`, + // A library name still wins over a same-named variable, as before. + `type = 5 type("a")`, + } + + for _, source := range programs { + plain := evaluate(source) + optimized := evaluateOptimized(source) + + if (plain == nil) != (optimized == nil) { + t.Errorf("%q: nil mismatch: plain=%v optimized=%v", source, plain, optimized) + continue + } + + if plain == nil { + continue + } + + if plain.Type() != optimized.Type() { + t.Errorf("%q: type mismatch: plain=%s optimized=%s", source, plain.Type(), optimized.Type()) + continue + } + + if plain.String() != optimized.String() { + t.Errorf("%q: value mismatch: plain=%q optimized=%q", source, plain.String(), optimized.String()) + } + } +} + +func evaluateOptimized(input string) object.Object { + scope := &object.Scope{ + Environment: object.NewEnvironment(), + } + + object.RegisterEvaluator(Evaluate) + modules.RegisterEvaluator(Evaluate) + + p := parser.New(scanner.New(input, "test.ghost")) + + return Evaluate(optimizer.Optimize(p.Parse()), scope) +} + func evaluate(input string) object.Object { scope := &object.Scope{ Environment: object.NewEnvironment(), diff --git a/evaluator/identifier.go b/evaluator/identifier.go index 1f135a7..94281a3 100644 --- a/evaluator/identifier.go +++ b/evaluator/identifier.go @@ -7,12 +7,18 @@ import ( ) func evaluateIdentifier(node *ast.Identifier, scope *object.Scope) object.Object { - if libraryModule, ok := library.Modules[node.Value]; ok { - return libraryModule - } + // Library globals take precedence over scope bindings. The optimizer marks + // identifiers that cannot name one, letting ordinary variables skip two + // string-keyed map lookups per read. An unoptimized AST is left unmarked + // and still consults the registries. + if node.LibraryBinding != ast.LibraryBindingLocal { + if libraryModule, ok := library.Modules[node.Value]; ok { + return libraryModule + } - if libraryFunction, ok := library.Functions[node.Value]; ok { - return libraryFunction + if libraryFunction, ok := library.Functions[node.Value]; ok { + return libraryFunction + } } if identifier, ok := scope.Environment.Get(node.Value); ok { diff --git a/evaluator/import.go b/evaluator/import.go index f6b25b8..63b409f 100644 --- a/evaluator/import.go +++ b/evaluator/import.go @@ -10,6 +10,7 @@ import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/log" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/optimizer" "ghostlang.org/x/ghost/parser" "ghostlang.org/x/ghost/scanner" "ghostlang.org/x/ghost/token" @@ -136,6 +137,8 @@ func evaluateFile(file string, tok token.Token, scope *object.Scope) object.Obje return nil } + program = optimizer.Optimize(program) + newScope := &object.Scope{Self: scope.Self, Environment: object.NewEnvironment()} newScope.Environment.SetDirectory(scope.Environment.GetDirectory()) diff --git a/evaluator/number.go b/evaluator/number.go index 57d1912..8a0c9f2 100644 --- a/evaluator/number.go +++ b/evaluator/number.go @@ -3,6 +3,7 @@ package evaluator import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/token" ) func evaluateNumber(node *ast.Number, scope *object.Scope) object.Object { @@ -17,35 +18,35 @@ func evaluateNumberInfix(node *ast.Infix, left object.Object, right object.Objec rightNum := right.(*object.Number) switch node.Operator { - case "+": + case token.PLUS: return leftNum.Add(rightNum) - case "-": + case token.MINUS: return leftNum.Sub(rightNum) - case "*": + case token.STAR: return leftNum.Mul(rightNum) - case "/": + case token.SLASH: if rightNum.IsZero() { return newError("%d:%d:%s: runtime error: division by zero", node.Token.Line, node.Token.Column, node.Token.File) } return leftNum.Div(rightNum) - case "%": + case token.PERCENT: if rightNum.IsZero() { return newError("%d:%d:%s: runtime error: division by zero", node.Token.Line, node.Token.Column, node.Token.File) } return leftNum.Mod(rightNum) - case "<": + case token.LESS: return toBooleanValue(leftNum.LessThan(rightNum)) - case "<=": + case token.LESSEQUAL: return toBooleanValue(leftNum.LessThanOrEqual(rightNum)) - case ">": + case token.GREATER: return toBooleanValue(leftNum.GreaterThan(rightNum)) - case ">=": + case token.GREATEREQUAL: return toBooleanValue(leftNum.GreaterThanOrEqual(rightNum)) - case "==": + case token.EQUALEQUAL: return toBooleanValue(leftNum.Equal(rightNum)) - case "!=": + case token.BANGEQUAL: return toBooleanValue(!leftNum.Equal(rightNum)) - case "..": + case token.DOTDOT: start := leftNum.Int64() end := rightNum.Int64() diff --git a/evaluator/postfix.go b/evaluator/postfix.go index d4f7fff..59f7f1c 100644 --- a/evaluator/postfix.go +++ b/evaluator/postfix.go @@ -3,11 +3,12 @@ package evaluator import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/token" ) func evaluatePostfix(node *ast.Postfix, scope *object.Scope) object.Object { switch node.Operator { - case "++": + case token.PLUSPLUS: value, ok := scope.Environment.Get(node.Token.Lexeme) if !ok { @@ -23,7 +24,7 @@ func evaluatePostfix(node *ast.Postfix, scope *object.Scope) object.Object { scope.Environment.Set(node.Token.Lexeme, newValue) return newValue - case "--": + case token.MINUSMINUS: value, ok := scope.Environment.Get(node.Token.Lexeme) if !ok { diff --git a/evaluator/prefix.go b/evaluator/prefix.go index d03ee5e..753127a 100644 --- a/evaluator/prefix.go +++ b/evaluator/prefix.go @@ -3,6 +3,7 @@ package evaluator import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/token" "ghostlang.org/x/ghost/value" ) @@ -14,18 +15,22 @@ func evaluatePrefix(node *ast.Prefix, scope *object.Scope) object.Object { } switch node.Operator { - case "!": - switch right { - case value.TRUE: - return value.FALSE - case value.FALSE: - return value.TRUE - case value.NULL: + case token.BANG: + // Compare by value rather than by pointer identity. Not every boolean + // reaching this point is one of the value.TRUE/value.FALSE singletons: + // string comparisons and library functions such as string.startsWith() + // and math.isNegative() build fresh boolean objects, and an identity + // check silently fell through to the default branch for those, making + // !(expression) yield false regardless of the operand. + switch right := right.(type) { + case *object.Boolean: + return toBooleanValue(!right.Value) + case *object.Null: return value.TRUE default: return value.FALSE } - case "-": + case token.MINUS: // Only works with number objects if right.Type() != object.NUMBER { return newError("%d:%d:%s: runtime error: unknown operator: -%s", node.Token.Line, node.Token.Column, node.Token.File, right.Type()) diff --git a/evaluator/string.go b/evaluator/string.go index d8ded4f..cf053a2 100644 --- a/evaluator/string.go +++ b/evaluator/string.go @@ -3,6 +3,7 @@ package evaluator import ( "ghostlang.org/x/ghost/ast" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/token" ) func evaluateString(node *ast.String, scope *object.Scope) object.Object { @@ -14,20 +15,20 @@ func evaluateStringInfix(node *ast.Infix, left object.Object, right object.Objec rightValue := right.String() switch node.Operator { - case "+": + case token.PLUS: return &object.String{Value: leftValue + rightValue} - case "<": - return &object.Boolean{Value: leftValue < rightValue} - case "<=": - return &object.Boolean{Value: leftValue <= rightValue} - case ">": - return &object.Boolean{Value: leftValue > rightValue} - case ">=": - return &object.Boolean{Value: leftValue >= rightValue} - case "==": - return &object.Boolean{Value: leftValue == rightValue} - case "!=": - return &object.Boolean{Value: leftValue != rightValue} + case token.LESS: + return toBooleanValue(leftValue < rightValue) + case token.LESSEQUAL: + return toBooleanValue(leftValue <= rightValue) + case token.GREATER: + return toBooleanValue(leftValue > rightValue) + case token.GREATEREQUAL: + return toBooleanValue(leftValue >= rightValue) + case token.EQUALEQUAL: + return toBooleanValue(leftValue == rightValue) + case token.BANGEQUAL: + return toBooleanValue(leftValue != rightValue) } return newError("%d:%d:%s: runtime error: unknown operator: %s %s %s", node.Token.Line, node.Token.Column, node.Token.File, right.Type(), node.Operator, left.Type()) diff --git a/ghost/ghost.go b/ghost/ghost.go index a1a9685..4af4aa6 100644 --- a/ghost/ghost.go +++ b/ghost/ghost.go @@ -6,6 +6,7 @@ import ( "ghostlang.org/x/ghost/library/modules" "ghostlang.org/x/ghost/log" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/optimizer" "ghostlang.org/x/ghost/parser" "ghostlang.org/x/ghost/scanner" "ghostlang.org/x/ghost/value" @@ -74,6 +75,8 @@ func (ghost *Ghost) Execute() object.Object { return object.NewError(parser.Errors()[0]) } + program = optimizer.Optimize(program) + result := evaluator.Evaluate(program, ghost.Scope) if object.IsError(result) { diff --git a/library/functions/type.go b/library/functions/type.go index 530fc7b..c8309b1 100644 --- a/library/functions/type.go +++ b/library/functions/type.go @@ -12,7 +12,7 @@ func Type(scope *object.Scope, tok token.Token, args ...object.Object) object.Ob return object.NewError("%d:%d: runtime error: type() expects 1 argument. got=%d", tok.Line, tok.Column, len(args)) } - objectType := string(args[0].Type()) + objectType := args[0].Type().String() return &object.String{Value: strings.ToLower(objectType)} } diff --git a/library/library.go b/library/library.go index 922bcfd..f5d968b 100644 --- a/library/library.go +++ b/library/library.go @@ -4,6 +4,7 @@ import ( "ghostlang.org/x/ghost/library/functions" "ghostlang.org/x/ghost/library/modules" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/optimizer" ) var Functions = map[string]*object.LibraryFunction{} @@ -22,6 +23,20 @@ func init() { RegisterFunction("print", functions.Print) RegisterFunction("type", functions.Type) + + optimizer.SetGlobalResolver(IsGlobal) +} + +// IsGlobal reports whether a name refers to a registered library module or +// function. The optimizer uses it to classify identifiers ahead of evaluation. +func IsGlobal(name string) bool { + if _, ok := Modules[name]; ok { + return true + } + + _, ok := Functions[name] + + return ok } func RegisterFunction(name string, function object.GoFunction) { diff --git a/library/modules/ghost.go b/library/modules/ghost.go index 641f5eb..40eda0c 100644 --- a/library/modules/ghost.go +++ b/library/modules/ghost.go @@ -6,6 +6,7 @@ import ( "strings" "ghostlang.org/x/ghost/object" + "ghostlang.org/x/ghost/optimizer" "ghostlang.org/x/ghost/parser" "ghostlang.org/x/ghost/scanner" "ghostlang.org/x/ghost/token" @@ -36,7 +37,7 @@ func ghostAbort(scope *object.Scope, tok token.Token, args ...object.Object) obj return object.NewError(obj.Value) } - return object.NewError("%d:%d: runtime error: ghost.abort() expects the first argument to be of type 'null' or 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[0].Type()))) + return object.NewError("%d:%d: runtime error: ghost.abort() expects the first argument to be of type 'null' or 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[0].Type().String())) } func ghostExecute(scope *object.Scope, tok token.Token, args ...object.Object) object.Object { @@ -47,12 +48,12 @@ func ghostExecute(scope *object.Scope, tok token.Token, args ...object.Object) o source, ok := args[0].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: ghost.execute() expects the first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[0].Type()))) + return object.NewError("%d:%d: runtime error: ghost.execute() expects the first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[0].Type().String())) } scanner := scanner.New(source.Value, tok.File) parser := parser.New(scanner) - program := parser.Parse() + program := optimizer.Optimize(parser.Parse()) return evaluate(program, scope) } @@ -65,7 +66,7 @@ func ghostExtend(scope *object.Scope, tok token.Token, args ...object.Object) ob basePath, ok := args[0].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: ghost.extend() expects the first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[0].Type()))) + return object.NewError("%d:%d: runtime error: ghost.extend() expects the first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[0].Type().String())) } path := path.Clean(scope.Environment.GetDirectory() + "/" + basePath.Value) diff --git a/library/modules/io.go b/library/modules/io.go index 429eadb..4bbd734 100644 --- a/library/modules/io.go +++ b/library/modules/io.go @@ -27,13 +27,13 @@ func ioAppend(scope *object.Scope, tok token.Token, args ...object.Object) objec basePath, ok := args[0].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: io.append() expects first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[0].Type()))) + return object.NewError("%d:%d: runtime error: io.append() expects first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[0].Type().String())) } content, ok := args[1].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: io.append() expects second argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[1].Type()))) + return object.NewError("%d:%d: runtime error: io.append() expects second argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[1].Type().String())) } cleanPath := path.Clean(scope.Environment.GetDirectory() + "/" + basePath.Value) @@ -59,7 +59,7 @@ func ioRead(scope *object.Scope, tok token.Token, args ...object.Object) object. basePath, ok := args[0].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: io.read() expects first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[0].Type()))) + return object.NewError("%d:%d: runtime error: io.read() expects first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[0].Type().String())) } path := path.Clean(scope.Environment.GetDirectory() + "/" + basePath.Value) @@ -80,13 +80,13 @@ func ioWrite(scope *object.Scope, tok token.Token, args ...object.Object) object basePath, ok := args[0].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: io.write() expects first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[0].Type()))) + return object.NewError("%d:%d: runtime error: io.write() expects first argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[0].Type().String())) } content, ok := args[1].(*object.String) if !ok { - return object.NewError("%d:%d: runtime error: io.write() expects second argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(string(args[1].Type()))) + return object.NewError("%d:%d: runtime error: io.write() expects second argument to be of type 'string'. got=%s", tok.Line, tok.Column, strings.ToLower(args[1].Type().String())) } path := path.Clean(scope.Environment.GetDirectory() + "/" + basePath.Value) diff --git a/object/boolean.go b/object/boolean.go index f200247..2304ba5 100644 --- a/object/boolean.go +++ b/object/boolean.go @@ -2,8 +2,6 @@ package object import "fmt" -const BOOLEAN = "BOOLEAN" - // Boolean objects consist of a boolean value. type Boolean struct { Value bool diff --git a/object/break.go b/object/break.go index d1480f4..0d009e6 100644 --- a/object/break.go +++ b/object/break.go @@ -1,7 +1,5 @@ package object -const BREAK = "BREAK" - // Break objects consist of a nil value. type Break struct{} diff --git a/object/class.go b/object/class.go index ffe0577..44eb0a1 100644 --- a/object/class.go +++ b/object/class.go @@ -4,8 +4,6 @@ import ( "ghostlang.org/x/ghost/ast" ) -const CLASS = "CLASS" - // Class objects consist of a body and an environment. type Class struct { Name *ast.Identifier diff --git a/object/continue.go b/object/continue.go index 04162b9..3435b60 100644 --- a/object/continue.go +++ b/object/continue.go @@ -1,7 +1,5 @@ package object -const CONTINUE = "CONTINUE" - // Continue objects consist of a nil value. type Continue struct{} diff --git a/object/environment.go b/object/environment.go index 794e97d..ab3a244 100644 --- a/object/environment.go +++ b/object/environment.go @@ -5,17 +5,32 @@ import ( "os" ) +// inlineCapacity is how many variables an environment stores inline, in the +// environment struct itself, before it falls back to a map. +// +// Every function call builds a new environment, and almost all of them hold +// only a handful of names: the parameters plus a few locals. A map is a poor +// fit for that. Allocating one, and growing its buckets as names are set, +// dominated the memory profile of call-heavy programs. Storing the first few +// names inline means an environment costs a single allocation and lookups are +// a short scan of adjacent memory rather than a hash. Environments that hold +// more names than this - module and class scopes, mainly - spill the remainder +// into the overflow map and keep their previous behavior. +const inlineCapacity = 4 + type Environment struct { - store map[string]Object + names [inlineCapacity]string + values [inlineCapacity]Object + count int + overflow map[string]Object + outer *Environment writer io.Writer directory string } func NewEnvironment() *Environment { - store := make(map[string]Object) - - return &Environment{store: store, writer: os.Stdout} + return &Environment{writer: os.Stdout} } func NewEnclosedEnvironment(outer *Environment) *Environment { @@ -26,12 +41,40 @@ func NewEnclosedEnvironment(outer *Environment) *Environment { return environment } +// local looks a name up in this environment only, ignoring the outer chain. +func (environment *Environment) local(name string) (Object, bool) { + for index := 0; index < environment.count; index++ { + if environment.names[index] == name { + return environment.values[index], true + } + } + + if environment.overflow != nil { + value, ok := environment.overflow[name] + + return value, ok + } + + return nil, false +} + +// All returns a copy of the names bound in this environment. func (environment *Environment) All() map[string]Object { - return environment.store + all := make(map[string]Object, environment.count+len(environment.overflow)) + + for index := 0; index < environment.count; index++ { + all[environment.names[index]] = environment.values[index] + } + + for name, value := range environment.overflow { + all[name] = value + } + + return all } func (environment *Environment) Has(name string) bool { - _, ok := environment.store[name] + _, ok := environment.local(name) if !ok && environment.outer != nil { _, ok = environment.outer.Get(name) @@ -41,7 +84,7 @@ func (environment *Environment) Has(name string) bool { } func (environment *Environment) Get(name string) (Object, bool) { - object, ok := environment.store[name] + object, ok := environment.local(name) if !ok && environment.outer != nil { object, ok = environment.outer.Get(name) @@ -50,14 +93,62 @@ func (environment *Environment) Get(name string) (Object, bool) { return object, ok } +// Set binds a name in this environment, replacing any existing binding here. +// It never walks the outer chain. func (environment *Environment) Set(name string, value Object) Object { - environment.store[name] = value + for index := 0; index < environment.count; index++ { + if environment.names[index] == name { + environment.values[index] = value + + return value + } + } + + if environment.overflow != nil { + if _, ok := environment.overflow[name]; ok { + environment.overflow[name] = value + + return value + } + } + + if environment.count < inlineCapacity { + environment.names[environment.count] = name + environment.values[environment.count] = value + environment.count++ + + return value + } + + if environment.overflow == nil { + environment.overflow = make(map[string]Object) + } + + environment.overflow[name] = value return value } func (environment *Environment) Delete(name string) { - delete(environment.store, name) + for index := 0; index < environment.count; index++ { + if environment.names[index] != name { + continue + } + + // Order is not meaningful here, so close the gap with the last entry + // and clear it so the removed value can be collected. + last := environment.count - 1 + + environment.names[index] = environment.names[last] + environment.values[index] = environment.values[last] + environment.names[last] = "" + environment.values[last] = nil + environment.count-- + + return + } + + delete(environment.overflow, name) } func (environment *Environment) SetWriter(writer io.Writer) { diff --git a/object/error.go b/object/error.go index 0a80973..0ca6504 100644 --- a/object/error.go +++ b/object/error.go @@ -4,8 +4,6 @@ import ( "fmt" ) -const ERROR = "ERROR" - // Error objects consist of a nil value. type Error struct { Message string diff --git a/object/function.go b/object/function.go index 6b4c87b..a3f247b 100644 --- a/object/function.go +++ b/object/function.go @@ -6,8 +6,6 @@ import ( "ghostlang.org/x/ghost/ast" ) -const FUNCTION = "FUNCTION" - // Function objects consist of a user-generated function. type Function struct { Parameters []*ast.Identifier diff --git a/object/instance.go b/object/instance.go index bfd1399..19a0586 100644 --- a/object/instance.go +++ b/object/instance.go @@ -6,8 +6,6 @@ import ( "ghostlang.org/x/ghost/token" ) -const INSTANCE = "INSTANCE" - // Instance objects consist of a body and an environment. type Instance struct { Class *Class diff --git a/object/library_function.go b/object/library_function.go index b8508f4..867f581 100644 --- a/object/library_function.go +++ b/object/library_function.go @@ -2,8 +2,6 @@ package object import "fmt" -const LIBRARY_FUNCTION = "LIBRARY_FUNCTION" - // LibraryFunction objects consist of a native Go function. type LibraryFunction struct { Name string diff --git a/object/library_module.go b/object/library_module.go index 6127298..b23a10b 100644 --- a/object/library_module.go +++ b/object/library_module.go @@ -1,7 +1,5 @@ package object -const LIBRARY_MODULE = "LIBRARY_MODULE" - // LibraryModule objects consist of a slice of LibraryFunctions. type LibraryModule struct { Name string diff --git a/object/library_property.go b/object/library_property.go index 29bc3ba..3f4daf4 100644 --- a/object/library_property.go +++ b/object/library_property.go @@ -2,8 +2,6 @@ package object import "fmt" -const LIBRARY_PROPERTY = "LIBRARY_PROPERTY" - // LibraryProperty objects consist of a native Go property. type LibraryProperty struct { Name string diff --git a/object/list.go b/object/list.go index b0d42af..fd154c1 100644 --- a/object/list.go +++ b/object/list.go @@ -5,8 +5,6 @@ import ( "strings" ) -const LIST = "LIST" - // List objects consist of a nil value. type List struct { Elements []Object @@ -105,16 +103,9 @@ func (list *List) pop(args []Object) (Object, bool) { } func (list *List) push(args []Object) (Object, bool) { - length := len(list.Elements) - newLength := length + 1 - - newElements := make([]Object, newLength) - copy(newElements, list.Elements) - newElements[length] = args[0] + list.Elements = append(list.Elements, args[0]) - list.Elements = newElements - - return NewInt(int64(newLength)), true + return NewInt(int64(len(list.Elements))), true } func (list *List) tail(args []Object) (Object, bool) { diff --git a/object/map.go b/object/map.go index efdce14..6ee284f 100644 --- a/object/map.go +++ b/object/map.go @@ -6,8 +6,6 @@ import ( "strings" ) -const MAP = "MAP" - // Map objects consist of a map value. type Map struct { Pairs map[MapKey]MapPair diff --git a/object/null.go b/object/null.go index 6e0c077..16efc81 100644 --- a/object/null.go +++ b/object/null.go @@ -1,7 +1,5 @@ package object -const NULL = "NULL" - // Null objects consist of a nil value. type Null struct{} diff --git a/object/number.go b/object/number.go index 2e2c29a..dbe1be8 100644 --- a/object/number.go +++ b/object/number.go @@ -6,8 +6,6 @@ import ( "strconv" ) -const NUMBER = "NUMBER" - // Number objects represent numeric values using either int64 or float64 internally. // Integer operations stay as int64 for speed and exactness. // Float operations use float64. Division always promotes to float. @@ -17,8 +15,31 @@ type Number struct { isFloat bool } -// NewInt creates a new integer Number. +// Bounds of the preallocated small-integer cache. Loop counters, indices, and +// list lengths overwhelmingly fall in this range, so interning them removes a +// heap allocation from nearly every arithmetic operation. Number is immutable +// (its fields are unexported and never reassigned after construction), which is +// what makes sharing these instances safe. +const ( + smallIntMin = -128 + smallIntMax = 1024 +) + +var smallInts [smallIntMax - smallIntMin + 1]Number + +func init() { + for index := range smallInts { + smallInts[index] = Number{i: int64(index + smallIntMin)} + } +} + +// NewInt creates a new integer Number, returning a shared instance for small +// values. func NewInt(i int64) *Number { + if i >= smallIntMin && i <= smallIntMax { + return &smallInts[i-smallIntMin] + } + return &Number{i: i} } diff --git a/object/object.go b/object/object.go index 7a494b9..baa156e 100644 --- a/object/object.go +++ b/object/object.go @@ -7,8 +7,62 @@ import ( var evaluator func(node ast.Node, scope *Scope) Object -// Type is the type of the object given as a string. -type Type string +// Type identifies the runtime type of an object. It is an integer rather than a +// string so that the type comparisons the evaluator performs on every operation +// are single-word compares instead of string compares, and so that MapKey hashes +// over a machine word. String() keeps the human-readable name for error messages. +type Type int + +const ( + BOOLEAN Type = iota + BREAK + CLASS + CONTINUE + ERROR + FUNCTION + INSTANCE + LIBRARY_FUNCTION + LIBRARY_MODULE + LIBRARY_PROPERTY + LIST + MAP + NULL + NUMBER + RETURN + SCOPE + STRING + TRAIT +) + +var typeNames = [...]string{ + BOOLEAN: "BOOLEAN", + BREAK: "BREAK", + CLASS: "CLASS", + CONTINUE: "CONTINUE", + ERROR: "ERROR", + FUNCTION: "FUNCTION", + INSTANCE: "INSTANCE", + LIBRARY_FUNCTION: "LIBRARY_FUNCTION", + LIBRARY_MODULE: "LIBRARY_MODULE", + LIBRARY_PROPERTY: "LIBRARY_PROPERTY", + LIST: "LIST", + MAP: "MAP", + NULL: "NULL", + NUMBER: "NUMBER", + RETURN: "RETURN", + SCOPE: "SCOPE", + STRING: "STRING", + TRAIT: "TRAIT", +} + +// String returns the name of the type, as used in runtime error messages. +func (t Type) String() string { + if int(t) < 0 || int(t) >= len(typeNames) { + return "UNKNOWN" + } + + return typeNames[t] +} // Object is the interface for all object values. type Object interface { diff --git a/object/return.go b/object/return.go index b282aeb..ecceafa 100644 --- a/object/return.go +++ b/object/return.go @@ -1,7 +1,5 @@ package object -const RETURN = "RETURN" - // Return objects consist of a value. type Return struct { Value Object diff --git a/object/scope.go b/object/scope.go index d8253af..3823ccd 100644 --- a/object/scope.go +++ b/object/scope.go @@ -1,7 +1,5 @@ package object -const SCOPE = "SCOPE" - // Scope objects consist of an environment and parent object. type Scope struct { Environment *Environment diff --git a/object/string.go b/object/string.go index 3947515..4a279e8 100644 --- a/object/string.go +++ b/object/string.go @@ -9,8 +9,6 @@ import ( "unicode/utf8" ) -const STRING = "STRING" - // String objects consist of a string value. type String struct { Value string diff --git a/object/trait.go b/object/trait.go index abc2073..0e9dad4 100644 --- a/object/trait.go +++ b/object/trait.go @@ -2,8 +2,6 @@ package object import "ghostlang.org/x/ghost/ast" -const TRAIT = "TRAIT" - // Trait objects consist of a body and an environment. type Trait struct { Name *ast.Identifier diff --git a/optimizer/fold.go b/optimizer/fold.go new file mode 100644 index 0000000..8bd4f1d --- /dev/null +++ b/optimizer/fold.go @@ -0,0 +1,257 @@ +package optimizer + +import ( + "math" + + "ghostlang.org/x/ghost/ast" + "ghostlang.org/x/ghost/token" +) + +// The fold rules below mirror evaluator/number.go, evaluator/string.go, +// evaluator/boolean.go and evaluator/prefix.go exactly. Where the evaluator +// would report a runtime error the expression is deliberately left unfolded so +// that the error still happens, with its original position, at run time. + +// foldInfix returns the literal an infix expression collapses to, or nil when +// it cannot be folded. +func foldInfix(node *ast.Infix) ast.ExpressionNode { + switch left := node.Left.(type) { + case *ast.Number: + right, ok := node.Right.(*ast.Number) + if !ok { + return nil + } + + return foldNumberInfix(node, left, right) + + case *ast.String: + right, ok := node.Right.(*ast.String) + if !ok { + return nil + } + + return foldStringInfix(node, left, right) + + case *ast.Boolean: + right, ok := node.Right.(*ast.Boolean) + if !ok { + return nil + } + + return foldBooleanInfix(node, left, right) + } + + return nil +} + +func foldNumberInfix(node *ast.Infix, left, right *ast.Number) ast.ExpressionNode { + // Integer operands stay integral; a float on either side promotes the + // result, matching object.Number. + isFloat := left.IsFloat || right.IsFloat + leftFloat, rightFloat := floatOf(left), floatOf(right) + + switch node.Operator { + case token.PLUS: + if isFloat { + return floatNode(node, leftFloat+rightFloat) + } + + return intNode(node, left.IntValue+right.IntValue) + + case token.MINUS: + if isFloat { + return floatNode(node, leftFloat-rightFloat) + } + + return intNode(node, left.IntValue-right.IntValue) + + case token.STAR: + if isFloat { + return floatNode(node, leftFloat*rightFloat) + } + + return intNode(node, left.IntValue*right.IntValue) + + case token.SLASH: + // Division by zero is a runtime error, so leave it for the evaluator. + // Division always promotes to float, matching object.Number.Div. + if isZero(right) { + return nil + } + + return floatNode(node, leftFloat/rightFloat) + + case token.PERCENT: + if isZero(right) { + return nil + } + + if isFloat { + return floatNode(node, math.Mod(leftFloat, rightFloat)) + } + + return intNode(node, left.IntValue%right.IntValue) + + case token.LESS: + if isFloat { + return booleanNode(node, leftFloat < rightFloat) + } + + return booleanNode(node, left.IntValue < right.IntValue) + + case token.LESSEQUAL: + if isFloat { + return booleanNode(node, leftFloat <= rightFloat) + } + + return booleanNode(node, left.IntValue <= right.IntValue) + + case token.GREATER: + if isFloat { + return booleanNode(node, leftFloat > rightFloat) + } + + return booleanNode(node, left.IntValue > right.IntValue) + + case token.GREATEREQUAL: + if isFloat { + return booleanNode(node, leftFloat >= rightFloat) + } + + return booleanNode(node, left.IntValue >= right.IntValue) + + case token.EQUALEQUAL: + if isFloat { + return booleanNode(node, leftFloat == rightFloat) + } + + return booleanNode(node, left.IntValue == right.IntValue) + + case token.BANGEQUAL: + if isFloat { + return booleanNode(node, leftFloat != rightFloat) + } + + return booleanNode(node, left.IntValue != right.IntValue) + } + + // token.DOTDOT is deliberately absent: a range builds a list object, and a + // folded literal would have to be a shared mutable value. + return nil +} + +func foldStringInfix(node *ast.Infix, left, right *ast.String) ast.ExpressionNode { + switch node.Operator { + case token.PLUS: + return &ast.String{Token: node.Token, Value: left.Value + right.Value} + case token.LESS: + return booleanNode(node, left.Value < right.Value) + case token.LESSEQUAL: + return booleanNode(node, left.Value <= right.Value) + case token.GREATER: + return booleanNode(node, left.Value > right.Value) + case token.GREATEREQUAL: + return booleanNode(node, left.Value >= right.Value) + case token.EQUALEQUAL: + return booleanNode(node, left.Value == right.Value) + case token.BANGEQUAL: + return booleanNode(node, left.Value != right.Value) + } + + return nil +} + +func foldBooleanInfix(node *ast.Infix, left, right *ast.Boolean) ast.ExpressionNode { + // Ghost evaluates both sides of and/or before dispatching, so folding two + // literal operands cannot skip a side effect. + switch node.Operator { + case token.AND: + return booleanNode(node, left.Value && right.Value) + case token.OR: + return booleanNode(node, left.Value || right.Value) + case token.EQUALEQUAL: + return booleanNode(node, left.Value == right.Value) + case token.BANGEQUAL: + return booleanNode(node, left.Value != right.Value) + } + + return nil +} + +// foldPrefix returns the literal a prefix expression collapses to, or nil when +// it cannot be folded. +func foldPrefix(node *ast.Prefix) ast.ExpressionNode { + switch node.Operator { + case token.MINUS: + // Negation is only defined for numbers; anything else is a runtime + // error and is left alone. + number, ok := node.Right.(*ast.Number) + if !ok { + return nil + } + + if number.IsFloat { + return floatNode(node, -number.FloatValue) + } + + return intNode(node, -number.IntValue) + + case token.BANG: + switch right := node.Right.(type) { + case *ast.Boolean: + return booleanNode(node, !right.Value) + case *ast.Null: + return booleanNode(node, true) + case *ast.Number, *ast.String: + // Matches the evaluator, where bang over a non-boolean, + // non-null operand is false. + return booleanNode(node, false) + } + } + + return nil +} + +// ============================================================================= +// Literal node constructors. Each keeps the original expression's token so that +// positions reported for the surrounding code stay meaningful. + +// nodeToken recovers the token to attach to a folded literal. +func nodeToken(node ast.Node) token.Token { + switch node := node.(type) { + case *ast.Infix: + return node.Token + case *ast.Prefix: + return node.Token + } + + return token.Token{} +} + +func intNode(node ast.Node, value int64) *ast.Number { + return &ast.Number{Token: nodeToken(node), IntValue: value} +} + +func floatNode(node ast.Node, value float64) *ast.Number { + return &ast.Number{Token: nodeToken(node), FloatValue: value, IsFloat: true} +} + +func booleanNode(node ast.Node, value bool) *ast.Boolean { + return &ast.Boolean{Token: nodeToken(node), Value: value} +} + +func floatOf(number *ast.Number) float64 { + if number.IsFloat { + return number.FloatValue + } + + return float64(number.IntValue) +} + +func isZero(number *ast.Number) bool { + if number.IsFloat { + return number.FloatValue == 0 + } + + return number.IntValue == 0 +} diff --git a/optimizer/optimizer.go b/optimizer/optimizer.go new file mode 100644 index 0000000..100d4b7 --- /dev/null +++ b/optimizer/optimizer.go @@ -0,0 +1,252 @@ +// Package optimizer rewrites the AST after parsing and before evaluation. +// +// The tree-walking evaluator re-evaluates every node each time control reaches +// it, so an expression built entirely from literals is recomputed on every +// iteration of a loop and every call of a function. Folding those expressions +// into a single literal node at parse time removes that work permanently. +// +// The pass is conservative: it only rewrites a node when the rewritten form is +// guaranteed to produce the same value and the same errors as evaluating the +// original. Anything it does not recognize is left untouched, so failing to +// fold is always safe. +package optimizer + +import ( + "ghostlang.org/x/ghost/ast" +) + +// globalResolver reports whether a name refers to a library global. The library +// package installs it during initialization. It is a hook rather than a direct +// dependency because library/modules already depends on this package, so +// importing library here would close an import cycle. +// +// When it is nil, identifiers are left unclassified and the evaluator falls +// back to consulting the registries itself. +var globalResolver func(name string) bool + +// SetGlobalResolver installs the function used to classify identifiers. +func SetGlobalResolver(resolver func(name string) bool) { + globalResolver = resolver +} + +// Optimize rewrites a parsed program in place and returns it. +func Optimize(program *ast.Program) *ast.Program { + if program == nil { + return nil + } + + for index, statement := range program.Statements { + program.Statements[index] = optimize(statement) + } + + return program +} + +// optimize walks a node, optimizing its children first so that folding sees +// already-folded operands, then attempts to fold the node itself. +func optimize(node ast.Node) ast.Node { + switch node := node.(type) { + case *ast.Program: + return Optimize(node) + + case *ast.Block: + if node == nil { + return node + } + + for index, statement := range node.Statements { + node.Statements[index] = optimize(statement) + } + + return node + + case *ast.Expression: + node.Expression = optimize(node.Expression) + + return node + + case *ast.Identifier: + // Classify the name once so the evaluator can skip the library + // registries for ordinary variables. + if globalResolver != nil { + if globalResolver(node.Value) { + node.LibraryBinding = ast.LibraryBindingGlobal + } else { + node.LibraryBinding = ast.LibraryBindingLocal + } + } + + return node + + case *ast.Infix: + node.Left = optimize(node.Left) + node.Right = optimize(node.Right) + + if folded := foldInfix(node); folded != nil { + return folded + } + + return node + + case *ast.Prefix: + node.Right = optimize(node.Right) + + if folded := foldPrefix(node); folded != nil { + return folded + } + + return node + + case *ast.Ternary: + node.Condition = optimize(node.Condition) + node.IfTrue = optimize(node.IfTrue) + node.IfFalse = optimize(node.IfFalse) + + return node + + case *ast.Assign: + node.Value = optimize(node.Value) + + return node + + case *ast.Compound: + node.Right = optimize(node.Right) + + return node + + case *ast.Call: + node.Callee = optimize(node.Callee) + optimizeExpressions(node.Arguments) + + return node + + case *ast.Method: + node.Left = optimize(node.Left) + optimizeExpressions(node.Arguments) + + return node + + case *ast.Property: + node.Left = optimize(node.Left) + + return node + + case *ast.Index: + node.Left = optimize(node.Left) + node.Index = optimize(node.Index) + + return node + + case *ast.List: + optimizeExpressions(node.Elements) + + return node + + case *ast.Map: + // Map keys are the map's own keys, so a folded key would need the entry + // reinserted. Values are rewritten in place, which is enough in + // practice and keeps the pass simple. + for key := range node.Pairs { + node.Pairs[key] = optimize(node.Pairs[key]) + } + + return node + + case *ast.If: + node.Condition = optimize(node.Condition) + node.Consequence = optimizeBlock(node.Consequence) + node.Alternative = optimizeBlock(node.Alternative) + + return node + + case *ast.While: + node.Condition = optimize(node.Condition) + node.Consequence = optimizeBlock(node.Consequence) + + return node + + case *ast.For: + if node.Initializer != nil { + node.Initializer = optimize(node.Initializer) + } + + if node.Condition != nil { + node.Condition = optimize(node.Condition) + } + + if node.Increment != nil { + node.Increment = optimize(node.Increment) + } + + node.Block = optimizeBlock(node.Block) + + return node + + case *ast.ForIn: + node.Iterable = optimize(node.Iterable) + node.Block = optimizeBlock(node.Block) + + return node + + case *ast.Switch: + node.Value = optimize(node.Value) + + for _, branch := range node.Cases { + if branch == nil { + continue + } + + optimizeExpressions(branch.Value) + + branch.Body = optimizeBlock(branch.Body) + } + + return node + + case *ast.Function: + for name, def := range node.Defaults { + node.Defaults[name] = optimize(def) + } + + node.Body = optimizeBlock(node.Body) + + return node + + case *ast.Class: + node.Body = optimizeBlock(node.Body) + + return node + + case *ast.Trait: + node.Body = optimizeBlock(node.Body) + + return node + + case *ast.Return: + if node.Value != nil { + node.Value = optimize(node.Value) + } + + return node + } + + return node +} + +func optimizeBlock(block *ast.Block) *ast.Block { + if block == nil { + return nil + } + + for index, statement := range block.Statements { + block.Statements[index] = optimize(statement) + } + + return block +} + +func optimizeExpressions(expressions []ast.ExpressionNode) { + for index, expression := range expressions { + expressions[index] = optimize(expression) + } +} diff --git a/optimizer/optimizer_test.go b/optimizer/optimizer_test.go new file mode 100644 index 0000000..80253fe --- /dev/null +++ b/optimizer/optimizer_test.go @@ -0,0 +1,241 @@ +package optimizer + +import ( + "testing" + + "ghostlang.org/x/ghost/ast" + "ghostlang.org/x/ghost/parser" + "ghostlang.org/x/ghost/scanner" +) + +func optimizeSource(t *testing.T, source string) ast.Node { + t.Helper() + + p := parser.New(scanner.New(source, "test.ghost")) + program := Optimize(p.Parse()) + + if len(p.Errors()) != 0 { + t.Fatalf("parser errors for %q: %v", source, p.Errors()) + } + + if len(program.Statements) != 1 { + t.Fatalf("expected 1 statement for %q, got %d", source, len(program.Statements)) + } + + statement, ok := program.Statements[0].(*ast.Expression) + + if !ok { + t.Fatalf("statement is not ast.Expression for %q. got=%T", source, program.Statements[0]) + } + + return statement.Expression +} + +func TestFoldsIntegerArithmetic(t *testing.T) { + tests := []struct { + source string + expected int64 + }{ + {"1 + 2", 3}, + {"10 - 4", 6}, + {"6 * 7", 42}, + {"7 % 3", 1}, + {"-5", -5}, + {"2 * 3 + 4 * 5 - 6", 20}, + {"1 + 2 + 3 + 4 + 5", 15}, + {"-(3 * 4)", -12}, + } + + for _, tt := range tests { + folded := optimizeSource(t, tt.source) + + number, ok := folded.(*ast.Number) + + if !ok { + t.Errorf("%q did not fold to a number. got=%T", tt.source, folded) + continue + } + + if number.IsFloat { + t.Errorf("%q folded to a float, expected an integer", tt.source) + continue + } + + if number.IntValue != tt.expected { + t.Errorf("%q folded to %d, expected %d", tt.source, number.IntValue, tt.expected) + } + } +} + +func TestFoldsFloatArithmetic(t *testing.T) { + tests := []struct { + source string + expected float64 + }{ + {"1.5 + 2.5", 4}, + {"3.0 * 2.0", 6}, + {"1 + 2.5", 3.5}, + // Division always promotes to a float, matching object.Number.Div. + {"6 / 3", 2}, + {"7 / 2", 3.5}, + } + + for _, tt := range tests { + folded := optimizeSource(t, tt.source) + + number, ok := folded.(*ast.Number) + + if !ok { + t.Errorf("%q did not fold to a number. got=%T", tt.source, folded) + continue + } + + if !number.IsFloat { + t.Errorf("%q folded to an integer, expected a float", tt.source) + continue + } + + if number.FloatValue != tt.expected { + t.Errorf("%q folded to %v, expected %v", tt.source, number.FloatValue, tt.expected) + } + } +} + +func TestFoldsComparisonsAndBooleans(t *testing.T) { + tests := []struct { + source string + expected bool + }{ + {"1 < 2", true}, + {"2 <= 2", true}, + {"3 > 4", false}, + {"1 == 1", true}, + {"1 != 1", false}, + {`"a" == "a"`, true}, + {`"a" < "b"`, true}, + {"true and false", false}, + {"true or false", true}, + {"!true", false}, + {"!false", true}, + {"!null", true}, + {"!(1 == 2)", true}, + } + + for _, tt := range tests { + folded := optimizeSource(t, tt.source) + + boolean, ok := folded.(*ast.Boolean) + + if !ok { + t.Errorf("%q did not fold to a boolean. got=%T", tt.source, folded) + continue + } + + if boolean.Value != tt.expected { + t.Errorf("%q folded to %t, expected %t", tt.source, boolean.Value, tt.expected) + } + } +} + +func TestFoldsStringConcatenation(t *testing.T) { + folded := optimizeSource(t, `"hello" + " " + "world"`) + + str, ok := folded.(*ast.String) + + if !ok { + t.Fatalf("string concatenation did not fold. got=%T", folded) + } + + if str.Value != "hello world" { + t.Errorf("folded to %q, expected %q", str.Value, "hello world") + } +} + +// Expressions that would raise a runtime error, or that depend on values only +// known at run time, must be left alone so that the evaluator still sees them. +func TestLeavesUnfoldableExpressions(t *testing.T) { + tests := []string{ + "1 / 0", // division by zero is a runtime error + "1 % 0", // modulo by zero is a runtime error + "1.0 / 0.0", // float division by zero is a runtime error too + "1 + true", // type mismatch is a runtime error + `1 + "a"`, // type mismatch is a runtime error + "1 .. 5", // a range builds a mutable list object + "a + 1", // depends on a runtime variable + "-true", // negating a non-number is a runtime error + "foo() + 1", // calls may have side effects + "[1, 2] + [3]", // lists are not folded + } + + for _, source := range tests { + folded := optimizeSource(t, source) + + switch folded.(type) { + case *ast.Number, *ast.String, *ast.Boolean: + t.Errorf("%q was folded to a literal (%T) but should have been left alone", source, folded) + } + } +} + +// Folding must reach into every construct that contains expressions, not just +// the top level of a program. +func TestFoldsInsideNestedConstructs(t *testing.T) { + source := `function outer() { + if (1 + 1 == 2) { + for (i = 0; i < 2 * 5; i = i + 1) { + total = total + (3 * 4) + } + } + return 8 - 3 + }` + + p := parser.New(scanner.New(source, "test.ghost")) + program := Optimize(p.Parse()) + + if len(p.Errors()) != 0 { + t.Fatalf("parser errors: %v", p.Errors()) + } + + // Walk the tree and assert no foldable infix expression survives. + var unfolded []string + + var walk func(node ast.Node) + walk = func(node ast.Node) { + switch node := node.(type) { + case *ast.Program: + for _, s := range node.Statements { + walk(s) + } + case *ast.Block: + for _, s := range node.Statements { + walk(s) + } + case *ast.Expression: + walk(node.Expression) + case *ast.Function: + walk(node.Body) + case *ast.If: + walk(node.Condition) + walk(node.Consequence) + case *ast.For: + walk(node.Condition) + walk(node.Block) + case *ast.Assign: + walk(node.Value) + case *ast.Return: + walk(node.Value) + case *ast.Infix: + if foldInfix(node) != nil { + unfolded = append(unfolded, "infix survived folding") + } + walk(node.Left) + walk(node.Right) + } + } + + walk(program) + + if len(unfolded) != 0 { + t.Errorf("found %d foldable expressions that were not folded", len(unfolded)) + } +} diff --git a/parser/compound.go b/parser/compound.go index a49d096..941e8ef 100644 --- a/parser/compound.go +++ b/parser/compound.go @@ -7,7 +7,7 @@ import ( func (parser *Parser) compoundExpression(left ast.ExpressionNode) ast.ExpressionNode { compound := &ast.Compound{ Token: parser.currentToken, - Operator: parser.currentToken.Lexeme, + Operator: parser.currentToken.Type, Left: left, } diff --git a/parser/infix.go b/parser/infix.go index a100c3e..15922df 100644 --- a/parser/infix.go +++ b/parser/infix.go @@ -7,7 +7,7 @@ import ( func (parser *Parser) infixExpression(left ast.ExpressionNode) ast.ExpressionNode { infix := &ast.Infix{ Token: parser.currentToken, - Operator: parser.currentToken.Lexeme, + Operator: parser.currentToken.Type, Left: left, } diff --git a/parser/parser_test.go b/parser/parser_test.go index 2f5accd..0c254a2 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -342,7 +342,7 @@ func TestInfixExpressions(t *testing.T) { t.Fatalf("statement is not ast.Infix. got=%T", statement.Expression) } - if infix.Operator != tt.operator { + if infix.Operator.String() != tt.operator { t.Fatalf("infix.Operator is not '%s'. got=%s", tt.operator, infix.Operator) } @@ -358,10 +358,10 @@ func TestInfixExpressions(t *testing.T) { func TestNumberLiteral(t *testing.T) { tests := []struct { - input string - isFloat bool - intValue int64 - floatValue float64 + input string + isFloat bool + intValue int64 + floatValue float64 }{ {"5", false, 5, 0}, {"3.14", true, 0, 3.14}, @@ -440,7 +440,7 @@ func TestPrefixExpressions(t *testing.T) { t.Fatalf("statement is not ast.Prefix. got=%T", statement.Expression) } - if prefix.Operator != tt.operator { + if prefix.Operator.String() != tt.operator { t.Fatalf("prefix.Operator is not '%s'. got=%s", tt.operator, prefix.Operator) } @@ -482,7 +482,7 @@ func TestPostfixExpressions(t *testing.T) { t.Fatalf("statement is not ast.Postfix. got=%T", statement.Expression) } - if postfix.Operator != tt.operator { + if postfix.Operator.String() != tt.operator { t.Fatalf("postfix.Operator is not '%s'. got=%s", tt.operator, postfix.Operator) } } @@ -992,7 +992,7 @@ func isInfixExpression(t *testing.T, expression ast.ExpressionNode, left interfa return false } - if operatorExpression.Operator != operator { + if operatorExpression.Operator.String() != operator { t.Errorf("expression.Operator is not '%s'. got=%q", operator, operatorExpression.Operator) return false } diff --git a/parser/postfix.go b/parser/postfix.go index bb6c1b1..64492d5 100644 --- a/parser/postfix.go +++ b/parser/postfix.go @@ -5,6 +5,6 @@ import "ghostlang.org/x/ghost/ast" func (parser *Parser) postfixExpression() ast.ExpressionNode { return &ast.Postfix{ Token: parser.previousToken, - Operator: parser.currentToken.Lexeme, + Operator: parser.currentToken.Type, } } diff --git a/parser/prefix.go b/parser/prefix.go index 74ff280..49d6a22 100644 --- a/parser/prefix.go +++ b/parser/prefix.go @@ -5,7 +5,7 @@ import "ghostlang.org/x/ghost/ast" func (parser *Parser) prefixExpression() ast.ExpressionNode { prefix := &ast.Prefix{ Token: parser.currentToken, - Operator: parser.currentToken.Lexeme, + Operator: parser.currentToken.Type, } parser.readToken() diff --git a/token/token.go b/token/token.go index 08dc318..1cc0255 100644 --- a/token/token.go +++ b/token/token.go @@ -2,8 +2,11 @@ package token import "fmt" -// Type is the type of the given token as a string. -type Type string +// Type identifies the kind of a token. It is an integer rather than a string so +// that the parser's precedence lookups and the evaluator's operator dispatch +// compare single words instead of strings, and so operator switches compile to +// jump tables. String() recovers the source spelling for error messages. +type Type int // Token contains the lexeme read by the scanner. type Token struct { @@ -21,73 +24,153 @@ func (token *Token) String() string { const ( // single-character tokens - COLON = ":" - COMMA = "," - LEFTBRACE = "{" - LEFTBRACKET = "[" - LEFTPAREN = "(" - MINUS = "-" - PLUS = "+" - QUESTION = "?" - RIGHTBRACE = "}" - RIGHTBRACKET = "]" - RIGHTPAREN = ")" - SEMICOLON = ";" - SLASH = "/" - STAR = "*" - PERCENT = "%" + COLON Type = iota + COMMA + LEFTBRACE + LEFTBRACKET + LEFTPAREN + MINUS + PLUS + QUESTION + RIGHTBRACE + RIGHTBRACKET + RIGHTPAREN + SEMICOLON + SLASH + STAR + PERCENT // one or two character tokens - BANG = "!" - BANGEQUAL = "!=" - DOT = "." - DOTDOT = ".." - EQUAL = "=" - EQUALEQUAL = "==" - GREATER = ">" - GREATEREQUAL = ">=" - LESS = "<" - LESSEQUAL = "<=" - PLUSEQUAL = "+=" - PLUSPLUS = "++" - MINUSEQUAL = "-=" - MINUSMINUS = "--" - STAREQUAL = "*=" - SLASHEQUAL = "/=" + BANG + BANGEQUAL + DOT + DOTDOT + EQUAL + EQUALEQUAL + GREATER + GREATEREQUAL + LESS + LESSEQUAL + PLUSEQUAL + PLUSPLUS + MINUSEQUAL + MINUSMINUS + STAREQUAL + SLASHEQUAL // literals - IDENTIFIER = "IDENTIFIER" - STRING = "STRING" - NUMBER = "NUMBER" + IDENTIFIER + STRING + NUMBER // keywords - AND = "and" - AS = "as" - BREAK = "break" - CASE = "case" - CLASS = "class" - CONTINUE = "continue" - DEFAULT = "default" - ELSE = "else" - EXTENDS = "extends" - FALSE = "false" - FOR = "for" - FROM = "from" - FUNCTION = "function" - IF = "if" - IMPORT = "import" - IN = "in" - NULL = "null" - OR = "or" - PRINT = "print" - RETURN = "return" - SUPER = "super" - SWITCH = "switch" - THIS = "this" - TRAIT = "trait" - TRUE = "true" - USE = "use" - WHILE = "while" - EOF = "eof" - INVALID = "__INVALID__" + AND + AS + BREAK + CASE + CLASS + CONTINUE + DEFAULT + ELSE + EXTENDS + FALSE + FOR + FROM + FUNCTION + IF + IMPORT + IN + NULL + OR + PRINT + RETURN + SUPER + SWITCH + THIS + TRAIT + TRUE + USE + WHILE + EOF + INVALID ) + +// typeNames maps each token type to its source spelling. These strings are what +// appear in parser and runtime error messages. +var typeNames = [...]string{ + COLON: ":", + COMMA: ",", + LEFTBRACE: "{", + LEFTBRACKET: "[", + LEFTPAREN: "(", + MINUS: "-", + PLUS: "+", + QUESTION: "?", + RIGHTBRACE: "}", + RIGHTBRACKET: "]", + RIGHTPAREN: ")", + SEMICOLON: ";", + SLASH: "/", + STAR: "*", + PERCENT: "%", + + BANG: "!", + BANGEQUAL: "!=", + DOT: ".", + DOTDOT: "..", + EQUAL: "=", + EQUALEQUAL: "==", + GREATER: ">", + GREATEREQUAL: ">=", + LESS: "<", + LESSEQUAL: "<=", + PLUSEQUAL: "+=", + PLUSPLUS: "++", + MINUSEQUAL: "-=", + MINUSMINUS: "--", + STAREQUAL: "*=", + SLASHEQUAL: "/=", + + IDENTIFIER: "IDENTIFIER", + STRING: "STRING", + NUMBER: "NUMBER", + + AND: "and", + AS: "as", + BREAK: "break", + CASE: "case", + CLASS: "class", + CONTINUE: "continue", + DEFAULT: "default", + ELSE: "else", + EXTENDS: "extends", + FALSE: "false", + FOR: "for", + FROM: "from", + FUNCTION: "function", + IF: "if", + IMPORT: "import", + IN: "in", + NULL: "null", + OR: "or", + PRINT: "print", + RETURN: "return", + SUPER: "super", + SWITCH: "switch", + THIS: "this", + TRAIT: "trait", + TRUE: "true", + USE: "use", + WHILE: "while", + EOF: "eof", + INVALID: "__INVALID__", +} + +// String returns the source spelling of the token type. +func (t Type) String() string { + if int(t) < 0 || int(t) >= len(typeNames) { + return "__INVALID__" + } + + return typeNames[t] +}