VM performance pass: pooled VMs, slab-allocated numbers, fast paths, stack-overflow fix

bytecode.go

@@ -14,6 +14,7 @@ type Bytecode struct {
 	FileSet      *parser.SourceFileSet
 	MainFunction *CompiledFunction
 	Constants    []Object
+	prepared     bool // functions translated for the VM, see prepare
 }
 
 // Size of the bytecode in bytes
@@ -32,9 +33,30 @@ func (b *Bytecode) Clone() *Bytecode {
 		FileSet:      b.FileSet,
 		MainFunction: b.MainFunction,
 		Constants:    append([]Object{}, b.Constants...),
+		prepared:     b.prepared,
 	}
 }
 
+// prepare translates every function into the form the VM executes (see
+// vm_code.go). Script.Compile and Decode call it once; VM.init calls it for
+// bytecode assembled by hand, where the first run pays for the translation.
+func (b *Bytecode) prepare() {
+	if b.prepared {
+		return
+	}
+	var tr translator
+	if b.MainFunction.code == nil {
+		b.MainFunction.code = tr.translate(b.MainFunction.Instructions,
+			b.MainFunction.NumLocals)
+	}
+	for _, c := range b.Constants {
+		if fn, ok := c.(*CompiledFunction); ok && fn.code == nil {
+			fn.code = tr.translate(fn.Instructions, fn.NumLocals)
+		}
+	}
+	b.prepared = true
+}
+
 // Encode writes Bytecode data to the writer.
 func (b *Bytecode) Encode(w io.Writer) error {
 	enc := gob.NewEncoder(w)
@@ -122,6 +144,12 @@ func (b *Bytecode) Decode(r io.Reader, modules *ModuleMap) error {
 		}
 		b.Constants[i] = fv
 	}
+	// the main function is not in Constants; it needs the same repair
+	if _, err := fixDecodedObject(b.MainFunction, modules); err != nil {
+		return err
+	}
+	b.prepared = false // an earlier Decode may have prepared this Bytecode
+	b.prepare()
 	return nil
 }
 
@@ -209,13 +237,16 @@ func (b *Bytecode) RemoveDuplicates() {
 	// update CONST instructions with new indexes
 	// main function
 	updateConstIndexes(b.MainFunction.Instructions, indexMap)
+	b.MainFunction.code = nil
 	// other compiled functions in constants
 	for _, c := range b.Constants {
 		switch c := c.(type) {
 		case *CompiledFunction:
 			updateConstIndexes(c.Instructions, indexMap)
+			c.code = nil // instructions changed; retranslate on prepare
 		}
 	}
+	b.prepared = false
 }
 
 func fixDecodedObject(
@@ -230,6 +261,10 @@ func fixDecodedObject(
 		return TrueValue, nil
 	case *Undefined:
 		return UndefinedValue, nil
+	case *CompiledFunction:
+		// not serialized; recompute so the VM can bound the operand stack
+		o.stackDepth = maxStackDepth(o.Instructions)
+		o.code = nil
 	case *Array:
 		for i, v := range o.Value {
 			fv, err := fixDecodedObject(v, modules)

bytecode_test.go

@@ -2,6 +2,7 @@ package tengo_test
 
 import (
 	"bytes"
+	"strings"
 	"testing"
 	"time"
 
@@ -296,3 +297,50 @@ func testBytecodeSerialization(t *testing.T, b *tengo.Bytecode) {
 	require.Equal(t, b.MainFunction, r.MainFunction)
 	require.Equal(t, b.Constants, r.Constants)
 }
+
+func TestBytecodeDecodeTwice(t *testing.T) {
+	// decoding into a Bytecode that an earlier Decode already prepared must
+	// translate the newly decoded functions too
+	first := compileForDecode(t, `f := func() { return 1 }; out = f()`)
+	second := compileForDecode(t, `g := func(x) { return x * 2 }; out = g(21)`)
+	var b tengo.Bytecode
+	decodeRoundTrip(t, first, &b)
+	decodeRoundTrip(t, second, &b)
+
+	globals := make([]tengo.Object, tengo.GlobalsSize)
+	require.NoError(t, tengo.NewVM(&b, globals, -1).Run())
+	require.Equal(t, &tengo.Int{Value: 42}, globals[0])
+}
+
+func TestBytecodeDecodeMainStackDepth(t *testing.T) {
+	// the main function's stack bound is not serialized; a decoded main that
+	// needs more operand slots than the VM has must fail with a stack
+	// overflow error instead of running past the end of the stack
+	src := "out = [" + strings.Repeat("1, ", tengo.StackSize) + "1]"
+	var b tengo.Bytecode
+	decodeRoundTrip(t, compileForDecode(t, src), &b)
+
+	globals := make([]tengo.Object, tengo.GlobalsSize)
+	err := tengo.NewVM(&b, globals, -1).Run()
+	require.Error(t, err)
+	require.True(t, strings.Contains(err.Error(), "stack overflow"), err.Error())
+}
+
+func compileForDecode(t *testing.T, src string) *tengo.Bytecode {
+	fileSet := parser.NewFileSet()
+	file := fileSet.AddFile("test", -1, len(src))
+	parsed, err := parser.NewParser(file, []byte(src), nil).ParseFile()
+	require.NoError(t, err)
+
+	symTable := tengo.NewSymbolTable()
+	symTable.Define("out")
+	c := tengo.NewCompiler(file, symTable, nil, nil, nil)
+	require.NoError(t, c.Compile(parsed))
+	return c.Bytecode()
+}
+
+func decodeRoundTrip(t *testing.T, b *tengo.Bytecode, into *tengo.Bytecode) {
+	var buf bytes.Buffer
+	require.NoError(t, b.Encode(&buf))
+	require.NoError(t, into.Decode(&buf, nil))
+}

compiler.go

@@ -494,6 +494,7 @@ func (c *Compiler) Compile(node parser.Node) error {
 			NumParameters: len(node.Type.Params.List),
 			VarArgs:       node.Type.Params.VarArgs,
 			SourceMap:     sourceMap,
+			stackDepth:    maxStackDepth(instructions),
 		}
 		if len(freeSymbols) > 0 {
 			c.emit(node, parser.OpClosure,
@@ -638,6 +639,7 @@ func (c *Compiler) Bytecode() *Bytecode {
 		MainFunction: &CompiledFunction{
 			Instructions: append(c.currentInstructions(), parser.OpSuspend),
 			SourceMap:    c.currentSourceMap(),
+			stackDepth:   maxStackDepth(c.currentInstructions()),
 		},
 		Constants: c.constants,
 	}

errors.go

@@ -52,6 +52,9 @@ var (
 
 	// ErrInvalidRangeStep is an error where the step parameter is less than or equal to 0 when using builtin range function.
 	ErrInvalidRangeStep = errors.New("range step must be greater than 0")
+
+	// ErrDivisionByZero represents an integer division or remainder by zero.
+	ErrDivisionByZero = errors.New("integer division by zero")
 )
 
 // ErrInvalidArgumentType represents an invalid argument value type error.

iterator.go

@@ -99,23 +99,34 @@ func (i *BytesIterator) Next() bool {
 
 // Key returns the key or index value of the current element.
 func (i *BytesIterator) Key() Object {
-	return &Int{Value: int64(i.i - 1)}
+	return NewInt(int64(i.i - 1))
 }
 
 // Value returns the value of the current element.
 func (i *BytesIterator) Value() Object {
-	return &Int{Value: int64(i.v[i.i-1])}
+	return NewInt(int64(i.v[i.i-1]))
 }
 
 // MapIterator represents an iterator for the map.
 type MapIterator struct {
 	ObjectImpl
 	v map[string]Object
-	k []string
+	// k holds the key objects for the whole iteration in one allocation;
+	// Key hands out pointers into it instead of allocating per step
+	k []String
 	i int
 	l int
 }
 
+// newMapIterator snapshots the keys of m.
+func newMapIterator(m map[string]Object) *MapIterator {
+	keys := make([]String, 0, len(m))
+	for k := range m {
+		keys = append(keys, String{Value: k})
+	}
+	return &MapIterator{v: m, k: keys, l: len(keys)}
+}
+
 // TypeName returns the name of the type.
 func (i *MapIterator) TypeName() string {
 	return "map-iterator"
@@ -149,14 +160,12 @@ func (i *MapIterator) Next() bool {
 
 // Key returns the key or index value of the current element.
 func (i *MapIterator) Key() Object {
-	k := i.k[i.i-1]
-	return &String{Value: k}
+	return &i.k[i.i-1]
 }
 
 // Value returns the value of the current element.
 func (i *MapIterator) Value() Object {
-	k := i.k[i.i-1]
-	return i.v[k]
+	return i.v[i.k[i.i-1].Value]
 }
 
 // StringIterator represents an iterator for a string.
@@ -200,7 +209,7 @@ func (i *StringIterator) Next() bool {
 
 // Key returns the key or index value of the current element.
 func (i *StringIterator) Key() Object {
-	return &Int{Value: int64(i.i - 1)}
+	return NewInt(int64(i.i - 1))
 }
 
 // Value returns the value of the current element.