Implement Tail Call Optimize into ReturnCall Instruction

.gitignore

@@ -19,3 +19,10 @@ rules.ninja
 node_modules
 /test/wasmBenchmarker/ctests/wasm
 /test/wasmBenchmarker/emsdk
+/.cache
+/build/.cache
+/build/.cmake
+/build/_deps
+/build/third_party
+/build/walrus
+compile_commands.json

src/interpreter/ByteCode.cpp

@@ -71,6 +71,21 @@ size_t ByteCode::getSize() const
         return ByteCode::pointerAlignedSize(sizeof(CallRef) + sizeof(ByteCodeStackOffset) * callRef->parameterOffsetsSize()
                                             + sizeof(ByteCodeStackOffset) * callRef->resultOffsetsSize());
     }
+    case ReturnCallOpcode: {
+        const ReturnCall* returnCall = reinterpret_cast<const ReturnCall*>(this);
+        return ByteCode::pointerAlignedSize(sizeof(ReturnCall) + sizeof(ByteCodeStackOffset) * returnCall->parameterOffsetsSize()
+                                            + sizeof(ByteCodeStackOffset) * returnCall->resultOffsetsSize());
+    }
+    case ReturnCallIndirectOpcode: {
+        const ReturnCallIndirect* returnCallIndirect = reinterpret_cast<const ReturnCallIndirect*>(this);
+        return ByteCode::pointerAlignedSize(sizeof(ReturnCallIndirect) + sizeof(ByteCodeStackOffset) * returnCallIndirect->parameterOffsetsSize()
+                                            + sizeof(ByteCodeStackOffset) * returnCallIndirect->resultOffsetsSize());
+    }
+    case ReturnCallRefOpcode: {
+        const ReturnCallRef* returnCallRef = reinterpret_cast<const ReturnCallRef*>(this);
+        return ByteCode::pointerAlignedSize(sizeof(ReturnCallRef) + sizeof(ByteCodeStackOffset) * returnCallRef->parameterOffsetsSize()
+                                            + sizeof(ByteCodeStackOffset) * returnCallRef->resultOffsetsSize());
+    }
     case EndOpcode: {
         const End* end = reinterpret_cast<const End*>(this);
         return ByteCode::pointerAlignedSize(sizeof(End) + sizeof(ByteCodeStackOffset) * end->offsetsSize());

src/interpreter/ByteCode.h

@@ -47,6 +47,9 @@ class FunctionType;
     F(Call)                     \
     F(CallIndirect)             \
     F(CallRef)                  \
+    F(ReturnCall)               \
+    F(ReturnCallIndirect)       \
+    F(ReturnCallRef)            \
     F(Select)                   \
     F(MemorySize)               \
     F(MemorySizeM64)            \
@@ -2128,6 +2131,175 @@ class CallRef : public ByteCode {
     uint16_t m_resultOffsetsSize;
 };
 
+class ReturnCall : public ByteCode {
+public:
+    ReturnCall(uint32_t index, uint16_t parameterOffsetsSize, uint16_t resultOffsetsSize,
+               FunctionType* functionType)
+        : ByteCode(Opcode::ReturnCallOpcode)
+        , m_index(index)
+        , m_parameterOffsetsSize(parameterOffsetsSize)
+        , m_resultOffsetsSize(resultOffsetsSize)
+    {
+    }
+
+    uint32_t index() const { return m_index; }
+    ByteCodeStackOffset* stackOffsets() const
+    {
+        return reinterpret_cast<ByteCodeStackOffset*>(reinterpret_cast<size_t>(this) + sizeof(ReturnCall));
+    }
+
+    uint16_t parameterOffsetsSize() const
+    {
+        return m_parameterOffsetsSize;
+    }
+
+    uint16_t resultOffsetsSize() const
+    {
+        return m_resultOffsetsSize;
+    }
+
+#if !defined(NDEBUG)
+    void dump(size_t pos)
+    {
+        printf("return_call ");
+        printf("index: %" PRId32 " ", m_index);
+        size_t c = 0;
+        auto arr = stackOffsets();
+        printf("paramOffsets: ");
+        for (size_t i = 0; i < m_parameterOffsetsSize; i++) {
+            printf("%" PRIu32 " ", (uint32_t)arr[c++]);
+        }
+        printf(" ");
+
+        printf("resultOffsets: ");
+        for (size_t i = 0; i < m_resultOffsetsSize; i++) {
+            printf("%" PRIu32 " ", (uint32_t)arr[c++]);
+        }
+    }
+#endif
+
+protected:
+    uint32_t m_index;
+    uint16_t m_parameterOffsetsSize;
+    uint16_t m_resultOffsetsSize;
+};
+
+class ReturnCallIndirect : public ByteCode {
+public:
+    ReturnCallIndirect(ByteCodeStackOffset stackOffset, uint32_t tableIndex, FunctionType* functionType,
+                       uint16_t parameterOffsetsSize, uint16_t resultOffsetsSize)
+        : ByteCode(Opcode::ReturnCallIndirectOpcode)
+        , m_calleeOffset(stackOffset)
+        , m_tableIndex(tableIndex)
+        , m_functionType(functionType)
+        , m_parameterOffsetsSize(parameterOffsetsSize)
+        , m_resultOffsetsSize(resultOffsetsSize)
+    {
+    }
+
+    ByteCodeStackOffset calleeOffset() const { return m_calleeOffset; }
+    uint32_t tableIndex() const { return m_tableIndex; }
+    FunctionType* functionType() const { return m_functionType; }
+    ByteCodeStackOffset* stackOffsets() const
+    {
+        return reinterpret_cast<ByteCodeStackOffset*>(reinterpret_cast<size_t>(this) + sizeof(ReturnCallIndirect));
+    }
+
+    uint16_t parameterOffsetsSize() const
+    {
+        return m_parameterOffsetsSize;
+    }
+
+    uint16_t resultOffsetsSize() const
+    {
+        return m_resultOffsetsSize;
+    }
+
+#if !defined(NDEBUG)
+    void dump(size_t pos)
+    {
+        printf("return_call_indirect ");
+        printf("tableIndex: %" PRId32 " ", m_tableIndex);
+        DUMP_BYTECODE_OFFSET(calleeOffset);
+
+        size_t c = 0;
+        auto arr = stackOffsets();
+        printf("paramOffsets: ");
+        for (size_t i = 0; i < m_parameterOffsetsSize; i++) {
+            printf("%" PRIu32 " ", (uint32_t)arr[c++]);
+        }
+        printf(" ");
+
+        printf("resultOffsets: ");
+        for (size_t i = 0; i < m_resultOffsetsSize; i++) {
+            printf("%" PRIu32 " ", (uint32_t)arr[c++]);
+        }
+    }
+#endif
+
+protected:
+    ByteCodeStackOffset m_calleeOffset;
+    uint32_t m_tableIndex;
+    FunctionType* m_functionType;
+    uint16_t m_parameterOffsetsSize;
+    uint16_t m_resultOffsetsSize;
+};
+
+class ReturnCallRef : public ByteCode {
+public:
+    ReturnCallRef(ByteCodeStackOffset stackOffset, FunctionType* functionType,
+                  uint16_t parameterOffsetsSize, uint16_t resultOffsetsSize)
+        : ByteCode(Opcode::ReturnCallRefOpcode)
+        , m_calleeOffset(stackOffset)
+        , m_functionType(functionType)
+        , m_parameterOffsetsSize(parameterOffsetsSize)
+        , m_resultOffsetsSize(resultOffsetsSize)
+    {
+    }
+
+    ByteCodeStackOffset calleeOffset() const { return m_calleeOffset; }
+    FunctionType* functionType() const { return m_functionType; }
+    ByteCodeStackOffset* stackOffsets() const
+    {
+        return reinterpret_cast<ByteCodeStackOffset*>(reinterpret_cast<size_t>(this) + sizeof(ReturnCallRef));
+    }
+
+    uint16_t parameterOffsetsSize() const
+    {
+        return m_parameterOffsetsSize;
+    }
+
+    uint16_t resultOffsetsSize() const
+    {
+        return m_resultOffsetsSize;
+    }
+
+#if !defined(NDEBUG)
+    void dump(size_t pos)
+    {
+        printf("return_call_ref ");
+        size_t c = 0;
+        auto arr = stackOffsets();
+        printf("paramOffsets: ");
+        for (size_t i = 0; i < m_parameterOffsetsSize; i++) {
+            printf("%" PRIu32 " ", (uint32_t)arr[c++]);
+        }
+        printf(" ");
+
+        printf("resultOffsets: ");
+        for (size_t i = 0; i < m_resultOffsetsSize; i++) {
+            printf("%" PRIu32 " ", (uint32_t)arr[c++]);
+        }
+    }
+#endif
+
+protected:
+    ByteCodeStackOffset m_calleeOffset;
+    FunctionType* m_functionType;
+    uint16_t m_parameterOffsetsSize;
+    uint16_t m_resultOffsetsSize;
+};
+
 #define DEFINE_LOAD_OP(className, opcodeType, opStr)                            \
     class className : public ByteCodeOffset2 {                                  \
     public:                                                                     \

src/interpreter/Interpreter.cpp

@@ -17,10 +17,12 @@
 
 #include "Walrus.h"
 
+#include "interpreter/ByteCode.h"
 #include "interpreter/Interpreter.h"
 #include "runtime/Instance.h"
 #include "runtime/Function.h"
 #include "runtime/Memory.h"
+#include "runtime/Store.h"
 #include "runtime/Table.h"
 #include "runtime/GCArray.h"
 #include "runtime/GCStruct.h"
@@ -1711,6 +1713,72 @@ ByteCodeStackOffset* Interpreter::interpret(ExecutionState& state,
         NEXT_INSTRUCTION();
     }
 
+    DEFINE_OPCODE(ReturnCall)
+        :
+    {
+        ReturnCall* code = (ReturnCall*)programCounter;
+        auto target = instance->function(code->index());
+
+        auto paramSize = code->parameterOffsetsSize();
+        auto offsets = code->stackOffsets();
+
+        auto store = instance->module()->store();
+        store->setTCO(bp, offsets, paramSize, code->resultOffsetsSize(), target);
+
+        return nullptr;
+    }
+
+    DEFINE_OPCODE(ReturnCallIndirect)
+        :
+    {
+        ReturnCallIndirect* code = (ReturnCallIndirect*)programCounter;
+        Table* table = instance->table(code->tableIndex());
+
+        uint32_t idx = readValue<uint32_t>(bp, code->calleeOffset());
+        if (UNLIKELY(idx >= table->size())) {
+            Trap::throwException(state, "undefined element");
+        }
+        auto target = reinterpret_cast<Function*>(table->uncheckedGetElement(idx));
+        if (UNLIKELY(Value::isNull(target))) {
+            Trap::throwException(state, "uninitialized element " + std::to_string(idx));
+        }
+        const FunctionType* ft = target->functionType();
+        if (UNLIKELY(!ft->equals(code->functionType()))) {
+            Trap::throwException(state, "indirect call type mismatch");
+        }
+
+        auto paramSize = code->parameterOffsetsSize();
+        auto offsets = code->stackOffsets();
+
+        auto store = instance->module()->store();
+        store->setTCO(bp, offsets, paramSize, code->resultOffsetsSize(), target);
+
+        return nullptr;
+    }
+
+    DEFINE_OPCODE(ReturnCallRef)
+        :
+    {
+        ReturnCallRef* code = (ReturnCallRef*)programCounter;
+
+        auto target = readValue<Function*>(bp, code->calleeOffset());
+        if (UNLIKELY(Value::isNull(target))) {
+            Trap::throwException(state, "null function reference");
+        }
+        const FunctionType* ft = target->functionType();
+        if (UNLIKELY(!ft->equals(code->functionType()))) {
+            Trap::throwException(state, "call by reference type mismatch");
+        }
+
+        auto paramSize = code->parameterOffsetsSize();
+        auto offsets = code->stackOffsets();
+
+        auto store = instance->module()->store();
+        store->setTCO(bp, offsets, paramSize, code->resultOffsetsSize(), target);
+
+        return nullptr;
+    }
+
     DEFINE_OPCODE(Select)
         :
     {
@@ -2976,11 +3044,22 @@ ByteCodeStackOffset* Interpreter::interpret(ExecutionState& state,
 
         uint8_t* ptr = userExceptionData.data();
         auto& param = tag->functionType()->param().types();
-        for (size_t i = 0; i < param.size(); i++) {
-            auto sz = valueStackAllocatedSize(param[i]);
-            memcpy(ptr, bp + code->dataOffsets()[i], sz);
-            ptr += sz;
+        auto store = instance->module()->store();
+
+        if (!store->hasTCO()) {
+            for (size_t i = 0; i < param.size(); i++) {
+                auto sz = valueStackAllocatedSize(param[i]);
+                memcpy(ptr, bp + code->dataOffsets()[i], sz);
+                ptr += sz;
+            }
+        } else {
+            for (size_t i = 0; i < param.size(); i++) {
+                auto sz = valueStackAllocatedSize(param[i]);
+                memcpy(ptr, bp + store->tcoBuffer()[i], sz);
+                ptr += sz;
+            }
         }
+
         Trap::throwException(state, tag, std::move(userExceptionData));
         ASSERT_NOT_REACHED();
         NEXT_INSTRUCTION();
@@ -3041,7 +3120,15 @@ NEVER_INLINE void Interpreter::callOperation(
 {
     Call* code = (Call*)programCounter;
     Function* target = instance->function(code->index());
+    auto store = instance->module()->store();
     target->interpreterCall(state, bp, code->stackOffsets(), code->parameterOffsetsSize(), code->resultOffsetsSize());
+
+    while (UNLIKELY(store->hasTCO())) {
+        auto resultOffsetCount = store->tcoResultOffsetCount();
+        target = store->tcoFunctionTarget();
+        target->interpreterCall(state, bp, code->stackOffsets(), code->parameterOffsetsSize(), resultOffsetCount);
+    }
+
     programCounter += ByteCode::pointerAlignedSize(sizeof(Call) + sizeof(ByteCodeStackOffset) * code->parameterOffsetsSize()
                                                    + sizeof(ByteCodeStackOffset) * code->resultOffsetsSize());
 }
@@ -3068,7 +3155,15 @@ NEVER_INLINE void Interpreter::callIndirectOperation(
         Trap::throwException(state, "indirect call type mismatch");
     }
 
+    auto store = instance->module()->store();
     target->interpreterCall(state, bp, code->stackOffsets(), code->parameterOffsetsSize(), code->resultOffsetsSize());
+
+    while (UNLIKELY(store->hasTCO())) {
+        auto resultOffsetCount = store->tcoResultOffsetCount();
+        target = store->tcoFunctionTarget();
+        target->interpreterCall(state, bp, code->stackOffsets(), code->parameterOffsetsSize(), resultOffsetCount);
+    }
+
     programCounter += ByteCode::pointerAlignedSize(sizeof(CallIndirect) + sizeof(ByteCodeStackOffset) * code->parameterOffsetsSize()
                                                    + sizeof(ByteCodeStackOffset) * code->resultOffsetsSize());
 }
@@ -3090,7 +3185,15 @@ NEVER_INLINE void Interpreter::callRefOperation(
         Trap::throwException(state, "call by reference type mismatch");
     }
 
+    auto store = instance->module()->store();
     target->interpreterCall(state, bp, code->stackOffsets(), code->parameterOffsetsSize(), code->resultOffsetsSize());
+
+    while (UNLIKELY(store->hasTCO())) {
+        auto resultOffsetCount = store->tcoResultOffsetCount();
+        target = store->tcoFunctionTarget();
+        target->interpreterCall(state, bp, code->stackOffsets(), code->parameterOffsetsSize(), resultOffsetCount);
+    }
+
     programCounter += ByteCode::pointerAlignedSize(sizeof(CallRef) + sizeof(ByteCodeStackOffset) * code->parameterOffsetsSize()
                                                    + sizeof(ByteCodeStackOffset) * code->resultOffsetsSize());
 }