diff --git a/src/evm/interpreter.cpp b/src/evm/interpreter.cpp index 7cad3439..38ee4e44 100644 --- a/src/evm/interpreter.cpp +++ b/src/evm/interpreter.cpp @@ -349,6 +349,783 @@ void BaseInterpreter::interpret() { (uint64_t)Frame->Msg.gas >= GasChunkCost[ChunkStartPc]) { const uint32_t ChunkEnd = GasChunkEnd[ChunkStartPc]; Frame->Msg.gas -= GasChunkCost[ChunkStartPc]; +#if defined(__GNUC__) + // =================== COMPUTED GOTO FAST PATH =================== + // Uses computed goto (GCC/Clang extension) for better branch + // prediction (one indirect branch predictor entry per opcode), + // local stack pointer for register allocation, and inlined hot + // opcodes to eliminate EVMResource static global loads. + { + uint64_t Pc = Frame->Pc; + size_t sp = Frame->Sp; + + // Dispatch table: 256 entries, one per opcode byte value. + // Initialized once at first call (label addresses are stable for + // non-nested functions in GCC/Clang). + static void *cgoto_table[256] = {}; + static bool cgoto_initialized = false; + if (!cgoto_initialized) { + for (int i = 0; i < 256; i++) + cgoto_table[i] = &&TARGET_UNDEFINED; + cgoto_table[0x00] = &&TARGET_STOP; + cgoto_table[0x01] = &&TARGET_ADD; + cgoto_table[0x02] = &&TARGET_MUL; + cgoto_table[0x03] = &&TARGET_SUB; + cgoto_table[0x04] = &&TARGET_DIV; + cgoto_table[0x05] = &&TARGET_SDIV; + cgoto_table[0x06] = &&TARGET_MOD; + cgoto_table[0x07] = &&TARGET_SMOD; + cgoto_table[0x08] = &&TARGET_ADDMOD; + cgoto_table[0x09] = &&TARGET_MULMOD; + cgoto_table[0x0a] = &&TARGET_EXP; + cgoto_table[0x0b] = &&TARGET_SIGNEXTEND; + cgoto_table[0x10] = &&TARGET_LT; + cgoto_table[0x11] = &&TARGET_GT; + cgoto_table[0x12] = &&TARGET_SLT; + cgoto_table[0x13] = &&TARGET_SGT; + cgoto_table[0x14] = &&TARGET_EQ; + cgoto_table[0x15] = &&TARGET_ISZERO; + cgoto_table[0x16] = &&TARGET_AND; + cgoto_table[0x17] = &&TARGET_OR; + cgoto_table[0x18] = &&TARGET_XOR; + cgoto_table[0x19] = &&TARGET_NOT; + cgoto_table[0x1a] = &&TARGET_BYTE; + cgoto_table[0x1b] = &&TARGET_SHL; + cgoto_table[0x1c] = &&TARGET_SHR; + cgoto_table[0x1d] = &&TARGET_SAR; + cgoto_table[0x1e] = &&TARGET_CLZ; + cgoto_table[0x20] = &&TARGET_KECCAK256; + cgoto_table[0x30] = &&TARGET_ADDRESS; + cgoto_table[0x31] = &&TARGET_BALANCE; + cgoto_table[0x32] = &&TARGET_ORIGIN; + cgoto_table[0x33] = &&TARGET_CALLER; + cgoto_table[0x34] = &&TARGET_CALLVALUE; + cgoto_table[0x35] = &&TARGET_CALLDATALOAD; + cgoto_table[0x36] = &&TARGET_CALLDATASIZE; + cgoto_table[0x37] = &&TARGET_CALLDATACOPY; + cgoto_table[0x38] = &&TARGET_CODESIZE; + cgoto_table[0x39] = &&TARGET_CODECOPY; + cgoto_table[0x3a] = &&TARGET_GASPRICE; + cgoto_table[0x3b] = &&TARGET_EXTCODESIZE; + cgoto_table[0x3c] = &&TARGET_EXTCODECOPY; + cgoto_table[0x3d] = &&TARGET_RETURNDATASIZE; + cgoto_table[0x3e] = &&TARGET_RETURNDATACOPY; + cgoto_table[0x3f] = &&TARGET_EXTCODEHASH; + cgoto_table[0x40] = &&TARGET_BLOCKHASH; + cgoto_table[0x41] = &&TARGET_COINBASE; + cgoto_table[0x42] = &&TARGET_TIMESTAMP; + cgoto_table[0x43] = &&TARGET_NUMBER; + cgoto_table[0x44] = &&TARGET_PREVRANDAO; + cgoto_table[0x45] = &&TARGET_GASLIMIT; + cgoto_table[0x46] = &&TARGET_CHAINID; + cgoto_table[0x47] = &&TARGET_SELFBALANCE; + cgoto_table[0x48] = &&TARGET_BASEFEE; + cgoto_table[0x49] = &&TARGET_BLOBHASH; + cgoto_table[0x4a] = &&TARGET_BLOBBASEFEE; + cgoto_table[0x50] = &&TARGET_POP; + cgoto_table[0x51] = &&TARGET_MLOAD; + cgoto_table[0x52] = &&TARGET_MSTORE; + cgoto_table[0x53] = &&TARGET_MSTORE8; + cgoto_table[0x54] = &&TARGET_SLOAD; + cgoto_table[0x55] = &&TARGET_SSTORE; + cgoto_table[0x56] = &&TARGET_JUMP; + cgoto_table[0x57] = &&TARGET_JUMPI; + cgoto_table[0x58] = &&TARGET_PC; + cgoto_table[0x59] = &&TARGET_MSIZE; + cgoto_table[0x5a] = &&TARGET_GAS; + cgoto_table[0x5b] = &&TARGET_JUMPDEST; + cgoto_table[0x5c] = &&TARGET_TLOAD; + cgoto_table[0x5d] = &&TARGET_TSTORE; + cgoto_table[0x5e] = &&TARGET_MCOPY; + cgoto_table[0x5f] = &&TARGET_PUSH0; + for (int i = 0x60; i <= 0x7f; i++) + cgoto_table[i] = &&TARGET_PUSHX; + for (int i = 0x80; i <= 0x8f; i++) + cgoto_table[i] = &&TARGET_DUPX; + for (int i = 0x90; i <= 0x9f; i++) + cgoto_table[i] = &&TARGET_SWAPX; + for (int i = 0xa0; i <= 0xa4; i++) + cgoto_table[i] = &&TARGET_LOGX; + cgoto_table[0xf0] = &&TARGET_CREATEX; + cgoto_table[0xf1] = &&TARGET_CALLX; + cgoto_table[0xf2] = &&TARGET_CALLX; + cgoto_table[0xf3] = &&TARGET_RETURN; + cgoto_table[0xf4] = &&TARGET_CALLX; + cgoto_table[0xf5] = &&TARGET_CREATEX; + cgoto_table[0xfa] = &&TARGET_CALLX; + cgoto_table[0xfd] = &&TARGET_REVERT; + cgoto_table[0xfe] = &&TARGET_INVALID; + cgoto_table[0xff] = &&TARGET_SELFDESTRUCT; + cgoto_initialized = true; + } + +// Dispatch to next opcode or exit if chunk boundary reached +#define DISPATCH_NEXT \ + do { \ + if (INTX_UNLIKELY(Pc >= ChunkEnd)) \ + goto cgoto_chunk_done; \ + goto *cgoto_table[static_cast(Code[Pc])]; \ + } while (0) + +// Write back local sp/Pc, set EVMResource, call handler, reload sp, +// advance Pc, check status, and dispatch next opcode +#define HANDLER_CALL(handler_expr) \ + do { \ + Frame->Sp = sp; \ + Frame->Pc = Pc; \ + EVMResource::setExecutionContext(Frame, &Context); \ + handler_expr; \ + sp = Frame->Sp; \ + ++Pc; \ + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) \ + goto cgoto_error; \ + DISPATCH_NEXT; \ + } while (0) + + // Initial dispatch + goto *cgoto_table[static_cast(Code[Pc])]; + + // ---- Inline binary arithmetic/logic ops ---- + TARGET_ADD : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = A + B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_MUL : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = A * B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SUB : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = A - B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_DIV : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = (B == 0) ? intx::uint256(0) : (A / B); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SDIV : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = (B == 0) ? intx::uint256(0) : intx::sdivrem(A, B).quot; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_MOD : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = (B == 0) ? intx::uint256(0) : A % B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SMOD : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = (B == 0) ? intx::uint256(0) : intx::sdivrem(A, B).rem; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_LT : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = intx::uint256(A < B); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_GT : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = intx::uint256(A > B); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SLT : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = intx::uint256(intx::slt(A, B)); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SGT : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = intx::uint256(intx::slt(B, A)); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_EQ : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = intx::uint256(A == B); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_AND : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = A & B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_OR : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = A | B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_XOR : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = A ^ B; + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SHL : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = (A < 256) ? (B << A) : intx::uint256(0); + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SHR : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + B = (A < 256) ? (B >> A) : intx::uint256(0); + --sp; + ++Pc; + DISPATCH_NEXT; + } + + // ---- Inline ternary ops ---- + TARGET_ADDMOD : { + if (INTX_UNLIKELY(sp < 3)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + auto &C = Frame->Stack[sp - 3]; + C = (C == 0) ? intx::uint256(0) : intx::addmod(A, B, C); + sp -= 2; + ++Pc; + DISPATCH_NEXT; + } + TARGET_MULMOD : { + if (INTX_UNLIKELY(sp < 3)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + auto &B = Frame->Stack[sp - 2]; + auto &C = Frame->Stack[sp - 3]; + C = (C == 0) ? intx::uint256(0) : intx::mulmod(A, B, C); + sp -= 2; + ++Pc; + DISPATCH_NEXT; + } + + // ---- Inline unary ops ---- + TARGET_ISZERO : { + if (INTX_UNLIKELY(sp < 1)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + A = intx::uint256(A == 0); + ++Pc; + DISPATCH_NEXT; + } + TARGET_NOT : { + if (INTX_UNLIKELY(sp < 1)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + A = ~A; + ++Pc; + DISPATCH_NEXT; + } + TARGET_CLZ : { + if (INTX_UNLIKELY(sp < 1)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + auto &A = Frame->Stack[sp - 1]; + A = intx::clz(A); + ++Pc; + DISPATCH_NEXT; + } + + // ---- Inline stack ops ---- + TARGET_POP : { + if (INTX_UNLIKELY(sp < 1)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + --sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_PUSH0 : { + if (INTX_UNLIKELY(sp >= MAXSTACK)) { + Context.setStatus(EVMC_STACK_OVERFLOW); + goto cgoto_error; + } + Frame->Stack[sp++] = 0; + ++Pc; + DISPATCH_NEXT; + } + TARGET_PUSHX : { + if (INTX_UNLIKELY(sp >= MAXSTACK)) { + Context.setStatus(EVMC_STACK_OVERFLOW); + goto cgoto_error; + } + Frame->Stack[sp++] = PushValueMap[Pc]; + const uint8_t NumBytes = static_cast(Code[Pc]) - + static_cast(evmc_opcode::OP_PUSH1) + + 1; + Pc += 1 + NumBytes; + DISPATCH_NEXT; + } + TARGET_DUPX : { + const uint32_t N = static_cast(Code[Pc]) - + static_cast(evmc_opcode::OP_DUP1) + 1; + if (INTX_UNLIKELY(sp < N)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + if (INTX_UNLIKELY(sp >= MAXSTACK)) { + Context.setStatus(EVMC_STACK_OVERFLOW); + goto cgoto_error; + } + Frame->Stack[sp] = Frame->Stack[sp - N]; + ++sp; + ++Pc; + DISPATCH_NEXT; + } + TARGET_SWAPX : { + const uint32_t N = static_cast(Code[Pc]) - + static_cast(evmc_opcode::OP_SWAP1) + 1; + if (INTX_UNLIKELY(sp < N + 1)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + const size_t TopIdx = sp - 1; + const size_t NthIdx = sp - 1 - N; + const intx::uint256 Tmp = Frame->Stack[TopIdx]; + Frame->Stack[TopIdx] = Frame->Stack[NthIdx]; + Frame->Stack[NthIdx] = Tmp; + ++Pc; + DISPATCH_NEXT; + } + + // ---- Inline control flow ops ---- + TARGET_JUMP : { + if (INTX_UNLIKELY(sp < 1)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + --sp; + const uint64_t Dest = Uint256ToUint64(Frame->Stack[sp]); + if (INTX_UNLIKELY(Dest >= CodeSize)) { + Context.setStatus(EVMC_BAD_JUMP_DESTINATION); + goto cgoto_error; + } + if (INTX_UNLIKELY(JumpDestMap[Dest] == 0)) { + Context.setStatus(EVMC_BAD_JUMP_DESTINATION); + goto cgoto_error; + } + Pc = Dest; + Frame->Sp = sp; + Frame->Pc = Pc; + goto cgoto_restart; + } + TARGET_JUMPI : { + if (INTX_UNLIKELY(sp < 2)) { + Context.setStatus(EVMC_STACK_UNDERFLOW); + goto cgoto_error; + } + --sp; + const uint64_t Dest = Uint256ToUint64(Frame->Stack[sp]); + --sp; + const intx::uint256 &Cond = Frame->Stack[sp]; + if (!Cond) { + ++Pc; + DISPATCH_NEXT; + } + if (INTX_UNLIKELY(Dest >= CodeSize)) { + Context.setStatus(EVMC_BAD_JUMP_DESTINATION); + goto cgoto_error; + } + if (INTX_UNLIKELY(JumpDestMap[Dest] == 0)) { + Context.setStatus(EVMC_BAD_JUMP_DESTINATION); + goto cgoto_error; + } + Pc = Dest; + Frame->Sp = sp; + Frame->Pc = Pc; + goto cgoto_restart; + } + TARGET_JUMPDEST : { + ++Pc; + DISPATCH_NEXT; + } + TARGET_STOP : { + Frame->Sp = sp; + Frame->Pc = Pc; + const uint64_t RemainingGas = Frame->Msg.gas; + Context.setReturnData(std::vector()); + Context.freeBackFrame(); + Frame = Context.getCurFrame(); + if (!Frame) { + const auto &ReturnData = Context.getReturnData(); + const uint64_t GasLeft = Context.getInstance()->getGas(); + evmc::Result ExeResult(EVMC_SUCCESS, GasLeft, + Context.getInstance()->getGasRefund(), + ReturnData.data(), ReturnData.size()); + Context.setExeResult(std::move(ExeResult)); + return; + } + Frame->Msg.gas += RemainingGas; + goto cgoto_restart; + } + TARGET_INVALID : { + Context.setStatus(EVMC_INVALID_INSTRUCTION); + goto cgoto_error; + } + TARGET_UNDEFINED : { + Context.setStatus(EVMC_UNDEFINED_INSTRUCTION); + goto cgoto_error; + } + + // ---- Complex handler ops (delegate to doExecute) ---- + TARGET_EXP: + HANDLER_CALL(ExpHandler::doExecute()); + TARGET_SIGNEXTEND: + HANDLER_CALL(SignExtendHandler::doExecute()); + TARGET_BYTE: + HANDLER_CALL(ByteHandler::doExecute()); + TARGET_SAR: + HANDLER_CALL(SarHandler::doExecute()); + TARGET_KECCAK256: + HANDLER_CALL(Keccak256Handler::doExecute()); + + // Environment information + TARGET_ADDRESS: + HANDLER_CALL(AddressHandler::doExecute()); + TARGET_BALANCE: + HANDLER_CALL(BalanceHandler::doExecute()); + TARGET_ORIGIN: + HANDLER_CALL(OriginHandler::doExecute()); + TARGET_CALLER: + HANDLER_CALL(CallerHandler::doExecute()); + TARGET_CALLVALUE: + HANDLER_CALL(CallValueHandler::doExecute()); + TARGET_CALLDATALOAD: + HANDLER_CALL(CallDataLoadHandler::doExecute()); + TARGET_CALLDATASIZE: + HANDLER_CALL(CallDataSizeHandler::doExecute()); + TARGET_CALLDATACOPY: + HANDLER_CALL(CallDataCopyHandler::doExecute()); + TARGET_CODESIZE: + HANDLER_CALL(CodeSizeHandler::doExecute()); + TARGET_CODECOPY: + HANDLER_CALL(CodeCopyHandler::doExecute()); + TARGET_GASPRICE: + HANDLER_CALL(GasPriceHandler::doExecute()); + TARGET_EXTCODESIZE: + HANDLER_CALL(ExtCodeSizeHandler::doExecute()); + TARGET_EXTCODECOPY: + HANDLER_CALL(ExtCodeCopyHandler::doExecute()); + TARGET_RETURNDATASIZE: + HANDLER_CALL(ReturnDataSizeHandler::doExecute()); + TARGET_RETURNDATACOPY: + HANDLER_CALL(ReturnDataCopyHandler::doExecute()); + TARGET_EXTCODEHASH: + HANDLER_CALL(ExtCodeHashHandler::doExecute()); + + // Block information + TARGET_BLOCKHASH: + HANDLER_CALL(BlockHashHandler::doExecute()); + TARGET_COINBASE: + HANDLER_CALL(CoinBaseHandler::doExecute()); + TARGET_TIMESTAMP: + HANDLER_CALL(TimeStampHandler::doExecute()); + TARGET_NUMBER: + HANDLER_CALL(NumberHandler::doExecute()); + TARGET_PREVRANDAO: + HANDLER_CALL(PrevRanDaoHandler::doExecute()); + TARGET_GASLIMIT: + HANDLER_CALL(GasLimitHandler::doExecute()); + TARGET_CHAINID: + HANDLER_CALL(ChainIdHandler::doExecute()); + TARGET_SELFBALANCE: + HANDLER_CALL(SelfBalanceHandler::doExecute()); + TARGET_BASEFEE: + HANDLER_CALL(BaseFeeHandler::doExecute()); + TARGET_BLOBHASH: + HANDLER_CALL(BlobHashHandler::doExecute()); + TARGET_BLOBBASEFEE: + HANDLER_CALL(BlobBaseFeeHandler::doExecute()); + + // Memory & storage + TARGET_MLOAD: + HANDLER_CALL(MLoadHandler::doExecute()); + TARGET_MSTORE: + HANDLER_CALL(MStoreHandler::doExecute()); + TARGET_MSTORE8: + HANDLER_CALL(MStore8Handler::doExecute()); + TARGET_SLOAD: + HANDLER_CALL(SLoadHandler::doExecute()); + TARGET_SSTORE: + HANDLER_CALL(SStoreHandler::doExecute()); + + // Misc + TARGET_PC: + HANDLER_CALL(PCHandler::doExecute()); + TARGET_MSIZE: + HANDLER_CALL(MSizeHandler::doExecute()); + TARGET_GAS: + HANDLER_CALL(GasHandler::doExecute()); + TARGET_TLOAD: + HANDLER_CALL(TLoadHandler::doExecute()); + TARGET_TSTORE: + HANDLER_CALL(TStoreHandler::doExecute()); + TARGET_MCOPY: + HANDLER_CALL(MCopyHandler::doExecute()); + + // Multi-opcode handlers: LOG, CALL, CREATE + TARGET_LOGX : { + Frame->Sp = sp; + Frame->Pc = Pc; + EVMResource::setExecutionContext(Frame, &Context); + LogHandler::OpCode = + static_cast(static_cast(Code[Pc])); + LogHandler::doExecute(); + sp = Frame->Sp; + ++Pc; + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) + goto cgoto_error; + DISPATCH_NEXT; + } + TARGET_CALLX : { + Frame->Sp = sp; + Frame->Pc = Pc; + EVMResource::setExecutionContext(Frame, &Context); + CallHandler::OpCode = + static_cast(static_cast(Code[Pc])); + CallHandler::doExecute(); + sp = Frame->Sp; + ++Pc; + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) + goto cgoto_error; + DISPATCH_NEXT; + } + TARGET_CREATEX : { + Frame->Sp = sp; + Frame->Pc = Pc; + EVMResource::setExecutionContext(Frame, &Context); + CreateHandler::OpCode = + static_cast(static_cast(Code[Pc])); + CreateHandler::doExecute(); + sp = Frame->Sp; + ++Pc; + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) + goto cgoto_error; + DISPATCH_NEXT; + } + + // ---- Special termination handlers (may change Frame) ---- + TARGET_RETURN : { + Frame->Sp = sp; + Frame->Pc = Pc; + EVMResource::setExecutionContext(Frame, &Context); + ReturnHandler::doExecute(); + Frame = Context.getCurFrame(); + if (!Frame) { + const auto &ReturnData = Context.getReturnData(); + const uint64_t GasLeft = Context.getInstance()->getGas(); + evmc::Result ExeResult(EVMC_SUCCESS, GasLeft, + Context.getInstance()->getGasRefund(), + ReturnData.data(), ReturnData.size()); + Context.setExeResult(std::move(ExeResult)); + return; + } + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) { + if (handleExecutionStatus(Frame, Context)) { + return; + } + goto cgoto_break_outer; + } + goto cgoto_restart; + } + TARGET_REVERT : { + Frame->Sp = sp; + Frame->Pc = Pc; + EVMResource::setExecutionContext(Frame, &Context); + RevertHandler::doExecute(); + Frame = Context.getCurFrame(); + if (!Frame) { + const auto &ReturnData = Context.getReturnData(); + const uint64_t GasLeft = Context.getInstance()->getGas(); + evmc::Result ExeResult(EVMC_REVERT, GasLeft, + Context.getInstance()->getGasRefund(), + ReturnData.data(), ReturnData.size()); + Context.setExeResult(std::move(ExeResult)); + return; + } + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) { + if (handleExecutionStatus(Frame, Context)) { + return; + } + goto cgoto_break_outer; + } + goto cgoto_restart; + } + TARGET_SELFDESTRUCT : { + Frame->Sp = sp; + Frame->Pc = Pc; + EVMResource::setExecutionContext(Frame, &Context); + SelfDestructHandler::doExecute(); + Frame = Context.getCurFrame(); + if (!Frame) { + const auto &ReturnData = Context.getReturnData(); + const uint64_t GasLeft = Context.getInstance()->getGas(); + evmc::Result ExeResult(EVMC_SUCCESS, GasLeft, + Context.getInstance()->getGasRefund(), + ReturnData.data(), ReturnData.size()); + Context.setExeResult(std::move(ExeResult)); + return; + } + if (INTX_UNLIKELY(Context.getStatus() != EVMC_SUCCESS)) { + if (handleExecutionStatus(Frame, Context)) { + return; + } + goto cgoto_break_outer; + } + goto cgoto_restart; + } + + // ---- Exit labels ---- + cgoto_chunk_done: + Frame->Sp = sp; + Frame->Pc = Pc; + goto cgoto_continue_outer; + + cgoto_restart: + goto cgoto_continue_outer; + + cgoto_error: + Frame->Sp = sp; + Frame->Pc = Pc; + if (handleExecutionStatus(Frame, Context)) { + return; + } + goto cgoto_break_outer; + +#undef DISPATCH_NEXT +#undef HANDLER_CALL + } + cgoto_continue_outer: + continue; + cgoto_break_outer: + break; +#else bool RestartDispatch = false; while (Frame->Pc < ChunkEnd) { const Byte OpcodeByte = Code[Frame->Pc]; @@ -761,6 +1538,7 @@ void BaseInterpreter::interpret() { continue; } continue; +#endif } Byte OpcodeByte = Code[Frame->Pc]; diff --git a/src/evm/interpreter.h b/src/evm/interpreter.h index 398c4f91..90775c14 100644 --- a/src/evm/interpreter.h +++ b/src/evm/interpreter.h @@ -10,7 +10,6 @@ #include "intx/intx.hpp" #include -#include #include namespace zen { @@ -70,7 +69,7 @@ struct EVMFrame { class InterpreterExecContext { private: runtime::EVMInstance *Inst; - std::deque FrameStack; + std::vector FrameStack; evmc_status_code Status = EVMC_SUCCESS; std::vector ReturnData; evmc::Result ExeResult; @@ -80,6 +79,17 @@ class InterpreterExecContext { InterpreterExecContext(runtime::EVMInstance *Inst) : Inst(Inst) {} + /// Reset state for reuse across calls. Keeps allocated capacity to avoid + /// re-allocating the ~32KB EVMFrame on every call. + void resetForNewCall(runtime::EVMInstance *NewInst) { + Inst = NewInst; + FrameStack.clear(); // keeps vector capacity + Status = EVMC_SUCCESS; + ReturnData.clear(); // keeps vector capacity + IsJump = false; + ExeResult = evmc::Result{EVMC_SUCCESS, 0, 0}; + } + EVMFrame *allocTopFrame(evmc_message *Msg); void freeBackFrame(); diff --git a/src/runtime/evm_instance.cpp b/src/runtime/evm_instance.cpp index dc8ebbd3..8233402e 100644 --- a/src/runtime/evm_instance.cpp +++ b/src/runtime/evm_instance.cpp @@ -60,6 +60,42 @@ EVMInstanceUniquePtr EVMInstance::newEVMInstance(Isolation &Iso, EVMInstance::~EVMInstance() {} +void EVMInstance::resetForNewCall(evmc_revision NewRev) { + // Reset gas accounting + Gas = 0; + GasRefund = 0; + Rev = NewRev; + InstanceExitCode = 0; + Err = common::ErrorCode::NoError; + + // Reset message stack (clear but keep capacity) + CurrentMessage = nullptr; + MessageStack.clear(); + GasRefundStack.clear(); + + // Reset memory: keep the 16MB allocation, just reset the size + MemoryBase = nullptr; + MemorySize = 0; + // Don't release Memory - reuse the allocation on next pushMessage() + MemoryStack.clear(); + + // Reset output + ReturnData.clear(); + ExeResult = evmc::Result{EVMC_SUCCESS, 0, 0}; + + // Reset execution cache: clear() keeps allocated bucket arrays, + // avoiding repeated alloc/free of unordered_map internals. + InstanceExecutionCache.BlockHashes.clear(); + InstanceExecutionCache.BlobHashes.clear(); + InstanceExecutionCache.CalldataLoads.clear(); + InstanceExecutionCache.ExtcodeHashes.clear(); + InstanceExecutionCache.Keccak256Results.clear(); + InstanceExecutionCache.TxContextCached = false; + + // Reset JIT stack + EVMStackSize = 0; +} + void EVMInstance::setGas(uint64_t NewGas) { Gas = NewGas; } void EVMInstance::pushMessage(evmc_message *Msg) { diff --git a/src/runtime/evm_instance.h b/src/runtime/evm_instance.h index a21a36b9..bfe4c27e 100644 --- a/src/runtime/evm_instance.h +++ b/src/runtime/evm_instance.h @@ -101,6 +101,10 @@ class EVMInstance final : public RuntimeObject { void setError(const Error &E) { Err = E; } void clearError() { Err = ErrorCode::NoError; } + /// Reset instance state for reuse in interpreter mode. + /// Avoids the cost of destroy + recreate on every EVMC execute() call. + void resetForNewCall(evmc_revision NewRev); + // can only called by hostapi directly // setExceptionByHostapi must be inline to capture the hostapi's frame // pointer diff --git a/src/vm/dt_evmc_vm.cpp b/src/vm/dt_evmc_vm.cpp index 2ee646ba..b79714c0 100644 --- a/src/vm/dt_evmc_vm.cpp +++ b/src/vm/dt_evmc_vm.cpp @@ -4,6 +4,7 @@ #include "dt_evmc_vm.h" #include "common/enums.h" #include "common/errors.h" +#include "evm/interpreter.h" #include "runtime/config.h" #include "runtime/evm_instance.h" #include "runtime/isolation.h" @@ -40,33 +41,64 @@ class ScopedConfig { RuntimeConfig PreviousConfig; }; -// CRC32 checksum -uint32_t crc32(const uint8_t *Data, size_t Size) { - static uint32_t Table[256]; - static bool TableInitialized = false; - if (!TableInitialized) { - for (uint32_t I = 0; I < 256; ++I) { - uint32_t C = I; - for (int J = 0; J < 8; ++J) - C = (C & 1) ? (0xEDB88320u ^ (C >> 1)) : (C >> 1); - Table[I] = C; - } - TableInitialized = true; +// ---- Address-based module cache types ---- + +struct CodeAddrRevKey { + evmc_address Addr; + evmc_revision Rev; +}; + +struct CodeAddrRevHash { + size_t operator()(const CodeAddrRevKey &K) const { + uint64_t H; + std::memcpy(&H, K.Addr.bytes + 12, sizeof(H)); + return H ^ (static_cast(K.Rev) * 2654435761u); + } +}; + +struct CodeAddrRevEqual { + bool operator()(const CodeAddrRevKey &A, const CodeAddrRevKey &B) const { + return A.Rev == B.Rev && + std::memcmp(A.Addr.bytes, B.Addr.bytes, sizeof(A.Addr.bytes)) == 0; + } +}; + +/// Validate that the cached module's code matches the provided code. +/// Checks code_size + first 256 bytes + last 256 bytes. +bool validateCodeMatch(const uint8_t *Code, size_t CodeSize, + const EVMModule *Mod) { + if (CodeSize != Mod->CodeSize) + return false; + if (CodeSize == 0) + return true; + auto *ModCode = reinterpret_cast(Mod->Code); + size_t HeadLen = std::min(CodeSize, static_cast(256)); + if (std::memcmp(Code, ModCode, HeadLen) != 0) + return false; + if (CodeSize > 256) { + size_t TailLen = std::min(CodeSize, static_cast(256)); + size_t TailOffset = CodeSize - TailLen; + if (std::memcmp(Code + TailOffset, ModCode + TailOffset, TailLen) != 0) + return false; } - uint32_t Crc = 0xFFFFFFFFu; - for (size_t I = 0; I < Size; ++I) - Crc = Table[(Crc ^ Data[I]) & 0xFFu] ^ (Crc >> 8); - return Crc ^ 0xFFFFFFFFu; + return true; } // VM interface for DTVM struct DTVM : evmc_vm { DTVM(); ~DTVM() { - for (auto &P : LoadedMods) { - EVMModule *Mod = P.second; - if (!RT->unloadEVMModule(Mod)) { - ZEN_LOG_ERROR("failed to unload EVM module"); + // Clean up cached instance first (before modules it may reference) + if (CachedInst && Iso) { + Iso->deleteEVMInstance(CachedInst); + CachedInst = nullptr; + } + // Unload all address-cached modules + if (RT) { + for (auto &P : AddrCache) { + if (!RT->unloadEVMModule(P.second)) { + ZEN_LOG_ERROR("failed to unload EVM module"); + } } } if (Iso) { @@ -78,8 +110,22 @@ struct DTVM : evmc_vm { .EnableEvmGasMetering = true}; std::unique_ptr RT; std::unique_ptr ExecHost; - std::unordered_map LoadedMods; Isolation *Iso = nullptr; + + // ---- Module & instance cache (shared by interpreter and multipass) ---- + // L0: pointer-based inline cache (fastest, 2 integer comparisons) + const uint8_t *LastCodePtr = nullptr; + size_t LastCodeSize = 0; + EVMModule *L0Mod = nullptr; + // L1: address-based cache map (code_address + rev -> module) + std::unordered_map + AddrCache; + uint64_t ModCounter = 0; + // Cached EVMInstance to avoid alloc/free (~33KB) on every call + EVMInstance *CachedInst = nullptr; + // Cached InterpreterExecContext (interpreter mode only) + std::unique_ptr CachedCtx; }; /// The implementation of the evmc_vm::destroy() method. @@ -121,12 +167,170 @@ enum evmc_set_option_result set_option(evmc_vm *VMInstance, const char *Name, return EVMC_SET_OPTION_INVALID_NAME; } +/// Ensure RT and Iso are initialized. Returns false on failure. +bool ensureRuntimeAndIsolation(DTVM *VM) { + if (!VM->RT) { + VM->RT = Runtime::newEVMRuntime(VM->Config, VM->ExecHost.get()); + if (!VM->RT) + return false; + } + if (!VM->Iso) { + VM->Iso = VM->RT->createManagedIsolation(); + if (!VM->Iso) + return false; + } + return true; +} + +/// Find or load a cached EVMModule using two-level cache: +/// L0 (code pointer+size) -> L1 (address map) -> Cold load. +/// Shared by both interpreter and multipass paths. +/// Returns nullptr on failure. +EVMModule *findModuleCached(DTVM *VM, const uint8_t *Code, size_t CodeSize, + evmc_revision Rev, const evmc_message *Msg) { + // L0: Code pointer + size check (instant, ~1ns, 2 integer comparisons) + if (Code == VM->LastCodePtr && CodeSize == VM->LastCodeSize && VM->L0Mod) { + return VM->L0Mod; + } + + EVMModule *Mod = nullptr; + + // L1: Address-based map lookup + CodeAddrRevKey AddrKey{Msg->code_address, Rev}; + auto It = VM->AddrCache.find(AddrKey); + if (It != VM->AddrCache.end() && + validateCodeMatch(Code, CodeSize, It->second)) { + Mod = It->second; + } else { + // Cold path: full module load + // If validation failed for an existing entry, evict the stale module + if (It != VM->AddrCache.end()) { + EVMModule *OldMod = It->second; + if (VM->CachedInst && VM->CachedInst->getModule() == OldMod) { + VM->Iso->deleteEVMInstance(VM->CachedInst); + VM->CachedInst = nullptr; + } + if (VM->L0Mod == OldMod) + VM->L0Mod = nullptr; + VM->RT->unloadEVMModule(OldMod); + VM->AddrCache.erase(It); + } + std::string ModName = "mod_" + std::to_string(VM->ModCounter++); + auto ModRet = VM->RT->loadEVMModule(ModName, Code, CodeSize, Rev); + if (!ModRet) + return nullptr; + Mod = *ModRet; + VM->AddrCache[AddrKey] = Mod; + } + + // Update L0 cache + VM->LastCodePtr = Code; + VM->LastCodeSize = CodeSize; + VM->L0Mod = Mod; + return Mod; +} + +/// Get or create a cached EVMInstance for the given module. +/// Reuses the existing instance if it was created for the same module. +/// Returns nullptr on failure. +EVMInstance *getOrCreateInstance(DTVM *VM, EVMModule *Mod, evmc_revision Rev) { + EVMInstance *TheInst = VM->CachedInst; + if (!TheInst || TheInst->getModule() != Mod) { + // Need a new instance (first call or different module) + if (TheInst) { + VM->Iso->deleteEVMInstance(TheInst); + VM->CachedInst = nullptr; + } + auto InstRet = VM->Iso->createEVMInstance(*Mod, 0); + if (!InstRet) + return nullptr; + TheInst = *InstRet; + VM->CachedInst = TheInst; + } + // Reset instance for this call (cheap: clears fields, keeps allocations) + TheInst->resetForNewCall(Rev); + return TheInst; +} + +/// Fast path for interpreter mode: reuse cached instance, call interpreter +/// directly. This avoids per-call EVMInstance alloc/free and bypasses +/// Runtime::callEVMMain overhead. +evmc_result executeInterpreterFastPath(DTVM *VM, + const evmc_host_interface *Host, + evmc_host_context *Context, + evmc_revision Rev, + const evmc_message *Msg, + const uint8_t *Code, size_t CodeSize) { + // Reinitialize host context + const auto *PrevInterface = VM->ExecHost->getInterface(); + auto *PrevContext = VM->ExecHost->getContext(); + VM->ExecHost->reinitialize(Host, Context); + + // Ensure runtime and isolation exist + if (!ensureRuntimeAndIsolation(VM)) { + VM->ExecHost->reinitialize(PrevInterface, PrevContext); + return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); + } + + // Module lookup: L0 -> L1 -> Cold (shared) + EVMModule *Mod = findModuleCached(VM, Code, CodeSize, Rev, Msg); + if (!Mod) { + VM->ExecHost->reinitialize(PrevInterface, PrevContext); + return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); + } + + // Instance reuse (shared) + EVMInstance *TheInst = getOrCreateInstance(VM, Mod, Rev); + if (!TheInst) { + VM->ExecHost->reinitialize(PrevInterface, PrevContext); + return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); + } + + // Trigger bytecodeCache build if not yet done (lazy, cached on module) + (void)Mod->getBytecodeCache(); + + // Call interpreter directly, bypassing Runtime::callEVMMain overhead + evmc_message MsgWithCode = *Msg; + MsgWithCode.code = reinterpret_cast(Mod->Code); + MsgWithCode.code_size = Mod->CodeSize; + TheInst->setExeResult(evmc::Result{EVMC_SUCCESS, 0, 0}); + TheInst->pushMessage(&MsgWithCode); + + // Reuse cached InterpreterExecContext to avoid ~32KB frame alloc per call + if (!VM->CachedCtx) { + VM->CachedCtx = std::make_unique(TheInst); + } else { + VM->CachedCtx->resetForNewCall(TheInst); + } + auto &Ctx = *VM->CachedCtx; + zen::evm::BaseInterpreter Interpreter(Ctx); + Ctx.allocTopFrame(&MsgWithCode); + Interpreter.interpret(); + + evmc::Result Result = + std::move(const_cast(Ctx.getExeResult())); + Result.gas_left = TheInst->getGas(); + + // Restore host context + VM->ExecHost->reinitialize(PrevInterface, PrevContext); + + return Result.release_raw(); +} + /// The implementation of the evmc_vm::execute() method. evmc_result execute(evmc_vm *EVMInstance, const evmc_host_interface *Host, evmc_host_context *Context, enum evmc_revision Rev, const evmc_message *Msg, const uint8_t *Code, size_t CodeSize) { auto *VM = static_cast(EVMInstance); + + // Interpreter mode: use optimized fast path (bypasses callEVMMain) + if (VM->Config.Mode == RunMode::InterpMode) { + return executeInterpreterFastPath(VM, Host, Context, Rev, Msg, Code, + CodeSize); + } + + // ---- Multipass / other modes: use callEVMMain for JIT execution ---- struct HostContextScope { WrappedHost *ExecHost; const evmc_host_interface *PrevInterface; @@ -142,12 +346,10 @@ evmc_result execute(evmc_vm *EVMInstance, const evmc_host_interface *Host, HostContextScope HostScope(VM->ExecHost.get(), Host, Context); - if (!VM->RT) { - VM->RT = Runtime::newEVMRuntime(VM->Config, VM->ExecHost.get()); - if (!VM->RT) { - return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); - } + if (!ensureRuntimeAndIsolation(VM)) { + return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); } + #ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK // Use interpreter mode for bytecode that would be too expensive to JIT. // The EVMAnalyzer performs a pattern-aware O(n) scan that detects: @@ -168,42 +370,22 @@ evmc_result execute(evmc_vm *EVMInstance, const evmc_host_interface *Host, } #endif // ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK - uint32_t CheckSum = crc32(Code, CodeSize); - uint64_t ModKey = (static_cast(Rev) << 32) | CheckSum; - std::string ModName = - std::to_string(CheckSum) + "_" + std::to_string(static_cast(Rev)); - auto ModRet = VM->RT->loadEVMModule(ModName, Code, CodeSize, Rev); - if (!ModRet) { - const Error &Err = ModRet.getError(); - ZEN_ASSERT(!Err.isEmpty()); - const auto &ErrMsg = Err.getFormattedMessage(false); - return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); - } - - EVMModule *Mod = *ModRet; - VM->LoadedMods[ModKey] = Mod; - if (!VM->Iso) { - VM->Iso = VM->RT->createManagedIsolation(); - } - if (!VM->Iso) { - return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); - } - - auto InstRet = VM->Iso->createEVMInstance(*Mod, 1000000000); - if (!InstRet) { + // Module lookup: L0 -> L1 -> Cold (shared with interpreter path) + EVMModule *Mod = findModuleCached(VM, Code, CodeSize, Rev, Msg); + if (!Mod) { return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); } - auto *TheInst = *InstRet; + // Instance reuse (shared with interpreter path) + auto *TheInst = getOrCreateInstance(VM, Mod, Rev); if (!TheInst) { return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); } - TheInst->setRevision(Rev); + // Execute via callEVMMain (handles both JIT and interpreter fallback) evmc_message Message = *Msg; evmc::Result Result; VM->RT->callEVMMain(*TheInst, Message, Result); - VM->Iso->deleteEVMInstance(TheInst); return Result.release_raw(); }