From 387efb6c7462778715f4d7568c32c4e498c96cd4 Mon Sep 17 00:00:00 2001 From: Aaron Adams Date: Mon, 22 Dec 2025 13:19:46 -0600 Subject: [PATCH] fix: prevent memory exhaustion in loops with bounded iteration outputs This change addresses memory exhaustion issues that occur when running workflows with loops containing agent blocks that make many tool calls. Problem: Memory accumulated unbounded in two key areas: 1. allIterationOutputs in LoopScope - every iteration pushed results 2. blockLogs in ExecutionContext - every block execution added logs Solution: Added memory management with configurable limits in constants.ts: - MAX_STORED_ITERATION_OUTPUTS (100) and MAX_ITERATION_OUTPUTS_SIZE_BYTES (50MB) - MAX_BLOCK_LOGS (500) and MAX_BLOCK_LOGS_SIZE_BYTES (100MB) Loop orchestrator (loop.ts): - New addIterationOutputsWithMemoryLimit() method - Periodic size checks (every 10 iterations) to avoid serialization overhead - Discards oldest iterations when limits exceeded Block executor (block-executor.ts): - New addBlockLogWithMemoryLimit() method - Periodic size checks (every 50 logs) - Discards oldest logs when limits exceeded Trade-offs: - Final aggregated results contain only recent iterations - Logs show warning when truncation occurs for debugging Fixes #2525 --- apps/sim/executor/constants.ts | 20 ++++++ apps/sim/executor/execution/block-executor.ts | 54 ++++++++++++++- apps/sim/executor/orchestrators/loop.ts | 69 ++++++++++++++++++- 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/apps/sim/executor/constants.ts b/apps/sim/executor/constants.ts index 01814c9b37..19e63a08e6 100644 --- a/apps/sim/executor/constants.ts +++ b/apps/sim/executor/constants.ts @@ -130,6 +130,26 @@ export const DEFAULTS = { MAX_LOOP_ITERATIONS: 1000, MAX_WORKFLOW_DEPTH: 10, EXECUTION_TIME: 0, + /** + * Maximum number of iteration outputs to retain in memory during loop execution. + * Older iterations are discarded to prevent memory exhaustion in long-running loops. + * The final aggregated results will contain only the most recent iterations. + */ + MAX_STORED_ITERATION_OUTPUTS: 100, + /** + * Maximum size in bytes for iteration outputs before triggering truncation. + * This is an approximate estimate based on JSON serialization. + */ + MAX_ITERATION_OUTPUTS_SIZE_BYTES: 50 * 1024 * 1024, // 50MB + /** + * Maximum number of block logs to retain in memory during execution. + * Older logs are discarded to prevent memory exhaustion in long-running workflows. + */ + MAX_BLOCK_LOGS: 500, + /** + * Maximum size in bytes for block logs before triggering truncation. + */ + MAX_BLOCK_LOGS_SIZE_BYTES: 100 * 1024 * 1024, // 100MB TOKENS: { PROMPT: 0, COMPLETION: 0, diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index b0723df04e..83d0b80c2d 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -59,7 +59,7 @@ export class BlockExecutor { let blockLog: BlockLog | undefined if (!isSentinel) { blockLog = this.createBlockLog(ctx, node.id, block, node) - ctx.blockLogs.push(blockLog) + this.addBlockLogWithMemoryLimit(ctx, blockLog) this.callOnBlockStart(ctx, node, block) } @@ -658,4 +658,56 @@ export class BlockExecutor { executionOutput.content = fullContent } + + /** + * Adds a block log to the execution context with memory management. + * Prevents unbounded memory growth by: + * 1. Limiting the number of stored logs (MAX_BLOCK_LOGS) + * 2. Checking estimated memory size (MAX_BLOCK_LOGS_SIZE_BYTES) + * + * When limits are exceeded, older logs are discarded to make room for newer ones. + */ + private addBlockLogWithMemoryLimit(ctx: ExecutionContext, blockLog: BlockLog): void { + ctx.blockLogs.push(blockLog) + + // Check log count limit + if (ctx.blockLogs.length > DEFAULTS.MAX_BLOCK_LOGS) { + const discardCount = ctx.blockLogs.length - DEFAULTS.MAX_BLOCK_LOGS + ctx.blockLogs = ctx.blockLogs.slice(discardCount) + logger.warn('Block logs exceeded count limit, discarding older logs', { + discardedCount: discardCount, + retainedCount: ctx.blockLogs.length, + maxAllowed: DEFAULTS.MAX_BLOCK_LOGS, + }) + } + + // Periodically check memory size (every 50 logs to avoid frequent serialization) + if (ctx.blockLogs.length % 50 === 0) { + const estimatedSize = this.estimateBlockLogsSize(ctx.blockLogs) + if (estimatedSize > DEFAULTS.MAX_BLOCK_LOGS_SIZE_BYTES) { + const halfLength = Math.floor(ctx.blockLogs.length / 2) + const discardCount = Math.max(halfLength, 1) + ctx.blockLogs = ctx.blockLogs.slice(discardCount) + logger.warn('Block logs exceeded memory limit, discarding older logs', { + estimatedSizeBytes: estimatedSize, + maxSizeBytes: DEFAULTS.MAX_BLOCK_LOGS_SIZE_BYTES, + discardedCount: discardCount, + retainedCount: ctx.blockLogs.length, + }) + } + } + } + + /** + * Estimates the memory size of block logs in bytes. + * Returns a value exceeding the limit on serialization failure to trigger cleanup. + */ + private estimateBlockLogsSize(logs: BlockLog[]): number { + try { + return JSON.stringify(logs).length * 2 + } catch { + // Return a value that exceeds the limit to trigger cleanup on serialization failure + return DEFAULTS.MAX_BLOCK_LOGS_SIZE_BYTES + 1 + } + } } diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index 2378cded58..a8c23f6287 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -141,7 +141,7 @@ export class LoopOrchestrator { } if (iterationResults.length > 0) { - scope.allIterationOutputs.push(iterationResults) + this.addIterationOutputsWithMemoryLimit(scope, iterationResults, loopId) } scope.currentIterationOutputs.clear() @@ -462,4 +462,71 @@ export class LoopOrchestrator { return [] } } + + /** + * Adds iteration outputs to the loop scope with memory management. + * Prevents unbounded memory growth by: + * 1. Limiting the number of stored iterations (MAX_STORED_ITERATION_OUTPUTS) + * 2. Checking estimated memory size (MAX_ITERATION_OUTPUTS_SIZE_BYTES) + * + * When limits are exceeded, older iterations are discarded to make room for newer ones. + * This ensures long-running loops don't cause memory exhaustion while still providing + * access to recent iteration results. + */ + private addIterationOutputsWithMemoryLimit( + scope: LoopScope, + iterationResults: NormalizedBlockOutput[], + loopId: string + ): void { + scope.allIterationOutputs.push(iterationResults) + + // Check iteration count limit + if (scope.allIterationOutputs.length > DEFAULTS.MAX_STORED_ITERATION_OUTPUTS) { + const discardCount = scope.allIterationOutputs.length - DEFAULTS.MAX_STORED_ITERATION_OUTPUTS + scope.allIterationOutputs = scope.allIterationOutputs.slice(discardCount) + logger.warn('Loop iteration outputs exceeded count limit, discarding older iterations', { + loopId, + iteration: scope.iteration, + discardedCount: discardCount, + retainedCount: scope.allIterationOutputs.length, + maxAllowed: DEFAULTS.MAX_STORED_ITERATION_OUTPUTS, + }) + } + + // Periodically check memory size limit (every 10 iterations to avoid frequent serialization) + if (scope.allIterationOutputs.length % 10 === 0) { + const estimatedSize = this.estimateObjectSize(scope.allIterationOutputs) + if (estimatedSize > DEFAULTS.MAX_ITERATION_OUTPUTS_SIZE_BYTES) { + // Discard oldest half of iterations when memory limit exceeded + const halfLength = Math.floor(scope.allIterationOutputs.length / 2) + const discardCount = Math.max(halfLength, 1) + scope.allIterationOutputs = scope.allIterationOutputs.slice(discardCount) + logger.warn('Loop iteration outputs exceeded memory limit, discarding older iterations', { + loopId, + iteration: scope.iteration, + estimatedSizeBytes: estimatedSize, + maxSizeBytes: DEFAULTS.MAX_ITERATION_OUTPUTS_SIZE_BYTES, + discardedCount: discardCount, + retainedCount: scope.allIterationOutputs.length, + }) + } + } + } + + /** + * Estimates the memory size of an object in bytes. + * This is an approximation based on JSON serialization size. + * Actual memory usage may vary due to object overhead and references. + */ + private estimateObjectSize(obj: unknown): number { + try { + // Use JSON.stringify length as a rough estimate + // Multiply by 2 for UTF-16 encoding overhead in JS strings + return JSON.stringify(obj).length * 2 + } catch { + // If serialization fails (circular refs, etc.), return a value that exceeds + // the limit to trigger cleanup as a safety measure + return DEFAULTS.MAX_ITERATION_OUTPUTS_SIZE_BYTES + 1 + } + } }