From 4431a1a48486e2049c36873fe9e606468e639f98 Mon Sep 17 00:00:00 2001 From: Martin Yankov <23098926+Lutherwaves@users.noreply.github.com> Date: Sat, 20 Dec 2025 04:59:08 +0200 Subject: [PATCH 1/2] fix(helm): add custom egress rules to realtime network policy (#2481) The realtime service network policy was missing the custom egress rules section that allows configuration of additional egress rules via values.yaml. This caused the realtime pods to be unable to connect to external databases (e.g., PostgreSQL on port 5432) when using external database configurations. The app network policy already had this section, but the realtime network policy was missing it, creating an inconsistency and preventing the realtime service from accessing external databases configured via networkPolicy.egress values. This fix adds the same custom egress rules template section to the realtime network policy, matching the app network policy behavior and allowing users to configure database connectivity via values.yaml. --- helm/sim/templates/networkpolicy.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/sim/templates/networkpolicy.yaml b/helm/sim/templates/networkpolicy.yaml index deac5a5dba..7ef8697417 100644 --- a/helm/sim/templates/networkpolicy.yaml +++ b/helm/sim/templates/networkpolicy.yaml @@ -141,6 +141,10 @@ spec: ports: - protocol: TCP port: 443 + # Allow custom egress rules + {{- with .Values.networkPolicy.egress }} + {{- toYaml . | nindent 2 }} + {{- end }} {{- end }} {{- if .Values.postgresql.enabled }} From 8778b6267e45f713562154645a7a2b8c880626a2 Mon Sep 17 00:00:00 2001 From: majiayu000 <1835304752@qq.com> Date: Mon, 22 Dec 2025 18:54:07 +0800 Subject: [PATCH 2/2] feat(scheduler): add internal scheduler for self-hosted environments Adds built-in scheduler that periodically polls /api/schedules/execute to trigger scheduled workflows in self-hosted environments. Enable by setting ENABLE_INTERNAL_SCHEDULER=true (enabled by default in docker-compose.prod.yml). Also requires CRON_SECRET to be configured. Fixes #1870 --- apps/sim/instrumentation-node.ts | 8 ++ apps/sim/lib/core/config/env.ts | 2 + .../lib/scheduler/internal-scheduler.test.ts | 97 +++++++++++++ apps/sim/lib/scheduler/internal-scheduler.ts | 134 ++++++++++++++++++ docker-compose.prod.yml | 4 + 5 files changed, 245 insertions(+) create mode 100644 apps/sim/lib/scheduler/internal-scheduler.test.ts create mode 100644 apps/sim/lib/scheduler/internal-scheduler.ts diff --git a/apps/sim/instrumentation-node.ts b/apps/sim/instrumentation-node.ts index 86c10996e6..4636bead03 100644 --- a/apps/sim/instrumentation-node.ts +++ b/apps/sim/instrumentation-node.ts @@ -115,4 +115,12 @@ async function initializeOpenTelemetry() { export async function register() { await initializeOpenTelemetry() + + // Initialize internal scheduler for self-hosted environments + try { + const { initializeInternalScheduler } = await import('./lib/scheduler/internal-scheduler') + initializeInternalScheduler() + } catch (error) { + logger.error('Failed to initialize internal scheduler', error) + } } diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index dc627c01a5..f3df8f03d4 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -125,6 +125,8 @@ export const env = createEnv({ TRIGGER_SECRET_KEY: z.string().min(1).optional(), // Trigger.dev secret key for background jobs TRIGGER_DEV_ENABLED: z.boolean().optional(), // Toggle to enable/disable Trigger.dev for async jobs CRON_SECRET: z.string().optional(), // Secret for authenticating cron job requests + ENABLE_INTERNAL_SCHEDULER: z.string().optional(), // Enable built-in scheduler for self-hosted environments + INTERNAL_SCHEDULER_INTERVAL_MS: z.string().optional(), // Internal scheduler poll interval (default: 60000ms) JOB_RETENTION_DAYS: z.string().optional().default('1'), // Days to retain job logs/data // Cloud Storage - AWS S3 diff --git a/apps/sim/lib/scheduler/internal-scheduler.test.ts b/apps/sim/lib/scheduler/internal-scheduler.test.ts new file mode 100644 index 0000000000..0692ee2abf --- /dev/null +++ b/apps/sim/lib/scheduler/internal-scheduler.test.ts @@ -0,0 +1,97 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/config/env', () => ({ + env: { + ENABLE_INTERNAL_SCHEDULER: 'true', + CRON_SECRET: 'test-secret', + NEXT_PUBLIC_APP_URL: 'http://localhost:3000', + INTERNAL_SCHEDULER_INTERVAL_MS: '1000', + }, +})) + +vi.mock('@/lib/logs/console/logger', () => ({ + createLogger: vi.fn().mockReturnValue({ + info: vi.fn(), + error: vi.fn(), + warn: vi.fn(), + debug: vi.fn(), + }), +})) + +const mockFetch = vi.fn() +global.fetch = mockFetch + +describe('Internal Scheduler', () => { + beforeEach(() => { + vi.clearAllMocks() + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ executedCount: 0 }), + }) + }) + + afterEach(() => { + vi.clearAllMocks() + }) + + it('should poll schedules endpoint with correct authentication', async () => { + const { startInternalScheduler, stopInternalScheduler } = await import( + './internal-scheduler' + ) + + startInternalScheduler() + + // Wait for the initial poll to complete + await new Promise((resolve) => setTimeout(resolve, 100)) + + expect(mockFetch).toHaveBeenCalledWith( + 'http://localhost:3000/api/schedules/execute', + expect.objectContaining({ + method: 'GET', + headers: expect.objectContaining({ + Authorization: 'Bearer test-secret', + 'User-Agent': 'sim-studio-internal-scheduler/1.0', + }), + }) + ) + + stopInternalScheduler() + }) + + it('should handle fetch errors gracefully', async () => { + mockFetch.mockRejectedValueOnce(new Error('Network error')) + + const { startInternalScheduler, stopInternalScheduler } = await import( + './internal-scheduler' + ) + + // Should not throw + startInternalScheduler() + await new Promise((resolve) => setTimeout(resolve, 100)) + stopInternalScheduler() + }) + + it('should handle non-ok responses', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 401, + text: async () => 'Unauthorized', + }) + + const { startInternalScheduler, stopInternalScheduler } = await import( + './internal-scheduler' + ) + + // Should not throw + startInternalScheduler() + await new Promise((resolve) => setTimeout(resolve, 100)) + stopInternalScheduler() + }) +}) + +describe('shouldEnableInternalScheduler', () => { + it('should return true when ENABLE_INTERNAL_SCHEDULER is true', async () => { + const { shouldEnableInternalScheduler } = await import('./internal-scheduler') + expect(shouldEnableInternalScheduler()).toBe(true) + }) +}) diff --git a/apps/sim/lib/scheduler/internal-scheduler.ts b/apps/sim/lib/scheduler/internal-scheduler.ts new file mode 100644 index 0000000000..2532a00b9b --- /dev/null +++ b/apps/sim/lib/scheduler/internal-scheduler.ts @@ -0,0 +1,134 @@ +/** + * Internal Scheduler for Self-Hosted Environments + * + * This module provides a built-in scheduler that periodically polls the + * /api/schedules/execute endpoint to trigger scheduled workflows. + * This is necessary for self-hosted environments that don't have access + * to external cron services like Vercel Cron Jobs. + * + * Enable by setting ENABLE_INTERNAL_SCHEDULER=true in your environment. + */ + +import { env } from '@/lib/core/config/env' +import { createLogger } from '@/lib/logs/console/logger' + +const logger = createLogger('InternalScheduler') + +const DEFAULT_POLL_INTERVAL_MS = 60000 // 1 minute + +let schedulerInterval: ReturnType | null = null +let isRunning = false + +/** + * Execute the schedule poll + */ +async function pollSchedules(): Promise { + if (isRunning) { + logger.debug('Previous poll still running, skipping this cycle') + return + } + + isRunning = true + + try { + const appUrl = env.NEXT_PUBLIC_APP_URL || env.BETTER_AUTH_URL || 'http://localhost:3000' + const cronSecret = env.CRON_SECRET + + if (!cronSecret) { + logger.warn('CRON_SECRET not configured, internal scheduler cannot authenticate') + return + } + + const response = await fetch(`${appUrl}/api/schedules/execute`, { + method: 'GET', + headers: { + Authorization: `Bearer ${cronSecret}`, + 'User-Agent': 'sim-studio-internal-scheduler/1.0', + }, + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error('Schedule poll failed', { + status: response.status, + error: errorText, + }) + return + } + + const result = await response.json() + if (result.executedCount > 0) { + logger.info(`Triggered ${result.executedCount} scheduled workflow(s)`) + } + } catch (error) { + logger.error('Error during schedule poll', error) + } finally { + isRunning = false + } +} + +/** + * Start the internal scheduler + */ +export function startInternalScheduler(): void { + if (schedulerInterval) { + logger.warn('Internal scheduler already running') + return + } + + const pollInterval = Number(env.INTERNAL_SCHEDULER_INTERVAL_MS) || DEFAULT_POLL_INTERVAL_MS + + logger.info(`Starting internal scheduler with poll interval: ${pollInterval}ms`) + + // Run immediately on start + void pollSchedules() + + // Then run at regular intervals + schedulerInterval = setInterval(() => { + void pollSchedules() + }, pollInterval) +} + +/** + * Stop the internal scheduler + */ +export function stopInternalScheduler(): void { + if (schedulerInterval) { + clearInterval(schedulerInterval) + schedulerInterval = null + logger.info('Internal scheduler stopped') + } +} + +/** + * Check if the internal scheduler should be enabled + */ +export function shouldEnableInternalScheduler(): boolean { + return env.ENABLE_INTERNAL_SCHEDULER === 'true' +} + +/** + * Initialize the internal scheduler if enabled + */ +export function initializeInternalScheduler(): void { + if (!shouldEnableInternalScheduler()) { + logger.debug('Internal scheduler disabled (set ENABLE_INTERNAL_SCHEDULER=true to enable)') + return + } + + if (!env.CRON_SECRET) { + logger.warn('Cannot start internal scheduler: CRON_SECRET is not configured') + return + } + + startInternalScheduler() + + // Graceful shutdown handlers + process.on('SIGTERM', () => { + stopInternalScheduler() + }) + + process.on('SIGINT', () => { + stopInternalScheduler() + }) +} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c6b79e6c1e..f4cfd9fbee 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -20,6 +20,10 @@ services: - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://localhost:3002} - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} + # Internal scheduler for self-hosted environments (enables scheduled workflows) + - ENABLE_INTERNAL_SCHEDULER=${ENABLE_INTERNAL_SCHEDULER:-true} + - CRON_SECRET=${CRON_SECRET:-default-cron-secret-change-me} + - INTERNAL_SCHEDULER_INTERVAL_MS=${INTERNAL_SCHEDULER_INTERVAL_MS:-60000} depends_on: db: condition: service_healthy