From ef9bea832df3e6cd20cac777cdc78df23434cfbf Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 09:42:51 -0500 Subject: [PATCH 01/23] docs(site): add canonical stem vs bullmq comparison --- .site/docs/comparisons/stem-vs-bullmq.md | 52 ++++++++++++++++++++++++ .site/docs/getting-started/index.md | 1 + .site/sidebars.ts | 1 + 3 files changed, 54 insertions(+) create mode 100644 .site/docs/comparisons/stem-vs-bullmq.md diff --git a/.site/docs/comparisons/stem-vs-bullmq.md b/.site/docs/comparisons/stem-vs-bullmq.md new file mode 100644 index 00000000..14682336 --- /dev/null +++ b/.site/docs/comparisons/stem-vs-bullmq.md @@ -0,0 +1,52 @@ +--- +title: Stem vs BullMQ +sidebar_label: Stem vs BullMQ +sidebar_position: 1 +slug: /comparisons/stem-vs-bullmq +--- + +This page is the canonical Stem comparison matrix for BullMQ-style features. +It focuses on capability parity, not API-level compatibility. + +**As of:** February 24, 2026 + +## Status semantics + +| Status | Meaning | +| --- | --- | +| `✓` | Functionally equivalent built-in capability exists in Stem. | +| `~` | Partial or non-isomorphic capability exists, but semantics differ from BullMQ/BullMQ-Pro. | +| `✗` | No built-in capability in Stem today. | + +## Feature matrix + +| BullMQ row | Stem | Rationale (with evidence) | +| --- | --- | --- | +| Backend | `✓` | Stem supports multiple backends/adapters (Redis, Postgres, SQLite, in-memory). See [Broker Overview](../brokers/overview.md) and [Developer Environment](../getting-started/developer-environment.md). | +| Observables | `✓` | Stem has built-in metrics, tracing, and lifecycle signals. See [Observability](../core-concepts/observability.md) and [Signals](../core-concepts/signals.md). | +| Group Rate Limit | `✗` | Stem supports per-task rate limiting but no built-in group-level limiter primitive. See [Rate Limiting](../core-concepts/rate-limiting.md). | +| Group Support | `✓` | Stem provides `Canvas.group` and `Canvas.chord` primitives. See [Canvas Patterns](../core-concepts/canvas.md). | +| Batches Support | `~` | Group/chord and workflow composition cover many batch workflows, but there is no dedicated BullMQ-Pro-style batch primitive. See [Canvas Patterns](../core-concepts/canvas.md) and [Workflows](../core-concepts/workflows.md). | +| Parent/Child Dependencies | `✓` | Stem supports dependency composition through chains, groups/chords, and workflow steps. See [Canvas Patterns](../core-concepts/canvas.md) and [Workflows](../core-concepts/workflows.md). | +| Deduplication (Debouncing) | `~` | `TaskOptions.unique` prevents duplicate enqueue claims, but semantics are lock/TTL-based rather than BullMQ-native dedupe APIs. See [Uniqueness](../core-concepts/uniqueness.md). | +| Deduplication (Throttling) | `~` | `uniqueFor` and lock TTL windows approximate throttling behavior, but are not a direct BullMQ equivalent. See [Uniqueness](../core-concepts/uniqueness.md). | +| Priorities | `✓` | Stem supports task priority and queue priority ranges. See [Tasks](../core-concepts/tasks.md) and [Routing](../core-concepts/routing.md). | +| Concurrency | `✓` | Workers support configurable concurrency and isolate pools. See [Workers](../workers/index.md) and [Worker Control](../workers/worker-control.md). | +| Delayed jobs | `✓` | Delayed execution is supported via enqueue options and broker scheduling. See [Quick Start](../getting-started/quick-start.md) and [Broker Overview](../brokers/overview.md). | +| Global events | `✓` | Stem exposes global lifecycle events through signal dispatch. See [Signals](../core-concepts/signals.md). | +| Rate Limiter | `✓` | Stem supports per-task rate limits with pluggable limiter backends. See [Rate Limiting](../core-concepts/rate-limiting.md). | +| Pause/Resume | `~` | Stem supports worker control (`shutdown`, `revoke`) and workflow suspend/resume, but not a direct queue pause/resume API. See [Worker Control](../workers/worker-control.md) and [Workflows](../core-concepts/workflows.md). | +| Sandboxed worker | `~` | Stem supports isolate-based execution boundaries, but this is not equivalent to BullMQ's Node child-process sandbox model. See [Worker Control](../workers/worker-control.md). | +| Repeatable jobs | `✓` | Stem Beat supports interval, cron, solar, and clocked schedules. See [Scheduler](../scheduler/index.md) and [Beat Scheduler Guide](../scheduler/beat-guide.md). | +| Atomic ops | `~` | Stem includes atomic behavior in specific stores/flows, but end-to-end transactional guarantees (for all enqueue/ack/result paths) are not universally built-in. See [Tasks idempotency guidance](../core-concepts/tasks.md#idempotency-checklist) and [Best Practices](../getting-started/best-practices.md). | +| Persistence | `✓` | Stem persists task/workflow/schedule state through pluggable backends/stores. See [Persistence & Stores](../core-concepts/persistence.md). | +| UI | `~` | Stem includes an experimental dashboard, not a fully mature operator UI parity target yet. See [Dashboard](../core-concepts/dashboard.md). | +| Optimized for | `~` | Stem is optimized for jobs/messages plus durable workflow orchestration, not only queue semantics. See [Core Concepts](../core-concepts/index.md) and [Workflows](../core-concepts/workflows.md). | + +## Update policy + +When this matrix changes: + +1. Update the **As of** date. +2. Keep row names aligned with BullMQ terminology. +3. Update rationale links so every status remains auditable. diff --git a/.site/docs/getting-started/index.md b/.site/docs/getting-started/index.md index 99a8ef53..7e964ed9 100644 --- a/.site/docs/getting-started/index.md +++ b/.site/docs/getting-started/index.md @@ -18,6 +18,7 @@ want to explore further. - **[Observe & Operate](./observability-and-ops.md)** – Enable OpenTelemetry export, inspect workers/queues/DLQ via CLI, and wire lifecycle signals. - **[Prepare for Production](./production-checklist.md)** – Apply signing/TLS, deploy with systemd or CLI multi-process tooling, and run quality gates before launch. - **[Troubleshooting](./troubleshooting.md)** – Common errors and quick fixes while onboarding. +- **[Stem vs BullMQ](../comparisons/stem-vs-bullmq.md)** – Canonical feature mapping with `✓/~ /✗` parity semantics. Once you complete the journey, continue with the in-depth material under [Core Concepts](../core-concepts/index.md) and [Workers](../workers/index.md). diff --git a/.site/sidebars.ts b/.site/sidebars.ts index 92f6803d..2c8544c3 100644 --- a/.site/sidebars.ts +++ b/.site/sidebars.ts @@ -29,6 +29,7 @@ const sidebars: SidebarsConfig = { type: "category", label: "Guides", items: [ + "comparisons/stem-vs-bullmq", "getting-started/observability-and-ops", "getting-started/production-checklist", "getting-started/troubleshooting", From 6321d542940343ccefe7f4a775f3eb548a8e2411 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 12:13:54 -0500 Subject: [PATCH 02/23] feat(stem): close bullmq parity gaps across events, control, and stores --- .site/docs/brokers/sqlite.md | 10 +- .site/docs/comparisons/stem-vs-bullmq.md | 18 +- .site/docs/core-concepts/canvas.md | 13 + .site/docs/core-concepts/cli-control.md | 21 +- .site/docs/core-concepts/index.md | 3 + .site/docs/core-concepts/persistence.md | 9 +- .site/docs/core-concepts/queue-events.md | 50 + .site/docs/core-concepts/rate-limiting.md | 21 + .site/docs/core-concepts/signals.md | 121 +-- .site/docs/workers/worker-control.md | 35 +- .site/sidebars.ts | 1 + .../docs_snippets/lib/canvas_batch.dart | 41 + .../docs_snippets/lib/cli_control.dart | 10 + .../docs_snippets/lib/persistence.dart | 24 +- .../docs_snippets/lib/queue_events.dart | 69 ++ .../docs_snippets/lib/rate_limiting.dart | 21 + .../example/docs_snippets/lib/signals.dart | 46 +- packages/stem/lib/src/canvas/canvas.dart | 176 ++++ packages/stem/lib/src/core/contracts.dart | 58 + packages/stem/lib/src/core/queue_events.dart | 269 +++++ packages/stem/lib/src/core/stem_event.dart | 11 + packages/stem/lib/src/signals/emitter.dart | 40 +- packages/stem/lib/src/signals/payloads.dart | 346 +++++- packages/stem/lib/src/signals/signal.dart | 36 +- .../stem/lib/src/signals/stem_signals.dart | 140 ++- packages/stem/lib/src/worker/worker.dart | 529 ++++++++-- .../runtime/workflow_introspection.dart | 21 +- packages/stem/lib/stem.dart | 2 + .../stem/test/unit/canvas/canvas_test.dart | 41 + .../stem/test/unit/core/contracts_test.dart | 11 + .../test/unit/core/queue_events_test.dart | 136 +++ .../stem/test/unit/core/stem_event_test.dart | 46 + .../stem/test/unit/signals/signal_test.dart | 86 +- .../test/unit/signals/stem_signals_test.dart | 152 +++ .../stem/test/unit/worker/worker_test.dart | 368 +++++++ packages/stem_adapter_tests/README.md | 11 + .../lib/src/contract_capabilities.dart | 22 + .../lib/src/queue_events_contract_suite.dart | 251 +++++ .../lib/src/revoke_store_contract_suite.dart | 194 ++++ .../lib/stem_adapter_tests.dart | 2 + .../test/contract_suite_exports_test.dart | 2 + .../queue_events_contract_suite_test.dart | 16 + .../revoke_store_contract_suite_test.dart | 12 + .../lib/src/cli/revoke_store_factory.dart | 3 + packages/stem_cli/lib/src/cli/worker.dart | 198 ++++ .../test/unit/cli/cli_worker_stats_test.dart | 89 +- .../unit/cli/revoke_store_factory_test.dart | 40 + .../postgres_broker_integration_test.dart | 22 + .../redis_broker_integration_test.dart | 39 + packages/stem_sqlite/lib/orm_registry.g.dart | 24 +- .../lib/src/control/sqlite_revoke_store.dart | 168 +++ .../lib/src/database/migrations.dart | 8 + .../m_20260224103000_add_revoke_store.dart | 35 + .../stem_sqlite/lib/src/models/models.dart | 1 + .../lib/src/models/stem_revoke_entry.dart | 47 + .../lib/src/models/stem_revoke_entry.orm.dart | 996 ++++++++++++++++++ .../lib/src/stack/sqlite_adapter.dart | 5 +- .../lib/src/workflow/sqlite_factories.dart | 16 + packages/stem_sqlite/lib/stem_sqlite.dart | 2 + .../test/broker/sqlite_broker_test.dart | 20 + .../control/sqlite_revoke_store_test.dart | 105 ++ 61 files changed, 5062 insertions(+), 247 deletions(-) create mode 100644 .site/docs/core-concepts/queue-events.md create mode 100644 packages/stem/example/docs_snippets/lib/canvas_batch.dart create mode 100644 packages/stem/example/docs_snippets/lib/queue_events.dart create mode 100644 packages/stem/lib/src/core/queue_events.dart create mode 100644 packages/stem/lib/src/core/stem_event.dart create mode 100644 packages/stem/test/unit/core/queue_events_test.dart create mode 100644 packages/stem/test/unit/core/stem_event_test.dart create mode 100644 packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart create mode 100644 packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart create mode 100644 packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart create mode 100644 packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart create mode 100644 packages/stem_sqlite/lib/src/control/sqlite_revoke_store.dart create mode 100644 packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart create mode 100644 packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart create mode 100644 packages/stem_sqlite/lib/src/models/stem_revoke_entry.orm.dart create mode 100644 packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart diff --git a/.site/docs/brokers/sqlite.md b/.site/docs/brokers/sqlite.md index 654c4273..48aa5ebd 100644 --- a/.site/docs/brokers/sqlite.md +++ b/.site/docs/brokers/sqlite.md @@ -5,8 +5,8 @@ sidebar_position: 2 slug: /brokers/sqlite --- -Stem ships a SQLite adapter in `stem_sqlite` that implements both the broker -and result backend contracts. It is designed for local development, demo +Stem ships a SQLite adapter in `stem_sqlite` that implements broker, result +backend, and revoke store contracts. It is designed for local development, demo environments, and single-node deployments that want a zero-infra dependency. ## When to use SQLite @@ -41,6 +41,12 @@ dependencies: ``` +## Quick start (revoke store) + +```dart title="persistence.dart" file=/../packages/stem/example/docs_snippets/lib/persistence.dart#persistence-revoke-store-sqlite + +``` + ## Configuration knobs SQLite adapters expose the same tuning hooks as other brokers/backends: diff --git a/.site/docs/comparisons/stem-vs-bullmq.md b/.site/docs/comparisons/stem-vs-bullmq.md index 14682336..a8a22818 100644 --- a/.site/docs/comparisons/stem-vs-bullmq.md +++ b/.site/docs/comparisons/stem-vs-bullmq.md @@ -24,18 +24,18 @@ It focuses on capability parity, not API-level compatibility. | --- | --- | --- | | Backend | `✓` | Stem supports multiple backends/adapters (Redis, Postgres, SQLite, in-memory). See [Broker Overview](../brokers/overview.md) and [Developer Environment](../getting-started/developer-environment.md). | | Observables | `✓` | Stem has built-in metrics, tracing, and lifecycle signals. See [Observability](../core-concepts/observability.md) and [Signals](../core-concepts/signals.md). | -| Group Rate Limit | `✗` | Stem supports per-task rate limiting but no built-in group-level limiter primitive. See [Rate Limiting](../core-concepts/rate-limiting.md). | +| Group Rate Limit | `✓` | Stem supports group-scoped rate limiting via `TaskOptions.groupRateLimit`, `groupRateKey`, and `groupRateKeyHeader`. See [Rate Limiting](../core-concepts/rate-limiting.md). | | Group Support | `✓` | Stem provides `Canvas.group` and `Canvas.chord` primitives. See [Canvas Patterns](../core-concepts/canvas.md). | -| Batches Support | `~` | Group/chord and workflow composition cover many batch workflows, but there is no dedicated BullMQ-Pro-style batch primitive. See [Canvas Patterns](../core-concepts/canvas.md) and [Workflows](../core-concepts/workflows.md). | +| Batches Support | `✓` | Stem exposes first-class batch APIs (`submitBatch`, `inspectBatch`) with durable batch lifecycle status. See [Canvas Patterns](../core-concepts/canvas.md). | | Parent/Child Dependencies | `✓` | Stem supports dependency composition through chains, groups/chords, and workflow steps. See [Canvas Patterns](../core-concepts/canvas.md) and [Workflows](../core-concepts/workflows.md). | | Deduplication (Debouncing) | `~` | `TaskOptions.unique` prevents duplicate enqueue claims, but semantics are lock/TTL-based rather than BullMQ-native dedupe APIs. See [Uniqueness](../core-concepts/uniqueness.md). | | Deduplication (Throttling) | `~` | `uniqueFor` and lock TTL windows approximate throttling behavior, but are not a direct BullMQ equivalent. See [Uniqueness](../core-concepts/uniqueness.md). | | Priorities | `✓` | Stem supports task priority and queue priority ranges. See [Tasks](../core-concepts/tasks.md) and [Routing](../core-concepts/routing.md). | | Concurrency | `✓` | Workers support configurable concurrency and isolate pools. See [Workers](../workers/index.md) and [Worker Control](../workers/worker-control.md). | | Delayed jobs | `✓` | Delayed execution is supported via enqueue options and broker scheduling. See [Quick Start](../getting-started/quick-start.md) and [Broker Overview](../brokers/overview.md). | -| Global events | `✓` | Stem exposes global lifecycle events through signal dispatch. See [Signals](../core-concepts/signals.md). | +| Global events | `✓` | Stem exposes global lifecycle events through `StemSignals`, plus queue-scoped custom events through `QueueEvents`. See [Signals](../core-concepts/signals.md) and [Queue Events](../core-concepts/queue-events.md). | | Rate Limiter | `✓` | Stem supports per-task rate limits with pluggable limiter backends. See [Rate Limiting](../core-concepts/rate-limiting.md). | -| Pause/Resume | `~` | Stem supports worker control (`shutdown`, `revoke`) and workflow suspend/resume, but not a direct queue pause/resume API. See [Worker Control](../workers/worker-control.md) and [Workflows](../core-concepts/workflows.md). | +| Pause/Resume | `✓` | Stem provides queue pause/resume commands (`stem worker pause`, `stem worker resume`) and persistent pause state when a revoke store is configured. See [Worker Control](../workers/worker-control.md). | | Sandboxed worker | `~` | Stem supports isolate-based execution boundaries, but this is not equivalent to BullMQ's Node child-process sandbox model. See [Worker Control](../workers/worker-control.md). | | Repeatable jobs | `✓` | Stem Beat supports interval, cron, solar, and clocked schedules. See [Scheduler](../scheduler/index.md) and [Beat Scheduler Guide](../scheduler/beat-guide.md). | | Atomic ops | `~` | Stem includes atomic behavior in specific stores/flows, but end-to-end transactional guarantees (for all enqueue/ack/result paths) are not universally built-in. See [Tasks idempotency guidance](../core-concepts/tasks.md#idempotency-checklist) and [Best Practices](../getting-started/best-practices.md). | @@ -45,6 +45,16 @@ It focuses on capability parity, not API-level compatibility. ## Update policy +## BullMQ events parity notes + +Stem supports the two common BullMQ event-listening styles: + +| BullMQ concept | Stem equivalent | +| --- | --- | +| `QueueEvents` listeners | `QueueEvents` + `QueueEventsProducer` (queue-scoped custom events) | +| Custom queue events | `producer.emit(queue, eventName, payload, headers, meta)` | +| Worker-specific event listeners | `StemSignals` convenience APIs with `workerId` filters (`onWorkerReady`, `onWorkerInit`, `onTaskFailure`, `onControlCommandCompleted`, etc.) | + When this matrix changes: 1. Update the **As of** date. diff --git a/.site/docs/core-concepts/canvas.md b/.site/docs/core-concepts/canvas.md index fd342c15..5caf079c 100644 --- a/.site/docs/core-concepts/canvas.md +++ b/.site/docs/core-concepts/canvas.md @@ -31,6 +31,19 @@ Groups fan out work and persist each branch in the result backend. ``` +## Batches + +Batches provide a first-class immutable submission API on top of durable group +state: + +- `canvas.submitBatch(signatures)` returns a stable `batchId` and task ids. +- `canvas.inspectBatch(batchId)` returns aggregate lifecycle status + (`pending`, `running`, `succeeded`, `failed`, `cancelled`, `partial`). + +```dart file=/../packages/stem/example/docs_snippets/lib/canvas_batch.dart#canvas-batch + +``` + ## Chords Chords combine a group with a callback. Once all body tasks succeed, the callback diff --git a/.site/docs/core-concepts/cli-control.md b/.site/docs/core-concepts/cli-control.md index 5704ae1f..155b0f51 100644 --- a/.site/docs/core-concepts/cli-control.md +++ b/.site/docs/core-concepts/cli-control.md @@ -20,7 +20,8 @@ manage workers, and operate schedules and routing. ## Remote control primer -The worker control commands (`ping`, `stats`, `inspect`, `revoke`, `shutdown`) +The worker control commands +(`ping`, `stats`, `inspect`, `revoke`, `shutdown`, `pause`, `resume`) publish control messages into the broker. Each command uses a request id and waits for replies on a per-request reply queue. @@ -35,8 +36,9 @@ Inspect vs control semantics: - **Inspect** (`ping`, `stats`, `inspect`) returns snapshots and does not mutate worker state. -- **Control** (`revoke`, `shutdown`) persists intent and asks workers to change - behavior (terminate tasks or shut down). +- **Control** (`revoke`, `shutdown`, `pause`, `resume`) persists intent and asks + workers to change behavior (terminate tasks, shut down, or pause queue + consumption). Payload highlights (as sent by the CLI): @@ -87,6 +89,14 @@ stem worker stats --worker worker-a ``` +```dart title="Pause queues" file=/../packages/stem/example/docs_snippets/lib/cli_control.dart#cli-control-worker-pause + +``` + +```dart title="Resume queues" file=/../packages/stem/example/docs_snippets/lib/cli_control.dart#cli-control-worker-resume + +``` + ```dart title="Apply schedules" file=/../packages/stem/example/docs_snippets/lib/cli_control.dart#cli-control-schedule-apply ``` @@ -154,6 +164,8 @@ stem worker ping stem worker stats stem worker revoke --task stem worker shutdown --mode warm +stem worker pause --queue default +stem worker resume --queue default ``` Expected output: @@ -223,6 +235,7 @@ Use this table to sanity-check which connection strings are required: | `stem worker status` | optional (follow) | optional (snapshot) | ❌ | ❌ | ❌ | | `stem worker revoke` | ✅ | optional | ❌ | optional | ❌ | | `stem worker shutdown` | ✅ | ❌ | ❌ | ❌ | ❌ | +| `stem worker pause/resume` | ✅ | ❌ | ❌ | optional | ❌ | | `stem schedule apply/list/dry-run` | ❌ | ❌ | ✅ | ❌ | ❌ | | `stem health` | ✅ | optional | ❌ | ❌ | ❌ | @@ -230,6 +243,8 @@ Notes: - The CLI resolves URLs from `STEM_BROKER_URL`, `STEM_RESULT_BACKEND_URL`, `STEM_SCHEDULE_STORE_URL`, and `STEM_REVOKE_STORE_URL`. +- `STEM_REVOKE_STORE_URL` supports `redis://`, `postgres://`, `sqlite:///`, + `file:///`, and `memory://` targets. - When a backend is “optional”, the command still runs but will skip that slice of data (for example, worker heartbeats without a result backend). - Schedule commands fall back to local schedule files when no schedule store diff --git a/.site/docs/core-concepts/index.md b/.site/docs/core-concepts/index.md index 59c71f37..a6379c07 100644 --- a/.site/docs/core-concepts/index.md +++ b/.site/docs/core-concepts/index.md @@ -17,7 +17,9 @@ behavior before touching production. - Worker lifecycle management, concurrency controls, and graceful shutdown. - Beat scheduler for interval/cron/solar/clocked jobs. - Canvas primitives (chains, groups, chords) for task composition. +- First-class batch submissions with durable aggregate status inspection. - Lifecycle signals for instrumentation and integrations. +- Queue-scoped custom events via `QueueEventsProducer`/`QueueEvents`. - Declarative routing across queues and broadcast channels. - Result backends and progress reporting via `TaskContext`. @@ -47,6 +49,7 @@ behavior before touching production. - **[Namespaces](./namespaces.md)** – Isolate environments and tenants. - **[Routing](./routing.md)** – Queue aliases, priorities, and broadcast channels. - **[Signals](./signals.md)** – Lifecycle hooks for instrumentation and integrations. +- **[Queue Events](./queue-events.md)** – Publish/listen to queue-scoped custom events. - **[Canvas Patterns](./canvas.md)** – Chains, groups, and chords for composing work. - **[Observability](./observability.md)** – Metrics, traces, logging, and lifecycle signals. - **[Persistence & Stores](./persistence.md)** – Result backends, schedule/lock stores, and revocation storage. diff --git a/.site/docs/core-concepts/persistence.md b/.site/docs/core-concepts/persistence.md index 9a7ea235..ce34cdc9 100644 --- a/.site/docs/core-concepts/persistence.md +++ b/.site/docs/core-concepts/persistence.md @@ -74,13 +74,18 @@ Switch to Postgres with `PostgresScheduleStore.connect` / `PostgresLockStore.con ## Revoke store -Store revocations in Redis or Postgres so workers can honour `stem worker revoke`: +Store revocations in Redis/Postgres/SQLite so workers can honour +`stem worker revoke`: ```bash export STEM_REVOKE_STORE_URL=postgres://postgres:postgres@localhost:5432/stem ``` -```dart file=/../packages/stem/example/docs_snippets/lib/persistence.dart#persistence-revoke-store +```dart title="Postgres revoke store" file=/../packages/stem/example/docs_snippets/lib/persistence.dart#persistence-revoke-store + +``` + +```dart title="SQLite revoke store" file=/../packages/stem/example/docs_snippets/lib/persistence.dart#persistence-revoke-store-sqlite ``` diff --git a/.site/docs/core-concepts/queue-events.md b/.site/docs/core-concepts/queue-events.md new file mode 100644 index 00000000..39da7f9a --- /dev/null +++ b/.site/docs/core-concepts/queue-events.md @@ -0,0 +1,50 @@ +--- +title: Queue Events +sidebar_label: Queue Events +sidebar_position: 9 +slug: /core-concepts/queue-events +--- + +Stem supports queue-scoped custom events similar to BullMQ `QueueEvents` and +"custom events" patterns. + +Use this when you need lightweight event streams for domain notifications +(`order.created`, `invoice.settled`) without creating task handlers. + +## API Surface + +- `QueueEventsProducer.emit(queue, eventName, payload, headers, meta)` +- `QueueEvents.start()` / `QueueEvents.close()` +- `QueueEvents.events` stream (all events for that queue) +- `QueueEvents.on(eventName)` stream (filtered by name) + +All events are delivered as `QueueCustomEvent`, which implements `StemEvent`. + +## Producer + Listener + +```dart title="lib/queue_events.dart" file=/../packages/stem/example/docs_snippets/lib/queue_events.dart#queue-events-producer-listener + +``` + +## Fan-out to Multiple Listeners + +Multiple listeners on the same queue receive each emitted event. + +```dart title="lib/queue_events.dart" file=/../packages/stem/example/docs_snippets/lib/queue_events.dart#queue-events-fanout + +``` + +## Semantics + +- Events are queue-scoped: listeners receive only events for their configured + queue. +- `on(eventName)` matches exact event names. +- `headers` and `meta` round-trip to listeners. +- Event names and queue names must be non-empty. +- Delivery follows the underlying broker's broadcast behavior for active + listeners (no historical replay API is built in to `QueueEvents`). + +## When to Use Queue Events vs Signals + +- Use [Signals](./signals.md) for runtime lifecycle hooks (task/worker/scheduler/control). +- Use Queue Events for application-domain events you publish and consume. diff --git a/.site/docs/core-concepts/rate-limiting.md b/.site/docs/core-concepts/rate-limiting.md index 3ba2f019..90ca3f60 100644 --- a/.site/docs/core-concepts/rate-limiting.md +++ b/.site/docs/core-concepts/rate-limiting.md @@ -9,6 +9,9 @@ Stem supports per-task rate limits via `TaskOptions.rateLimit` and a pluggable `RateLimiter` interface. This lets you throttle hot handlers with a shared Redis-backed limiter or custom driver. +Stem also supports group-scoped rate limits with `TaskOptions.groupRateLimit` +for shared quotas across multiple task types/tenants. + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -125,6 +128,8 @@ Run the `rate_limit_delay` example for a full demo: - `100/m` — 100 tokens per minute - `500/h` — 500 tokens per hour +`groupRateLimit` uses the same syntax. + ## How it works - The worker parses `rateLimit` for each task. @@ -134,6 +139,22 @@ Run the `rate_limit_delay` example for a full demo: worker’s retry strategy. - If granted, the task executes immediately. +## Group rate limiting + +Group rate limits share a limiter bucket across related tasks. + +- `groupRateLimit`: limiter policy for the shared group bucket +- `groupRateKey`: optional static key (if omitted, Stem resolves from header) +- `groupRateKeyHeader`: header used when `groupRateKey` is not set + (default: `tenant`) +- `groupRateLimiterFailureMode`: + - `failOpen`: continue execution if limiter backend fails + - `failClosed`: requeue/retry when limiter backend fails + +```dart title="lib/rate_limiting.dart" file=/../packages/stem/example/docs_snippets/lib/rate_limiting.dart#rate-limit-group-task-options + +``` + ## Redis-backed limiter example The `example/rate_limit_delay` demo ships a Redis fixed-window limiter. It: diff --git a/.site/docs/core-concepts/signals.md b/.site/docs/core-concepts/signals.md index 449161a3..9b056a3f 100644 --- a/.site/docs/core-concepts/signals.md +++ b/.site/docs/core-concepts/signals.md @@ -8,10 +8,15 @@ slug: /core-concepts/signals import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -Stem now exposes Celery-style lifecycle signals so instrumentation can react to -publish, worker, scheduler, and control-plane events without modifying the -runtime. All payloads live in `package:stem/src/signals/payloads.dart` and map -directly to Stem's task and worker data structures. +Stem exposes lifecycle signals so instrumentation can react to publish, worker, +scheduler, workflow, and control-plane events without modifying runtime code. + +All signal payloads implement `StemEvent` and dispatch through +`Signal`, giving every handler a shared event shape: + +- `eventName` +- `occurredAt` +- `attributes` ## Signal Catalog @@ -20,28 +25,23 @@ directly to Stem's task and worker data structures. | Publish | `beforeTaskPublish`, `afterTaskPublish` | `Envelope`, attempt metadata, task id | `before_task_publish`, `after_task_publish` | | Worker lifecycle | `workerInit`, `workerReady`, `workerStopping`, `workerShutdown`, `workerHeartbeat`, `workerChildInit`, `workerChildShutdown` | `WorkerInfo`, optional reason/timestamps | `worker_init`, `worker_ready`, `worker_shutting_down`, `worker_shutdown`, `heartbeat_sent`, `worker_process_init/shutdown` | | Task lifecycle | `taskReceived`, `taskPrerun`, `taskPostrun`, `taskRetry`, `taskSucceeded`, `taskFailed`, `taskRevoked` | `Envelope`, `WorkerInfo`, attempt, result/error context | `task_received`, `task_prerun`, `task_postrun`, `task_retry`, `task_success`, `task_failure`, `task_revoked` | +| Workflow lifecycle | `workflowRunStarted`, `workflowRunSuspended`, `workflowRunResumed`, `workflowRunCompleted`, `workflowRunFailed`, `workflowRunCancelled` | run id, workflow name, status, optional step metadata | n/a | | Scheduler | `scheduleEntryDue`, `scheduleEntryDispatched`, `scheduleEntryFailed` | `ScheduleEntry`, tick timestamp, drift, error stack | `beat_scheduler_ready`, `beat_schedule` | | Control plane | `controlCommandReceived`, `controlCommandCompleted` | `ControlCommandMessage`, reply status, payload/error maps | `control_command_sent`, `control_command_received` | -## Ordering & Semantics +## Ordering & Dispatch Semantics - `beforeTaskPublish` fires immediately before broker IO; `afterTaskPublish` runs once persistence succeeds. -- `taskReceived` is emitted after middleware but before the task context is - created. -- `taskPrerun` precedes handler execution; `taskPostrun` runs after completion, - regardless of outcome. Success and failure signals fire before `taskPostrun`. -- Worker lifecycle events follow `workerInit` → `workerReady` → optional - `workerStopping` → `workerShutdown`. Heartbeats (`workerHeartbeat`) include - namespace and queue metadata. -- Scheduler signals fire `scheduleEntryDue` → dispatch → - `scheduleEntryDispatched` or `scheduleEntryFailed`. -- Handler exceptions are caught, logged, and surfaced via - `StemSignals.signalDispatchFailed` while the dispatcher continues processing - remaining listeners unless `SignalContext.cancel()` is invoked. - -Handlers execute sequentially (respecting registration priority). `async` -callbacks are awaited so ordering remains deterministic. +- `taskReceived` is emitted before handler execution. +- `taskPrerun` precedes handler execution; `taskPostrun` runs after completion. +- Worker lifecycle follows `workerInit` -> `workerReady` -> optional + `workerStopping` -> `workerShutdown`. +- Scheduler signals emit due -> dispatched/failed. +- Dispatch is sequential and priority-aware; `async` callbacks are awaited. +- Listener errors are routed to `StemSignals.configure(onError: ...)` and do not + crash the worker loop. +- `SignalContext.cancel()` stops lower-priority listeners for the current emit. ## Configuration @@ -57,10 +57,7 @@ Environment knobs: - `STEM_SIGNALS_ENABLED=false` disables all signals. - `STEM_SIGNALS_DISABLED=worker-heartbeat,task-prerun` disables selected ones. -Workers automatically apply the configuration passed through -`ObservabilityConfig`, enabling cluster-wide rollouts without code changes. - -## Listening for signals +## Listening for Signals @@ -83,6 +80,13 @@ Workers automatically apply the configuration passed through ``` + + + +```dart title="lib/signals.dart" file=/../packages/stem/example/docs_snippets/lib/signals.dart#signals-worker-scoped + +``` + @@ -97,18 +101,37 @@ Workers automatically apply the configuration passed through ``` + + + +```dart title="lib/signals.dart" file=/../packages/stem/example/docs_snippets/lib/signals.dart#signals-stem-event + +``` + +Worker-scoped filtering is available on these convenience helpers: + +- `onWorkerInit`, `onWorkerReady`, `onWorkerStopping`, `onWorkerShutdown` +- `onWorkerHeartbeat`, `onWorkerChildInit`, `onWorkerChildShutdown` +- `onTaskReceived`, `onTaskPrerun`, `onTaskPostrun`, `onTaskSuccess`, + `onTaskFailure`, `onTaskRetry`, `onTaskRevoked` +- `onControlCommandReceived`, `onControlCommandCompleted` + +## Custom Queue Events + +Signals cover runtime lifecycle hooks. For application-domain events (BullMQ +`QueueEvents` style), use [`QueueEventsProducer` and `QueueEvents`](./queue-events.md). + ## Adapters & Middleware -- `StemSignalEmitter` builds payloads and emits signals; it powers Stem itself - and is available for custom middleware or broker integrations. +- `StemSignalEmitter` builds payloads and emits signals; Stem runtime uses this + same emitter internally. - `SignalMiddleware.coordinator()` forwards enqueue middleware to publish - signals, while `SignalMiddleware.worker()` emits `taskReceived`, - `taskPrerun`, and `taskFailed` from existing worker middleware chains. Success - and postrun events remain wired through the runtime so result payloads stay - accurate. + signals. +- `SignalMiddleware.worker()` emits receive/prerun/failure hooks from existing + worker middleware chains. ```dart title="lib/signals.dart" file=/../packages/stem/example/docs_snippets/lib/signals.dart#signals-middleware-producer @@ -118,44 +141,14 @@ Workers automatically apply the configuration passed through ``` -## Example (Docker Compose) - -`examples/signals_demo` spins up a Redis broker, producer, and worker. Running: - -```bash -docker compose up --build -``` - -streams every signal as structured JSON, showcasing retries, failures, worker -heartbeats, scheduler drift, and control commands. - -### Retry-Focused Walkthrough - -For a minimal reproduction of retry cadence, try `examples/retry_task`. The -worker connects to Redis with `blockTime=100ms`, `claimInterval=200ms`, and -`defaultVisibilityTimeout=2s`, and uses -`ExponentialJitterRetryStrategy(base: 200ms, max: 1s)` so each retry happens in -under a second. The producer enqueues a single task with `maxRetries=2`, and the -console prints every `task_retry`, `task_failed`, and `task_postrun` signal. -Experiment with the strategy or Redis timings to see how they shape retry -frequency. - ## Celery Comparison | Celery | Stem | Notes | | --- | --- | --- | -| `task_prerun` / `task_postrun` | `taskPrerun` / `taskPostrun` | Payload includes the Stem `TaskContext` for heartbeats and lease helpers. | -| `worker_ready` | `workerReady` | Provides queue/broadcast subscriptions for visibility. | +| `task_prerun` / `task_postrun` | `taskPrerun` / `taskPostrun` | Payload includes `TaskContext` and worker metadata. | +| `worker_ready` | `workerReady` | Worker-scoped filters available via `onWorkerReady(workerId: ...)`. | | `worker_process_init/shutdown` | `workerChildInit` / `workerChildShutdown` | Mirrors isolate pool spawn/recycle notifications. | -| `before_task_publish` | `beforeTaskPublish` | Fires before middleware or broker calls. | +| `before_task_publish` | `beforeTaskPublish` | Fires before broker writes. | | `beat_schedule` | `scheduleEntryDispatched` | Carries scheduled vs executed timestamps plus drift duration. | -Signals tied to Celery-specific pools remain out of scope; raise a proposal if -additional parity is required. - -## Performance & Testing - -Dispatch short-circuits when disabled, keeping the hot path cheap. Unit tests -exercise dispatcher priority, handler error handling, worker/scheduler events, -retry semantics, and the `SignalMiddleware` adapter, ensuring migrations from -Celery receive the expected hook coverage. +Signals tied to Celery-specific pools remain out of scope. diff --git a/.site/docs/workers/worker-control.md b/.site/docs/workers/worker-control.md index 13ec9e60..37e88621 100644 --- a/.site/docs/workers/worker-control.md +++ b/.site/docs/workers/worker-control.md @@ -26,6 +26,8 @@ revocation durability, and termination semantics for inline vs isolate handlers. | `stem worker stats` | Summarize inflight counts, queue depth, and metadata. | | `stem worker revoke` | Persist revocations and broadcast terminate/best-effort revokes. | | `stem worker shutdown` | Request warm/soft/hard shutdown via the control channel. | +| `stem worker pause` | Pause one or more queues on target workers. | +| `stem worker resume` | Resume paused queues on target workers. | | `stem worker status` | Stream heartbeats or snapshot the backend (existing command). | | `stem worker healthcheck` | Probe worker processes for readiness/liveness. | | `stem worker diagnose` | Run local diagnostics for pid/log/env configuration issues. | @@ -46,6 +48,12 @@ stem worker inspect --json # Revoke a task and request termination stem worker revoke --task 1761057... --terminate + +# Pause default queue on one worker +stem worker pause --worker worker-a --queue default + +# Resume that queue later +stem worker resume --worker worker-a --queue default ``` For a runnable lab that exercises ping/stats/revoke/shutdown against real @@ -133,11 +141,12 @@ CLI resolves the backing store in this order: 3. `STEM_BROKER_URL` Supported schemes: Redis (`redis://`, `rediss://`), Postgres (`postgres://`, -`postgresql://`), a newline-delimited file (`file:///path/to/revokes.stem` or -bare path), and in-memory (`memory://` – useful for tests). Workers hydrate the -revocation cache at startup, prune expired records, and apply new control -messages. The CLI writes through the store *before* broadcasting control -messages to guarantee durability precedes visibility. +`postgresql://`), SQLite (`sqlite:///path/to/stem.sqlite`), a +newline-delimited file (`file:///path/to/revokes.stem` or bare path), and +in-memory (`memory://` – useful for tests). Workers hydrate the revocation +cache at startup, prune expired records, and apply new control messages. The +CLI writes through the store *before* broadcasting control messages to +guarantee durability precedes visibility. ## Termination Semantics @@ -204,6 +213,16 @@ By default, workers install signal handlers that map `SIGTERM` to warm, with `WorkerLifecycleConfig(installSignalHandlers: false)` when embedding Stem inside a larger host that already owns signal routing. +## Queue Pause/Resume + +`stem worker pause` and `stem worker resume` target queue names (repeatable +`--queue`) and optionally specific workers (`--worker`). Paused queues are +requeued instead of executed until resumed. + +- Pause/resume state is persisted when a revoke store is configured. +- Without a revoke store, pause/resume still works for active workers but does + not survive worker restarts. + Lifecycle guards can also recycle isolates automatically: ```dart file=/../packages/stem/example/docs_snippets/lib/worker_control.dart#worker-control-lifecycle @@ -229,6 +248,12 @@ example, to use Postgres alongside the result backend: export STEM_REVOKE_STORE_URL=postgres://stem:secret@db:5432/stem ``` +For local single-node deployments, SQLite works as well: + +```bash +export STEM_REVOKE_STORE_URL=sqlite:///var/lib/stem/revokes.sqlite +``` + ## Additional Resources - `stem worker --help` – built-in CLI usage for each subcommand. diff --git a/.site/sidebars.ts b/.site/sidebars.ts index 2c8544c3..ad93cfab 100644 --- a/.site/sidebars.ts +++ b/.site/sidebars.ts @@ -53,6 +53,7 @@ const sidebars: SidebarsConfig = { "core-concepts/namespaces", "core-concepts/routing", "core-concepts/signals", + "core-concepts/queue-events", "core-concepts/canvas", "core-concepts/observability", "core-concepts/dashboard", diff --git a/packages/stem/example/docs_snippets/lib/canvas_batch.dart b/packages/stem/example/docs_snippets/lib/canvas_batch.dart new file mode 100644 index 00000000..908fcbd5 --- /dev/null +++ b/packages/stem/example/docs_snippets/lib/canvas_batch.dart @@ -0,0 +1,41 @@ +// Canvas batch examples for documentation. +// ignore_for_file: unused_local_variable, unused_import, dead_code, avoid_print + +import 'package:stem/stem.dart'; + +// #region canvas-batch +Future main() async { + final app = await StemApp.inMemory( + tasks: [ + FunctionTaskHandler( + name: 'batch.double', + entrypoint: (context, args) async { + final value = args['value'] as int? ?? 0; + return value * 2; + }, + ), + ], + workerConfig: const StemWorkerConfig( + consumerName: 'batch-worker', + concurrency: 1, + prefetchMultiplier: 1, + ), + ); + await app.start(); + + final submission = await app.canvas.submitBatch([ + task('batch.double', args: {'value': 1}), + task('batch.double', args: {'value': 2}), + task('batch.double', args: {'value': 3}), + ]); + + final status = await app.canvas.inspectBatch(submission.batchId); + print( + 'Batch ${submission.batchId} state=${status?.state} ' + 'completed=${status?.completed}/${status?.expected}', + ); + + await app.close(); +} + +// #endregion canvas-batch diff --git a/packages/stem/example/docs_snippets/lib/cli_control.dart b/packages/stem/example/docs_snippets/lib/cli_control.dart index c45848a1..3c344c19 100644 --- a/packages/stem/example/docs_snippets/lib/cli_control.dart +++ b/packages/stem/example/docs_snippets/lib/cli_control.dart @@ -35,6 +35,14 @@ const String workerRevokeCommand = 'stem worker revoke --task '; const String workerShutdownCommand = 'stem worker shutdown --mode warm'; // #endregion cli-control-worker-shutdown +// #region cli-control-worker-pause +const String workerPauseCommand = 'stem worker pause --queue default'; +// #endregion cli-control-worker-pause + +// #region cli-control-worker-resume +const String workerResumeCommand = 'stem worker resume --queue default'; +// #endregion cli-control-worker-resume + // #region cli-control-schedule-apply const String scheduleApplyCommand = 'stem schedule apply --file config/schedules.yaml --yes'; @@ -60,6 +68,8 @@ Future main() async { stdout.writeln(' $workerStatsCommand'); stdout.writeln(' $workerRevokeCommand'); stdout.writeln(' $workerShutdownCommand'); + stdout.writeln(' $workerPauseCommand'); + stdout.writeln(' $workerResumeCommand'); stdout.writeln('Schedules:'); stdout.writeln(' $scheduleApplyCommand'); diff --git a/packages/stem/example/docs_snippets/lib/persistence.dart b/packages/stem/example/docs_snippets/lib/persistence.dart index ebb6de0a..76438061 100644 --- a/packages/stem/example/docs_snippets/lib/persistence.dart +++ b/packages/stem/example/docs_snippets/lib/persistence.dart @@ -126,7 +126,7 @@ Future configureBeatStores() async { // #endregion persistence-beat-stores // #region persistence-revoke-store -Future configureRevokeStore() async { +Future configurePostgresRevokeStore() async { final broker = InMemoryBroker(); final backend = InMemoryResultBackend(); final revokeStore = await PostgresRevokeStore.connect( @@ -146,6 +146,28 @@ Future configureRevokeStore() async { } // #endregion persistence-revoke-store +// #region persistence-revoke-store-sqlite +Future configureSqliteRevokeStore() async { + final broker = InMemoryBroker(); + final backend = InMemoryResultBackend(); + final revokeStore = await SqliteRevokeStore.open( + File('stem_revoke.sqlite'), + namespace: 'stem', + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + revokeStore: revokeStore, + ); + + await worker.shutdown(); + await revokeStore.close(); + await backend.close(); + await broker.close(); +} +// #endregion persistence-revoke-store-sqlite + class GzipPayloadEncoder extends TaskPayloadEncoder { const GzipPayloadEncoder(); diff --git a/packages/stem/example/docs_snippets/lib/queue_events.dart b/packages/stem/example/docs_snippets/lib/queue_events.dart new file mode 100644 index 00000000..8b59f72c --- /dev/null +++ b/packages/stem/example/docs_snippets/lib/queue_events.dart @@ -0,0 +1,69 @@ +// Queue custom event examples for documentation. +// ignore_for_file: unused_local_variable, unused_import, dead_code, avoid_print + +import 'dart:async'; + +import 'package:stem/stem.dart'; + +// #region queue-events-producer-listener +Future queueEventsProducerListener(Broker broker) async { + final producer = QueueEventsProducer(broker: broker); + final listener = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-events', + ); + await listener.start(); + + final subscription = listener.on('order.created').listen((event) { + print('Order created: ${event.payload['orderId']}'); + print('Trace id: ${event.headers['x-trace-id']}'); + }); + + await producer.emit( + 'orders', + 'order.created', + payload: const {'orderId': 'ord-1001'}, + headers: const {'x-trace-id': 'trace-123'}, + meta: const {'tenant': 'acme'}, + ); + + await Future.delayed(const Duration(milliseconds: 200)); + await subscription.cancel(); + await listener.close(); +} +// #endregion queue-events-producer-listener + +// #region queue-events-fanout +Future queueEventsFanout(Broker broker) async { + final producer = QueueEventsProducer(broker: broker); + final listenerA = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-a', + ); + final listenerB = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-b', + ); + await listenerA.start(); + await listenerB.start(); + + final subscriptionA = listenerA.events.listen((event) { + print('A saw ${event.name}'); + }); + final subscriptionB = listenerB.events.listen((event) { + print('B saw ${event.name}'); + }); + + await producer.emit('orders', 'order.updated', payload: const {'id': 'o-1'}); + + await Future.delayed(const Duration(milliseconds: 200)); + await subscriptionA.cancel(); + await subscriptionB.cancel(); + await listenerA.close(); + await listenerB.close(); +} + +// #endregion queue-events-fanout diff --git a/packages/stem/example/docs_snippets/lib/rate_limiting.dart b/packages/stem/example/docs_snippets/lib/rate_limiting.dart index 6eb64962..701ec7c3 100644 --- a/packages/stem/example/docs_snippets/lib/rate_limiting.dart +++ b/packages/stem/example/docs_snippets/lib/rate_limiting.dart @@ -70,6 +70,27 @@ class RateLimitedTask extends TaskHandler { // #endregion rate-limit-task-options // #endregion rate-limit-task +// #region rate-limit-group-task-options +class GroupRateLimitedTask extends TaskHandler { + @override + String get name => 'demo.groupRateLimited'; + + @override + TaskOptions get options => const TaskOptions( + groupRateLimit: '20/m', + groupRateKeyHeader: 'tenant', + groupRateLimiterFailureMode: RateLimiterFailureMode.failClosed, + maxRetries: 5, + ); + + @override + Future call(TaskContext context, Map args) async { + final tenant = args['tenant'] as String? ?? 'global'; + print('Handled group-rate-limited task for $tenant'); + } +} +// #endregion rate-limit-group-task-options + // #region rate-limit-producer Future enqueueRateLimited(Stem stem) async { return stem.enqueue( diff --git a/packages/stem/example/docs_snippets/lib/signals.dart b/packages/stem/example/docs_snippets/lib/signals.dart index cf94e9ca..97f7667f 100644 --- a/packages/stem/example/docs_snippets/lib/signals.dart +++ b/packages/stem/example/docs_snippets/lib/signals.dart @@ -44,12 +44,50 @@ List registerPublishSignals() { // #region signals-worker-listeners SignalSubscription registerWorkerSignals() { - return StemSignals.workerReady.connect((payload, _) { + return StemSignals.onWorkerReady((payload, _) { print('Worker ready: ${payload.worker.id}'); - }); + }, workerId: 'signals-worker'); } // #endregion signals-worker-listeners +// #region signals-worker-scoped +List registerWorkerScopedSignals() { + return [ + StemSignals.onTaskFailure( + (payload, _) { + print( + 'Task failed on worker ${payload.worker.id}: ${payload.taskName}', + ); + }, + taskName: 'signals.demo', + workerId: 'signals-worker', + ), + StemSignals.onControlCommandCompleted( + (payload, _) { + print( + 'Control ${payload.command.type} -> ${payload.status} on ${payload.worker.id}', + ); + }, + workerId: 'signals-worker', + commandType: 'ping', + ), + ]; +} +// #endregion signals-worker-scoped + +// #region signals-stem-event +SignalSubscription registerStemEventView() { + return StemSignals.onTaskSuccess((payload, context) { + final event = context.event; + if (event != null) { + print( + 'Event ${event.eventName} at ${event.occurredAt.toIso8601String()}', + ); + } + }); +} +// #endregion signals-stem-event + // #region signals-scheduler-listeners SignalSubscription registerSchedulerSignals() { return StemSignals.scheduleEntryDispatched.connect((payload, _) { @@ -60,7 +98,7 @@ SignalSubscription registerSchedulerSignals() { // #region signals-control-listeners SignalSubscription registerControlSignals() { - return StemSignals.controlCommandCompleted.connect((payload, _) { + return StemSignals.onControlCommandCompleted((payload, _) { print('Control command: ${payload.command.type}'); }); } @@ -93,6 +131,8 @@ Future main() async { ...registerPublishSignals(), ...registerTaskSignals(), registerWorkerSignals(), + ...registerWorkerScopedSignals(), + registerStemEventView(), registerSchedulerSignals(), registerControlSignals(), ]; diff --git a/packages/stem/lib/src/canvas/canvas.dart b/packages/stem/lib/src/canvas/canvas.dart index 83763fd4..52acf161 100644 --- a/packages/stem/lib/src/canvas/canvas.dart +++ b/packages/stem/lib/src/canvas/canvas.dart @@ -151,6 +151,99 @@ class ChordResult { final List values; } +/// Lifecycle states for first-class batch submissions. +enum BatchLifecycleState { + /// Batch has no completed children yet. + pending, + + /// Batch has some completed children but is not terminal. + running, + + /// All children succeeded. + succeeded, + + /// All children failed. + failed, + + /// All children were cancelled. + cancelled, + + /// Batch completed with mixed terminal outcomes. + partial, +} + +/// Handle returned when a batch is submitted. +class BatchSubmission { + /// Creates a batch submission snapshot. + const BatchSubmission({ + required this.batchId, + required this.taskIds, + }); + + /// Stable batch identifier. + final String batchId; + + /// Child task ids dispatched for the batch. + final List taskIds; +} + +/// Durable batch status projection derived from group metadata/results. +class BatchStatus { + /// Creates a durable batch status snapshot. + const BatchStatus({ + required this.batchId, + required this.state, + required this.expected, + required this.completed, + required this.succeededCount, + required this.failedCount, + required this.cancelledCount, + required this.failedTaskIds, + required this.cancelledTaskIds, + required this.meta, + }); + + /// Stable batch identifier. + final String batchId; + + /// Current batch lifecycle state. + final BatchLifecycleState state; + + /// Total children expected for the batch. + final int expected; + + /// Number of children currently in terminal states. + final int completed; + + /// Number of children that succeeded. + final int succeededCount; + + /// Number of children that failed. + final int failedCount; + + /// Number of children that were cancelled. + final int cancelledCount; + + /// Child task ids that failed. + final List failedTaskIds; + + /// Child task ids that were cancelled. + final List cancelledTaskIds; + + /// Persisted batch metadata. + final Map meta; + + /// Number of children not yet complete. + int get pendingCount => (expected - completed).clamp(0, expected); + + /// Whether the batch is in a terminal state. + bool get isTerminal => + state == BatchLifecycleState.succeeded || + state == BatchLifecycleState.failed || + state == BatchLifecycleState.cancelled || + state == BatchLifecycleState.partial; +} + /// A high-level API for composing and dispatching tasks. /// /// [Canvas] publishes [Envelope]s to a [Broker] and records status in a @@ -318,6 +411,45 @@ class Canvas { ); } + /// Submits an immutable batch of task signatures under one batch id. + /// + /// Batches are backed by durable group records and can be inspected via + /// [inspectBatch]. The batch is immutable once submitted. + Future submitBatch( + List> signatures, { + String? batchId, + Duration? ttl, + }) async { + if (signatures.isEmpty) { + throw ArgumentError('Batch must include at least one task'); + } + final id = batchId ?? _generateId('batch'); + final createdAt = DateTime.now().toUtc().toIso8601String(); + await backend.initGroup( + GroupDescriptor( + id: id, + expected: signatures.length, + ttl: ttl, + meta: { + 'stem.batch': true, + 'stem.batch.createdAt': createdAt, + 'stem.batch.taskCount': signatures.length, + }, + ), + ); + final dispatch = await group(signatures, groupId: id); + final taskIds = List.from(dispatch.taskIds); + await dispatch.dispose(); + return BatchSubmission(batchId: id, taskIds: taskIds); + } + + /// Reads durable lifecycle status for a submitted [batchId]. + Future inspectBatch(String batchId) async { + final status = await backend.getGroup(batchId); + if (status == null) return null; + return _buildBatchStatus(status); + } + /// Runs tasks sequentially, passing each result to the next. /// /// Each task is published only after the previous task succeeds. The result @@ -550,6 +682,50 @@ class Canvas { } } + BatchStatus _buildBatchStatus(GroupStatus status) { + final entries = status.results.entries.toList(growable: false); + final succeeded = entries + .where((entry) => entry.value.state == TaskState.succeeded) + .length; + final failedEntries = entries + .where((entry) => entry.value.state == TaskState.failed) + .toList(growable: false); + final cancelledEntries = entries + .where((entry) => entry.value.state == TaskState.cancelled) + .toList(growable: false); + final failed = failedEntries.length; + final cancelled = cancelledEntries.length; + final completed = entries.length; + + BatchLifecycleState state; + if (completed == 0) { + state = BatchLifecycleState.pending; + } else if (completed < status.expected) { + state = BatchLifecycleState.running; + } else if (failed == 0 && cancelled == 0) { + state = BatchLifecycleState.succeeded; + } else if (succeeded == 0 && cancelled == 0) { + state = BatchLifecycleState.failed; + } else if (succeeded == 0 && failed == 0) { + state = BatchLifecycleState.cancelled; + } else { + state = BatchLifecycleState.partial; + } + + return BatchStatus( + batchId: status.id, + state: state, + expected: status.expected, + completed: completed, + succeededCount: succeeded, + failedCount: failed, + cancelledCount: cancelled, + failedTaskIds: failedEntries.map((entry) => entry.key).toList(), + cancelledTaskIds: cancelledEntries.map((entry) => entry.key).toList(), + meta: status.meta, + ); + } + (Envelope, TaskPayloadEncoder) _prepareEnvelope(Envelope envelope) { final handler = registry.resolve(envelope.name); final argsEncoder = _resolveArgsEncoder(handler); diff --git a/packages/stem/lib/src/core/contracts.dart b/packages/stem/lib/src/core/contracts.dart index 7bb75925..c74ae271 100644 --- a/packages/stem/lib/src/core/contracts.dart +++ b/packages/stem/lib/src/core/contracts.dart @@ -853,6 +853,10 @@ class TaskOptions { this.softTimeLimit, this.hardTimeLimit, this.rateLimit, + this.groupRateLimit, + this.groupRateKey, + this.groupRateKeyHeader = 'tenant', + this.groupRateLimiterFailureMode = RateLimiterFailureMode.failOpen, this.unique = false, this.uniqueFor, this.priority = 0, @@ -872,12 +876,21 @@ class TaskOptions { retryValue.cast(), ); } + final failureMode = + _parseFailureMode( + json['groupRateLimiterFailureMode'], + ) ?? + RateLimiterFailureMode.failOpen; return TaskOptions( queue: json['queue'] as String? ?? 'default', maxRetries: (json['maxRetries'] as num?)?.toInt() ?? 0, softTimeLimit: _durationFromJson(json['softTimeLimitMs']), hardTimeLimit: _durationFromJson(json['hardTimeLimitMs']), rateLimit: json['rateLimit'] as String?, + groupRateLimit: json['groupRateLimit'] as String?, + groupRateKey: json['groupRateKey'] as String?, + groupRateKeyHeader: json['groupRateKeyHeader'] as String? ?? 'tenant', + groupRateLimiterFailureMode: failureMode, unique: json['unique'] as bool? ?? false, uniqueFor: _durationFromJson(json['uniqueForMs']), priority: (json['priority'] as num?)?.toInt() ?? 0, @@ -902,6 +915,18 @@ class TaskOptions { /// The rate limit for tasks with these options. final String? rateLimit; + /// Group-scoped rate limit shared by tasks that resolve to the same group key. + final String? groupRateLimit; + + /// Optional static group key used for group-scoped rate limiting. + final String? groupRateKey; + + /// Header key used to resolve group identity when [groupRateKey] is not set. + final String groupRateKeyHeader; + + /// Behavior to apply when group limiter calls fail. + final RateLimiterFailureMode groupRateLimiterFailureMode; + /// Whether tasks with these options should be unique. final bool unique; @@ -927,6 +952,10 @@ class TaskOptions { Duration? softTimeLimit, Duration? hardTimeLimit, String? rateLimit, + String? groupRateLimit, + String? groupRateKey, + String? groupRateKeyHeader, + RateLimiterFailureMode? groupRateLimiterFailureMode, bool? unique, Duration? uniqueFor, int? priority, @@ -940,6 +969,11 @@ class TaskOptions { softTimeLimit: softTimeLimit ?? this.softTimeLimit, hardTimeLimit: hardTimeLimit ?? this.hardTimeLimit, rateLimit: rateLimit ?? this.rateLimit, + groupRateLimit: groupRateLimit ?? this.groupRateLimit, + groupRateKey: groupRateKey ?? this.groupRateKey, + groupRateKeyHeader: groupRateKeyHeader ?? this.groupRateKeyHeader, + groupRateLimiterFailureMode: + groupRateLimiterFailureMode ?? this.groupRateLimiterFailureMode, unique: unique ?? this.unique, uniqueFor: uniqueFor ?? this.uniqueFor, priority: priority ?? this.priority, @@ -956,6 +990,10 @@ class TaskOptions { 'softTimeLimitMs': softTimeLimit?.inMilliseconds, 'hardTimeLimitMs': hardTimeLimit?.inMilliseconds, 'rateLimit': rateLimit, + 'groupRateLimit': groupRateLimit, + 'groupRateKey': groupRateKey, + 'groupRateKeyHeader': groupRateKeyHeader, + 'groupRateLimiterFailureMode': groupRateLimiterFailureMode.name, 'unique': unique, 'uniqueForMs': uniqueFor?.inMilliseconds, 'priority': priority, @@ -985,6 +1023,17 @@ class TaskOptions { } return null; } + + static RateLimiterFailureMode? _parseFailureMode(Object? value) { + final raw = value?.toString().trim().toLowerCase(); + if (raw == null || raw.isEmpty) return null; + for (final mode in RateLimiterFailureMode.values) { + if (mode.name.toLowerCase() == raw) { + return mode; + } + } + return null; + } } /// Retry policy configuration for tasks and publish attempts. @@ -2010,6 +2059,15 @@ abstract class RateLimiter { }); } +/// Defines behavior when a limiter backend call fails. +enum RateLimiterFailureMode { + /// Continue executing even when limiter calls fail. + failOpen, + + /// Block execution and retry later when limiter calls fail. + failClosed, +} + /// Result of attempting to acquire tokens from the rate limiter. class RateLimitDecision { /// Creates a rate limit decision outcome. diff --git a/packages/stem/lib/src/core/queue_events.dart b/packages/stem/lib/src/core/queue_events.dart new file mode 100644 index 00000000..f5004de6 --- /dev/null +++ b/packages/stem/lib/src/core/queue_events.dart @@ -0,0 +1,269 @@ +import 'dart:async'; + +import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/envelope.dart'; +import 'package:stem/src/core/stem_event.dart'; + +const String _queueEventEnvelopeName = '__stem.queue.event__'; +const String _queueEventChannelPrefix = 'stem:events'; + +String _queueEventChannel(String queue) => '$_queueEventChannelPrefix:$queue'; + +/// Represents a custom queue event emitted through [QueueEventsProducer]. +class QueueCustomEvent implements StemEvent { + /// Creates a queue custom event. + const QueueCustomEvent({ + required this.id, + required this.queue, + required this.name, + required this.payload, + required this.emittedAt, + this.headers = const {}, + this.meta = const {}, + }); + + /// Message identifier used by the underlying broker envelope. + final String id; + + /// Queue scope for this event. + final String queue; + + /// Custom event name. + final String name; + + /// Event payload. + final Map payload; + + /// Timestamp when the event was emitted. + final DateTime emittedAt; + + /// Event headers. + final Map headers; + + /// Additional metadata supplied by the publisher. + final Map meta; + + @override + String get eventName => name; + + @override + DateTime get occurredAt => emittedAt; + + @override + Map get attributes => { + 'id': id, + 'queue': queue, + 'payload': payload, + 'headers': headers, + 'meta': meta, + }; + + /// Converts the event to a JSON-compatible map. + Map toJson() => { + 'id': id, + 'queue': queue, + 'name': name, + 'payload': payload, + 'emittedAt': emittedAt.toIso8601String(), + 'headers': headers, + 'meta': meta, + }; +} + +/// Emits queue-scoped custom events. +class QueueEventsProducer { + /// Creates a queue event producer bound to a [broker]. + const QueueEventsProducer({required this.broker}); + + /// Broker used for event delivery. + final Broker broker; + + /// Emits [eventName] on [queue] and returns the event id. + Future emit( + String queue, + String eventName, { + Map payload = const {}, + Map headers = const {}, + Map meta = const {}, + }) async { + final normalizedQueue = queue.trim(); + if (normalizedQueue.isEmpty) { + throw ArgumentError.value(queue, 'queue', 'Queue name must not be empty'); + } + final normalizedEventName = eventName.trim(); + if (normalizedEventName.isEmpty) { + throw ArgumentError.value( + eventName, + 'eventName', + 'Event name must not be empty', + ); + } + + final emittedAt = DateTime.now().toUtc(); + final envelope = Envelope( + name: _queueEventEnvelopeName, + args: { + 'eventName': normalizedEventName, + 'payload': payload, + 'queue': normalizedQueue, + 'emittedAt': emittedAt.toIso8601String(), + }, + headers: headers, + queue: normalizedQueue, + meta: meta, + ); + await broker.publish( + envelope, + routing: RoutingInfo.broadcast( + channel: _queueEventChannel(normalizedQueue), + ), + ); + return envelope.id; + } +} + +/// Listens for queue-scoped custom events emitted by [QueueEventsProducer]. +class QueueEvents { + /// Creates a queue event listener for [queue]. + QueueEvents({ + required this.broker, + required String queue, + String? consumerName, + this.prefetch = 10, + }) : queue = queue.trim(), + consumerName = + consumerName ?? + 'stem-queue-events-${generateEnvelopeId().replaceAll('-', '')}'; + + /// Broker used for event consumption. + final Broker broker; + + /// Queue scope for this listener. + final String queue; + + /// Consumer identity used by broker adapters. + final String consumerName; + + /// Prefetch size used by broker consumption. + final int prefetch; + + StreamSubscription? _subscription; + final StreamController _events = + StreamController.broadcast(); + bool _started = false; + bool _closed = false; + + /// Stream of received custom events. + Stream get events => _events.stream; + + /// Returns a filtered stream for [eventName]. + Stream on(String eventName) { + final normalized = eventName.trim(); + if (normalized.isEmpty) { + throw ArgumentError.value( + eventName, + 'eventName', + 'Event name must not be empty', + ); + } + return events.where((event) => event.name == normalized); + } + + /// Starts consuming queue events. + Future start() async { + if (_closed) { + throw StateError('QueueEvents is already closed.'); + } + if (_started) { + return; + } + if (queue.isEmpty) { + throw ArgumentError.value( + queue, + 'queue', + 'Queue name must not be empty', + ); + } + + _started = true; + _subscription = broker + .consume( + RoutingSubscription( + queues: const [], + broadcastChannels: [ + _queueEventChannel(queue), + ], + ), + prefetch: prefetch, + consumerName: consumerName, + ) + .listen( + _onDelivery, + onError: (Object error, StackTrace stackTrace) { + _events.addError(error, stackTrace); + }, + ); + } + + /// Stops consuming events and closes the stream. + Future close() async { + if (_closed) { + return; + } + _closed = true; + await _subscription?.cancel(); + _subscription = null; + await _events.close(); + } + + Future _onDelivery(Delivery delivery) async { + try { + final event = _eventFromEnvelope(delivery.envelope); + if (event != null && event.queue == queue) { + _events.add(event); + } + } on Object catch (error, stackTrace) { + _events.addError(error, stackTrace); + } finally { + try { + await broker.ack(delivery); + } on Object { + // Best-effort acknowledgement to avoid poisoning the stream. + } + } + } +} + +QueueCustomEvent? _eventFromEnvelope(Envelope envelope) { + if (envelope.name != _queueEventEnvelopeName) { + return null; + } + final args = envelope.args; + final eventName = args['eventName']?.toString(); + if (eventName == null || eventName.trim().isEmpty) { + throw const FormatException('Queue event is missing "eventName".'); + } + final queue = (args['queue']?.toString() ?? envelope.queue).trim(); + if (queue.isEmpty) { + throw const FormatException('Queue event is missing "queue".'); + } + final emittedAtRaw = args['emittedAt']?.toString(); + final emittedAt = emittedAtRaw == null + ? envelope.enqueuedAt.toUtc() + : DateTime.parse(emittedAtRaw).toUtc(); + + final rawPayload = args['payload']; + final payload = rawPayload is Map + ? rawPayload.cast() + : const {}; + + return QueueCustomEvent( + id: envelope.id, + queue: queue, + name: eventName.trim(), + payload: payload, + emittedAt: emittedAt, + headers: envelope.headers, + meta: envelope.meta, + ); +} diff --git a/packages/stem/lib/src/core/stem_event.dart b/packages/stem/lib/src/core/stem_event.dart new file mode 100644 index 00000000..0382377c --- /dev/null +++ b/packages/stem/lib/src/core/stem_event.dart @@ -0,0 +1,11 @@ +/// Shared base contract for events emitted across Stem components. +abstract interface class StemEvent { + /// Canonical event name. + String get eventName; + + /// Timestamp when the event occurred. + DateTime get occurredAt; + + /// Event attributes for diagnostics/observability. + Map get attributes; +} diff --git a/packages/stem/lib/src/signals/emitter.dart b/packages/stem/lib/src/signals/emitter.dart index 10a6f900..e0045e25 100644 --- a/packages/stem/lib/src/signals/emitter.dart +++ b/packages/stem/lib/src/signals/emitter.dart @@ -160,7 +160,11 @@ class StemSignalEmitter { /// Emits the worker-init signal. Future workerInit(WorkerInfo worker, {String? reason, String? sender}) { return StemSignals.workerInit.emit( - WorkerLifecyclePayload(worker: worker, reason: reason), + WorkerLifecyclePayload( + worker: worker, + reason: reason, + signalName: StemSignals.workerInitName, + ), sender: _senderOverride(sender), ); } @@ -172,7 +176,11 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workerReady.emit( - WorkerLifecyclePayload(worker: worker, reason: reason), + WorkerLifecyclePayload( + worker: worker, + reason: reason, + signalName: StemSignals.workerReadyName, + ), sender: _senderOverride(sender), ); } @@ -184,7 +192,11 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workerStopping.emit( - WorkerLifecyclePayload(worker: worker, reason: reason), + WorkerLifecyclePayload( + worker: worker, + reason: reason, + signalName: StemSignals.workerStoppingName, + ), sender: _senderOverride(sender), ); } @@ -196,7 +208,11 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workerShutdown.emit( - WorkerLifecyclePayload(worker: worker, reason: reason), + WorkerLifecyclePayload( + worker: worker, + reason: reason, + signalName: StemSignals.workerShutdownName, + ), sender: _senderOverride(sender), ); } @@ -220,9 +236,13 @@ class StemSignalEmitter { required bool initializing, String? sender, }) { + final signalName = initializing + ? StemSignals.workerChildInitName + : StemSignals.workerChildShutdownName; final payload = WorkerChildLifecyclePayload( worker: worker, isolateId: isolateId, + signalName: signalName, ); final effectiveSender = _senderOverride(sender); if (initializing) { @@ -323,7 +343,7 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workflowRunStarted.emit( - payload, + payload.withSignalName(StemSignals.workflowRunStartedName), sender: _senderOverride(sender), ); } @@ -334,7 +354,7 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workflowRunSuspended.emit( - payload, + payload.withSignalName(StemSignals.workflowRunSuspendedName), sender: _senderOverride(sender), ); } @@ -345,7 +365,7 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workflowRunResumed.emit( - payload, + payload.withSignalName(StemSignals.workflowRunResumedName), sender: _senderOverride(sender), ); } @@ -356,7 +376,7 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workflowRunCompleted.emit( - payload, + payload.withSignalName(StemSignals.workflowRunCompletedName), sender: _senderOverride(sender), ); } @@ -364,7 +384,7 @@ class StemSignalEmitter { /// Emits the workflow-run-failed signal. Future workflowRunFailed(WorkflowRunPayload payload, {String? sender}) { return StemSignals.workflowRunFailed.emit( - payload, + payload.withSignalName(StemSignals.workflowRunFailedName), sender: _senderOverride(sender), ); } @@ -375,7 +395,7 @@ class StemSignalEmitter { String? sender, }) { return StemSignals.workflowRunCancelled.emit( - payload, + payload.withSignalName(StemSignals.workflowRunCancelledName), sender: _senderOverride(sender), ); } diff --git a/packages/stem/lib/src/signals/payloads.dart b/packages/stem/lib/src/signals/payloads.dart index d0b4e3f2..016d7ce8 100644 --- a/packages/stem/lib/src/signals/payloads.dart +++ b/packages/stem/lib/src/signals/payloads.dart @@ -1,6 +1,7 @@ import 'package:stem/src/control/control_messages.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; +import 'package:stem/src/core/stem_event.dart'; /// Status of a workflow run emitted via signals. enum WorkflowRunStatus { @@ -40,7 +41,7 @@ class WorkerInfo { } /// Payload emitted before a task is published to the broker. -class BeforeTaskPublishPayload { +class BeforeTaskPublishPayload implements StemEvent { /// Creates a new [BeforeTaskPublishPayload] instance. const BeforeTaskPublishPayload({ required this.envelope, @@ -52,10 +53,24 @@ class BeforeTaskPublishPayload { /// The attempt number for this task. final int attempt; + + @override + String get eventName => 'before-task-publish'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': envelope.id, + 'taskName': envelope.name, + 'queue': envelope.queue, + 'attempt': attempt, + }; } /// Payload emitted after a task has been published to the broker. -class AfterTaskPublishPayload { +class AfterTaskPublishPayload implements StemEvent { /// Creates a new [AfterTaskPublishPayload] instance. const AfterTaskPublishPayload({ required this.envelope, @@ -71,10 +86,24 @@ class AfterTaskPublishPayload { /// The unique identifier for the task. final String taskId; + + @override + String get eventName => 'after-task-publish'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': envelope.name, + 'queue': envelope.queue, + 'attempt': attempt, + }; } /// Payload emitted when a task is received by a worker. -class TaskReceivedPayload { +class TaskReceivedPayload implements StemEvent { /// Creates a new [TaskReceivedPayload] instance. const TaskReceivedPayload({required this.envelope, required this.worker}); @@ -89,10 +118,24 @@ class TaskReceivedPayload { /// The name of the task. String get taskName => envelope.name; + + @override + String get eventName => 'task-received'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': taskName, + 'queue': envelope.queue, + 'workerId': worker.id, + }; } /// Payload emitted before a task begins execution. -class TaskPrerunPayload { +class TaskPrerunPayload implements StemEvent { /// Creates a new [TaskPrerunPayload] instance. const TaskPrerunPayload({ required this.envelope, @@ -117,10 +160,25 @@ class TaskPrerunPayload { /// The attempt number for this task execution. int get attempt => envelope.attempt; + + @override + String get eventName => 'task-prerun'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': taskName, + 'queue': envelope.queue, + 'attempt': attempt, + 'workerId': worker.id, + }; } /// Payload emitted after a task finishes execution. -class TaskPostrunPayload { +class TaskPostrunPayload implements StemEvent { /// Creates a new [TaskPostrunPayload] instance. const TaskPostrunPayload({ required this.envelope, @@ -153,10 +211,26 @@ class TaskPostrunPayload { /// The attempt number for this task execution. int get attempt => envelope.attempt; + + @override + String get eventName => 'task-postrun'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': taskName, + 'queue': envelope.queue, + 'attempt': attempt, + 'workerId': worker.id, + 'state': state.name, + }; } /// Payload emitted when a task is scheduled for retry. -class TaskRetryPayload { +class TaskRetryPayload implements StemEvent { /// Creates a new [TaskRetryPayload] instance. const TaskRetryPayload({ required this.envelope, @@ -185,10 +259,27 @@ class TaskRetryPayload { /// The attempt number for this task execution. int get attempt => envelope.attempt; + + @override + String get eventName => 'task-retry'; + + @override + DateTime get occurredAt => nextRetryAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': taskName, + 'queue': envelope.queue, + 'attempt': attempt, + 'workerId': worker.id, + 'reason': reason.toString(), + 'nextRetryAt': nextRetryAt.toUtc().toIso8601String(), + }; } /// Payload emitted when a task completes successfully. -class TaskSuccessPayload { +class TaskSuccessPayload implements StemEvent { /// Creates a new [TaskSuccessPayload] instance. const TaskSuccessPayload({ required this.envelope, @@ -213,10 +304,25 @@ class TaskSuccessPayload { /// The attempt number for this task execution. int get attempt => envelope.attempt; + + @override + String get eventName => 'task-succeeded'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': taskName, + 'queue': envelope.queue, + 'attempt': attempt, + 'workerId': worker.id, + }; } /// Payload emitted when a task fails. -class TaskFailurePayload { +class TaskFailurePayload implements StemEvent { /// Creates a new [TaskFailurePayload] instance. const TaskFailurePayload({ required this.envelope, @@ -245,10 +351,27 @@ class TaskFailurePayload { /// The attempt number for this task execution. int get attempt => envelope.attempt; + + @override + String get eventName => 'task-failed'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': taskId, + 'taskName': taskName, + 'queue': envelope.queue, + 'attempt': attempt, + 'workerId': worker.id, + 'error': error.toString(), + if (stackTrace != null) 'stackTrace': stackTrace.toString(), + }; } /// Payload emitted when a task is revoked. -class TaskRevokedPayload { +class TaskRevokedPayload implements StemEvent { /// Creates a new [TaskRevokedPayload] instance. const TaskRevokedPayload({ required this.envelope, @@ -264,22 +387,61 @@ class TaskRevokedPayload { /// The reason for revoking the task. final String reason; + + @override + String get eventName => 'task-revoked'; + + @override + DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + + @override + Map get attributes => { + 'taskId': envelope.id, + 'taskName': envelope.name, + 'queue': envelope.queue, + 'workerId': worker.id, + 'reason': reason, + }; } /// Payload emitted for worker lifecycle events (start, stop, etc.). -class WorkerLifecyclePayload { +class WorkerLifecyclePayload implements StemEvent { /// Creates a new [WorkerLifecyclePayload] instance. - const WorkerLifecyclePayload({required this.worker, this.reason}); + WorkerLifecyclePayload({ + required this.worker, + this.reason, + this.signalName = 'worker-lifecycle', + DateTime? timestamp, + }) : _occurredAt = (timestamp ?? DateTime.now()).toUtc(); /// The worker involved in the lifecycle event. final WorkerInfo worker; /// Optional reason for the lifecycle event (e.g., shutdown reason). final String? reason; + + /// Canonical signal name for this lifecycle event. + final String signalName; + + final DateTime _occurredAt; + + @override + String get eventName => signalName; + + @override + DateTime get occurredAt => _occurredAt; + + @override + Map get attributes => { + 'workerId': worker.id, + 'queues': worker.queues, + 'broadcasts': worker.broadcasts, + if (reason != null) 'reason': reason, + }; } /// Payload emitted when a worker sends a heartbeat. -class WorkerHeartbeatPayload { +class WorkerHeartbeatPayload implements StemEvent { /// Creates a new [WorkerHeartbeatPayload] instance. const WorkerHeartbeatPayload({required this.worker, required this.timestamp}); @@ -288,33 +450,68 @@ class WorkerHeartbeatPayload { /// The timestamp when the heartbeat was sent. final DateTime timestamp; + + @override + String get eventName => 'worker-heartbeat'; + + @override + DateTime get occurredAt => timestamp.toUtc(); + + @override + Map get attributes => { + 'workerId': worker.id, + 'queues': worker.queues, + 'broadcasts': worker.broadcasts, + 'timestamp': timestamp.toUtc().toIso8601String(), + }; } /// Payload emitted for worker child isolate lifecycle events. -class WorkerChildLifecyclePayload { +class WorkerChildLifecyclePayload implements StemEvent { /// Creates a new [WorkerChildLifecyclePayload] instance. - const WorkerChildLifecyclePayload({ + WorkerChildLifecyclePayload({ required this.worker, required this.isolateId, - }); + this.signalName = 'worker-child-lifecycle', + DateTime? timestamp, + }) : _occurredAt = (timestamp ?? DateTime.now()).toUtc(); /// The parent worker managing the child isolate. final WorkerInfo worker; /// The unique identifier for the child isolate. final int isolateId; + + /// Canonical signal name for this child lifecycle event. + final String signalName; + + final DateTime _occurredAt; + + @override + String get eventName => signalName; + + @override + DateTime get occurredAt => _occurredAt; + + @override + Map get attributes => { + 'workerId': worker.id, + 'isolateId': isolateId, + }; } /// Payload emitted for workflow run events. -class WorkflowRunPayload { +class WorkflowRunPayload implements StemEvent { /// Creates a new [WorkflowRunPayload] instance. - const WorkflowRunPayload({ + WorkflowRunPayload({ required this.runId, required this.workflow, required this.status, this.step, this.metadata = const {}, - }); + this.signalName, + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? DateTime.now()).toUtc(); /// The unique identifier for the workflow run. final String runId; @@ -330,10 +527,41 @@ class WorkflowRunPayload { /// Additional metadata associated with the workflow run. final Map metadata; + + /// Optional canonical signal name when this payload is emitted. + final String? signalName; + + final DateTime _occurredAt; + + @override + String get eventName => signalName ?? 'workflow-run-${status.name}'; + + @override + DateTime get occurredAt => _occurredAt; + + /// Returns a copy of this payload bound to a concrete signal name. + WorkflowRunPayload withSignalName(String signalName) => WorkflowRunPayload( + runId: runId, + workflow: workflow, + status: status, + step: step, + metadata: metadata, + signalName: signalName, + occurredAt: _occurredAt, + ); + + @override + Map get attributes => { + 'runId': runId, + 'workflow': workflow, + 'status': status.name, + if (step != null) 'step': step, + if (metadata.isNotEmpty) 'metadata': metadata, + }; } /// Payload emitted when a schedule entry becomes due for execution. -class ScheduleEntryDuePayload { +class ScheduleEntryDuePayload implements StemEvent { /// Creates a new [ScheduleEntryDuePayload] instance. const ScheduleEntryDuePayload({required this.entry, required this.tickAt}); @@ -342,10 +570,22 @@ class ScheduleEntryDuePayload { /// The time at which the entry became due. final DateTime tickAt; + + @override + String get eventName => 'schedule-entry-due'; + + @override + DateTime get occurredAt => tickAt.toUtc(); + + @override + Map get attributes => { + 'entryId': entry.id, + 'tickAt': tickAt.toUtc().toIso8601String(), + }; } /// Payload emitted when a schedule entry has been dispatched. -class ScheduleEntryDispatchedPayload { +class ScheduleEntryDispatchedPayload implements StemEvent { /// Creates a new [ScheduleEntryDispatchedPayload] instance. const ScheduleEntryDispatchedPayload({ required this.entry, @@ -365,10 +605,24 @@ class ScheduleEntryDispatchedPayload { /// The time difference between scheduled and actual execution. final Duration drift; + + @override + String get eventName => 'schedule-entry-dispatched'; + + @override + DateTime get occurredAt => executedAt.toUtc(); + + @override + Map get attributes => { + 'entryId': entry.id, + 'scheduledFor': scheduledFor.toUtc().toIso8601String(), + 'executedAt': executedAt.toUtc().toIso8601String(), + 'driftMs': drift.inMilliseconds, + }; } /// Payload emitted when a schedule entry fails to execute. -class ScheduleEntryFailedPayload { +class ScheduleEntryFailedPayload implements StemEvent { /// Creates a new [ScheduleEntryFailedPayload] instance. const ScheduleEntryFailedPayload({ required this.entry, @@ -388,10 +642,24 @@ class ScheduleEntryFailedPayload { /// The stack trace associated with the error. final StackTrace stackTrace; + + @override + String get eventName => 'schedule-entry-failed'; + + @override + DateTime get occurredAt => scheduledFor.toUtc(); + + @override + Map get attributes => { + 'entryId': entry.id, + 'scheduledFor': scheduledFor.toUtc().toIso8601String(), + 'error': error.toString(), + 'stackTrace': stackTrace.toString(), + }; } /// Payload emitted when a control command is received by a worker. -class ControlCommandReceivedPayload { +class ControlCommandReceivedPayload implements StemEvent { /// Creates a new [ControlCommandReceivedPayload] instance. const ControlCommandReceivedPayload({ required this.worker, @@ -403,10 +671,26 @@ class ControlCommandReceivedPayload { /// The control command that was received. final ControlCommandMessage command; + + @override + String get eventName => 'control-command-received'; + + @override + DateTime get occurredAt => DateTime.now().toUtc(); + + @override + Map get attributes => { + 'workerId': worker.id, + 'requestId': command.requestId, + 'type': command.type, + 'targets': command.targets, + 'payload': command.payload, + if (command.timeoutMs != null) 'timeoutMs': command.timeoutMs, + }; } /// Payload emitted when a control command completes execution. -class ControlCommandCompletedPayload { +class ControlCommandCompletedPayload implements StemEvent { /// Creates a new [ControlCommandCompletedPayload] instance. const ControlCommandCompletedPayload({ required this.worker, @@ -430,4 +714,20 @@ class ControlCommandCompletedPayload { /// Error information if the command failed, if any. final Map? error; + + @override + String get eventName => 'control-command-completed'; + + @override + DateTime get occurredAt => DateTime.now().toUtc(); + + @override + Map get attributes => { + 'workerId': worker.id, + 'requestId': command.requestId, + 'type': command.type, + 'status': status, + if (response != null) 'response': response, + if (error != null) 'error': error, + }; } diff --git a/packages/stem/lib/src/signals/signal.dart b/packages/stem/lib/src/signals/signal.dart index c102914e..1258678a 100644 --- a/packages/stem/lib/src/signals/signal.dart +++ b/packages/stem/lib/src/signals/signal.dart @@ -1,17 +1,24 @@ import 'dart:async'; +import 'package:stem/src/core/stem_event.dart'; + /// Signature for signal handlers. -typedef SignalHandler = +typedef SignalHandler = FutureOr Function(T payload, SignalContext context); /// Predicate used to filter signal payloads. -typedef SignalPredicate = bool Function(T payload, SignalContext context); +typedef SignalPredicate = + bool Function(T payload, SignalContext context); /// Context passed to every signal dispatch. class SignalContext { /// Creates a signal dispatch context. - SignalContext({required this.name, this.sender, DateTime? timestamp}) - : timestamp = timestamp ?? DateTime.now(); + SignalContext({ + required this.name, + this.sender, + DateTime? timestamp, + this.event, + }) : timestamp = timestamp ?? DateTime.now(); /// Signal identifier. final String name; @@ -22,6 +29,9 @@ class SignalContext { /// Time when dispatch started. final DateTime timestamp; + /// Shared event wrapper for this dispatch. + final StemEvent? event; + bool _cancelled = false; /// Marks the signal as cancelled, preventing handlers with lower priority @@ -70,7 +80,7 @@ class SignalSubscription { } /// Filters signal emissions based on payload and context. -class SignalFilter { +class SignalFilter { /// Creates a filter using [predicate]. factory SignalFilter.where(SignalPredicate predicate) => SignalFilter._(predicate); @@ -81,7 +91,8 @@ class SignalFilter { static bool _alwaysTrue(T _, SignalContext _) => true; /// Allows all payloads through the filter. - static SignalFilter allowAll() => SignalFilter._(_alwaysTrue); + static SignalFilter allowAll() => + SignalFilter._(_alwaysTrue); /// Returns whether the [payload] passes the filter. bool matches(T payload, SignalContext context) => @@ -99,7 +110,7 @@ class SignalFilter { } /// Dispatchable signal with typed payloads and listener management. -class Signal { +class Signal { /// Creates a signal with the given [name] and default filter. Signal({ required this.name, @@ -143,11 +154,16 @@ class Signal { /// Emits the signal to all matching listeners. Future emit(T payload, {String? sender}) async { - if (!config.enabled || _listeners.isEmpty) { + if (!config.enabled || !hasListeners) { return; } - final context = SignalContext(name: name, sender: sender); + final context = SignalContext( + name: name, + sender: sender, + timestamp: payload.occurredAt, + event: payload, + ); final snapshot = List<_Listener>.from(_listeners); for (final listener in snapshot) { if (!_listeners.contains(listener)) { @@ -172,7 +188,7 @@ class Signal { } } -class _Listener { +class _Listener { _Listener({ required this.handler, required this.filter, diff --git a/packages/stem/lib/src/signals/stem_signals.dart b/packages/stem/lib/src/signals/stem_signals.dart index 6431be60..96bb7434 100644 --- a/packages/stem/lib/src/signals/stem_signals.dart +++ b/packages/stem/lib/src/signals/stem_signals.dart @@ -32,6 +32,7 @@ library; import 'package:contextual/contextual.dart'; +import 'package:stem/src/core/stem_event.dart'; import 'package:stem/src/observability/logging.dart'; import 'package:stem/src/signals/payloads.dart'; import 'package:stem/src/signals/signal.dart'; @@ -363,6 +364,12 @@ class StemSignals { workerHeartbeat, workerChildInit, workerChildShutdown, + workflowRunStarted, + workflowRunSuspended, + workflowRunResumed, + workflowRunCompleted, + workflowRunFailed, + workflowRunCancelled, scheduleEntryDue, scheduleEntryDispatched, scheduleEntryFailed, @@ -412,56 +419,113 @@ class StemSignals { static SignalSubscription onTaskPrerun( SignalHandler handler, { String? taskName, + String? workerId, }) { - return taskPrerun.connect(handler, filter: _taskNameFilter(taskName)); + return taskPrerun.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); } /// Subscribes to the task postrun signal with optional filtering. static SignalSubscription onTaskPostrun( SignalHandler handler, { String? taskName, + String? workerId, }) { - return taskPostrun.connect(handler, filter: _taskNameFilter(taskName)); + return taskPostrun.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); } /// Subscribes to the task success signal with optional filtering. static SignalSubscription onTaskSuccess( SignalHandler handler, { String? taskName, + String? workerId, }) { - return taskSucceeded.connect(handler, filter: _taskNameFilter(taskName)); + return taskSucceeded.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); } /// Subscribes to the task failure signal with optional filtering. static SignalSubscription onTaskFailure( SignalHandler handler, { String? taskName, + String? workerId, }) { - return taskFailed.connect(handler, filter: _taskNameFilter(taskName)); + return taskFailed.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); } /// Subscribes to the task retry signal with optional filtering. static SignalSubscription onTaskRetry( SignalHandler handler, { String? taskName, + String? workerId, }) { - return taskRetry.connect(handler, filter: _taskNameFilter(taskName)); + return taskRetry.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); } /// Subscribes to the task received signal with optional filtering. static SignalSubscription onTaskReceived( SignalHandler handler, { String? taskName, + String? workerId, }) { - return taskReceived.connect(handler, filter: _taskNameFilter(taskName)); + return taskReceived.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); } /// Subscribes to the task revoked signal with optional filtering. static SignalSubscription onTaskRevoked( SignalHandler handler, { String? taskName, + String? workerId, + }) { + return taskRevoked.connect( + handler, + filter: _mergeFilters([ + _taskNameFilter(taskName), + _workerIdFilter(workerId), + ]), + ); + } + + /// Subscribes to the worker init signal with optional filtering. + static SignalSubscription onWorkerInit( + SignalHandler handler, { + String? workerId, }) { - return taskRevoked.connect(handler, filter: _taskNameFilter(taskName)); + return workerInit.connect(handler, filter: _workerIdFilter(workerId)); } /// Subscribes to the worker heartbeat signal with optional filtering. @@ -491,6 +555,22 @@ class StemSignals { ); } + /// Subscribes to the worker stopping signal with optional filtering. + static SignalSubscription onWorkerStopping( + SignalHandler handler, { + String? workerId, + }) { + return workerStopping.connect(handler, filter: _workerIdFilter(workerId)); + } + + /// Subscribes to the worker shutdown signal with optional filtering. + static SignalSubscription onWorkerShutdown( + SignalHandler handler, { + String? workerId, + }) { + return workerShutdown.connect(handler, filter: _workerIdFilter(workerId)); + } + /// Subscribes to the schedule entry due signal with optional filtering. static SignalSubscription onScheduleEntryDue( SignalHandler handler, { @@ -529,10 +609,14 @@ class StemSignals { static SignalSubscription onControlCommandReceived( SignalHandler handler, { String? commandType, + String? workerId, }) { return controlCommandReceived.connect( handler, - filter: _commandTypeFilter(commandType), + filter: _mergeFilters([ + _commandTypeFilter(commandType), + _workerIdFilter(workerId), + ]), ); } @@ -541,10 +625,14 @@ class StemSignals { static SignalSubscription onControlCommandCompleted( SignalHandler handler, { String? commandType, + String? workerId, }) { return controlCommandCompleted.connect( handler, - filter: _commandTypeFilter(commandType), + filter: _mergeFilters([ + _commandTypeFilter(commandType), + _workerIdFilter(workerId), + ]), ); } @@ -553,13 +641,12 @@ class StemSignals { SignalHandler handler, { String? workerId, }) { - return workerReady.connect( - handler, - filter: _workerIdFilter(workerId), - ); + return workerReady.connect(handler, filter: _workerIdFilter(workerId)); } - static SignalFilter? _taskNameFilter(String? taskName) { + static SignalFilter? _taskNameFilter( + String? taskName, + ) { if (taskName == null) return null; return SignalFilter.where( (payload, _) => _payloadTaskName(payload as Object) == taskName, @@ -579,7 +666,9 @@ class StemSignals { return null; } - static SignalFilter? _workerIdFilter(String? workerId) { + static SignalFilter? _workerIdFilter( + String? workerId, + ) { if (workerId == null) return null; return SignalFilter.where( (payload, _) => _payloadWorkerId(payload as Object) == workerId, @@ -602,7 +691,9 @@ class StemSignals { return null; } - static SignalFilter? _scheduleIdFilter(String? entryId) { + static SignalFilter? _scheduleIdFilter( + String? entryId, + ) { if (entryId == null) return null; return SignalFilter.where( (payload, _) => _payloadScheduleId(payload as Object) == entryId, @@ -616,7 +707,9 @@ class StemSignals { return null; } - static SignalFilter? _commandTypeFilter(String? commandType) { + static SignalFilter? _commandTypeFilter( + String? commandType, + ) { if (commandType == null) return null; return SignalFilter.where( (payload, _) => _payloadCommandType(payload as Object) == commandType, @@ -633,6 +726,19 @@ class StemSignals { return null; } + static SignalFilter? _mergeFilters( + Iterable?> filters, + ) { + SignalFilter? merged; + for (final filter in filters) { + if (filter == null) { + continue; + } + merged = merged == null ? filter : merged.and(filter); + } + return merged; + } + static SignalDispatchConfig _dispatchConfigFor(String name) => SignalDispatchConfig( enabled: _configuration.isEnabled(name), diff --git a/packages/stem/lib/src/worker/worker.dart b/packages/stem/lib/src/worker/worker.dart index 7d6edeef..dde5a086 100644 --- a/packages/stem/lib/src/worker/worker.dart +++ b/packages/stem/lib/src/worker/worker.dart @@ -109,6 +109,7 @@ import 'package:stem/src/core/encoder_keys.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/retry.dart'; import 'package:stem/src/core/stem.dart'; +import 'package:stem/src/core/stem_event.dart'; import 'package:stem/src/core/task_invocation.dart'; import 'package:stem/src/core/task_payload_encoder.dart'; import 'package:stem/src/core/unique_task_coordinator.dart'; @@ -585,16 +586,19 @@ class Worker { bool _running = false; final Map _activeDeliveries = {}; final Map _inflightPerQueue = {}; + final Set _queueSubscriptionNames = {}; int _inflight = 0; Timer? _workerHeartbeatTimer; DateTime? _lastLeaseRenewal; int? _lastQueueDepth; final Map _revocations = {}; + final Map _queuePauses = {}; int _latestRevocationVersion = 0; DateTime? _startedAt; int _startedCount = 0; int _completedCount = 0; int _failedCount = 0; + static const String _queuePauseTaskPrefix = '__stem.queue.pause__:'; /// A stream of events emitted during task processing. /// @@ -627,53 +631,10 @@ class Worker { _recordInflightGauge(); _recordConcurrencyGauge(); unawaited(_publishWorkerHeartbeat()); - final queueNames = _effectiveQueues; - if (queueNames.isEmpty) { + if (_effectiveQueues.isEmpty) { throw StateError('Worker subscription resolved no queues.'); } - for (var index = 0; index < queueNames.length; index += 1) { - final queueName = queueNames[index]; - final stream = broker.consume( - RoutingSubscription( - queues: [queueName], - broadcastChannels: index == 0 - ? _broadcastSubscriptions - : const [], - ), - prefetch: prefetch, - consumerName: consumerName, - ); - // Subscriptions are tracked and cancelled in _cancelAllSubscriptions(). - // ignore: cancel_subscriptions - final subscription = stream.listen( - (delivery) { - // Fire-and-forget; handler manages its own lifecycle. - final task = _handle(delivery); - unawaited( - task.catchError((Object error, StackTrace stack) { - _events.add( - WorkerEvent( - type: WorkerEventType.error, - envelope: delivery.envelope, - error: error, - stackTrace: stack, - ), - ); - }), - ); - }, - onError: (Object error, StackTrace stack) { - _events.add( - WorkerEvent( - type: WorkerEventType.error, - error: error, - stackTrace: stack, - ), - ); - }, - ); - _subscriptions[queueName] = subscription; - } + await _refreshQueueSubscriptions(); _startControlPlane(); _startAutoscaler(); _installSignalHandlers(); @@ -736,6 +697,7 @@ class Worker { _cancelTimers(); await heartbeatTransport.close(); _revocations.clear(); + _queuePauses.clear(); _latestRevocationVersion = 0; _inflightPerQueue.clear(); _inflight = 0; @@ -813,8 +775,96 @@ class Worker { return; } + if (_isQueuePaused(envelope.queue)) { + await _handlePausedQueueDelivery(delivery, envelope, resultEncoder); + return; + } + final decodedArgs = _decodeArgs(envelope, argsEncoder); + final groupRateSpec = handler.options.groupRateLimit != null + ? _parseRate(handler.options.groupRateLimit!) + : null; + if (rateLimiter != null && groupRateSpec != null) { + final groupKey = _groupRateLimitKey(handler.options, envelope); + try { + final decision = await rateLimiter!.acquire( + 'group:$groupKey', + tokens: groupRateSpec.tokens, + interval: groupRateSpec.period, + meta: {'task': envelope.name, 'queue': envelope.queue}, + ); + if (!decision.allowed) { + final backoff = + decision.retryAfter ?? + retryStrategy.nextDelay( + envelope.attempt, + StateError('group-rate-limit'), + StackTrace.current, + ); + await _retryDelivery( + delivery, + envelope, + resultEncoder, + backoff: backoff, + extra: { + 'rateLimited': true, + 'groupRateLimited': true, + 'groupKey': groupKey, + 'retryAfterMs': backoff.inMilliseconds, + }, + ); + return; + } + } on Object catch (error, stack) { + final mode = handler.options.groupRateLimiterFailureMode; + if (mode == RateLimiterFailureMode.failOpen) { + StemMetrics.instance.increment( + 'stem.rate_limiter.degraded', + tags: { + 'task': envelope.name, + 'queue': envelope.queue, + 'scope': 'group', + 'mode': mode.name, + }, + ); + stemLogger.warning( + 'Group rate limiter unavailable; continuing in fail-open mode', + Context( + _logContext({ + 'task': envelope.name, + 'id': envelope.id, + 'queue': envelope.queue, + 'group': groupKey, + 'error': error.toString(), + 'stack': stack.toString(), + }), + ), + ); + } else { + final backoff = retryStrategy.nextDelay( + envelope.attempt, + error, + stack, + ); + await _retryDelivery( + delivery, + envelope, + resultEncoder, + backoff: backoff, + extra: { + 'groupRateLimited': true, + 'groupKey': groupKey, + 'rateLimiterUnavailable': true, + 'failureMode': mode.name, + 'error': error.toString(), + }, + ); + return; + } + } + } + final rateSpec = handler.options.rateLimit != null ? _parseRate(handler.options.rateLimit!) : null; @@ -833,32 +883,15 @@ class Worker { StateError('rate-limit'), StackTrace.current, ); - await broker.nack(delivery, requeue: false); - await broker.publish( - envelope.copyWith(notBefore: DateTime.now().add(backoff)), - ); - await backend.set( - envelope.id, - TaskState.retried, - attempt: envelope.attempt, - meta: _statusMeta( - envelope, - resultEncoder, - extra: { - 'rateLimited': true, - 'retryAfterMs': backoff.inMilliseconds, - }, - ), - ); - _events.add( - WorkerEvent( - type: WorkerEventType.retried, - envelope: envelope, - data: { - 'rateLimited': true, - 'retryAfterMs': backoff.inMilliseconds, - }, - ), + await _retryDelivery( + delivery, + envelope, + resultEncoder, + backoff: backoff, + extra: { + 'rateLimited': true, + 'retryAfterMs': backoff.inMilliseconds, + }, ); return; } @@ -2240,6 +2273,70 @@ class Worker { String _rateLimitKey(TaskOptions options, Envelope envelope) => '${envelope.name}:${envelope.headers['tenant'] ?? 'global'}'; + /// Resolves a shared group rate-limit key for the current task. + String _groupRateLimitKey(TaskOptions options, Envelope envelope) { + final configured = options.groupRateKey?.trim(); + if (configured != null && configured.isNotEmpty) { + return configured; + } + final header = options.groupRateKeyHeader.trim(); + if (header.isNotEmpty) { + final fromHeader = envelope.headers[header]?.trim(); + if (fromHeader != null && fromHeader.isNotEmpty) { + return fromHeader; + } + } + return 'global'; + } + + /// Requeues a delivery after [backoff] and records retry metadata. + Future _retryDelivery( + Delivery delivery, + Envelope envelope, + TaskPayloadEncoder resultEncoder, { + required Duration backoff, + Map extra = const {}, + }) async { + await broker.nack(delivery, requeue: false); + await broker.publish( + envelope.copyWith(notBefore: DateTime.now().add(backoff)), + ); + final data = { + ...extra, + if (!extra.containsKey('retryAfterMs')) + 'retryAfterMs': backoff.inMilliseconds, + }; + await backend.set( + envelope.id, + TaskState.retried, + attempt: envelope.attempt, + meta: _statusMeta(envelope, resultEncoder, extra: data), + ); + _events.add( + WorkerEvent( + type: WorkerEventType.retried, + envelope: envelope, + data: data, + ), + ); + } + + /// Requeues deliveries from paused queues without executing handlers. + Future _handlePausedQueueDelivery( + Delivery delivery, + Envelope envelope, + TaskPayloadEncoder resultEncoder, + ) async { + const backoff = Duration(milliseconds: 250); + await _retryDelivery( + delivery, + envelope, + resultEncoder, + backoff: backoff, + extra: {'queuePaused': true, 'queue': envelope.queue}, + ); + } + /// Parses a rate limit string such as "10/m" into a spec. _RateSpec? _parseRate(String rate) { final parts = rate.split('/'); @@ -2360,11 +2457,95 @@ class Worker { return result; } + /// Refreshes queue subscriptions after pause/resume changes. + Future _refreshQueueSubscriptions() async { + final existing = List.from(_queueSubscriptionNames); + for (final queueName in existing) { + final subscription = _subscriptions.remove(queueName); + _queueSubscriptionNames.remove(queueName); + if (subscription == null) continue; + try { + await subscription.cancel(); + } on Object catch (error, stack) { + stemLogger.warning( + 'Failed to cancel queue subscription: $error', + Context( + _logContext({ + 'queue': queueName, + 'stack': stack.toString(), + }), + ), + ); + } + } + + final activeQueues = _effectiveQueues + .where((queueName) => !_isQueuePaused(queueName)) + .toList(growable: false); + if (activeQueues.isEmpty) { + stemLogger.info( + 'All subscribed queues are paused.', + Context( + _logContext({ + 'worker': _workerIdentifier, + 'queues': _effectiveQueues.join(','), + }), + ), + ); + return; + } + + for (var index = 0; index < activeQueues.length; index += 1) { + final queueName = activeQueues[index]; + final stream = broker.consume( + RoutingSubscription( + queues: [queueName], + broadcastChannels: index == 0 + ? _broadcastSubscriptions + : const [], + ), + prefetch: prefetch, + consumerName: consumerName, + ); + // Subscriptions are tracked and cancelled in _cancelAllSubscriptions(). + // ignore: cancel_subscriptions + final subscription = stream.listen( + (delivery) { + final task = _handle(delivery); + unawaited( + task.catchError((Object error, StackTrace stack) { + _events.add( + WorkerEvent( + type: WorkerEventType.error, + envelope: delivery.envelope, + error: error, + stackTrace: stack, + ), + ); + }), + ); + }, + onError: (Object error, StackTrace stack) { + _events.add( + WorkerEvent( + type: WorkerEventType.error, + error: error, + stackTrace: stack, + ), + ); + }, + ); + _subscriptions[queueName] = subscription; + _queueSubscriptionNames.add(queueName); + } + } + /// Cancels broker subscriptions and clears tracking state. Future _cancelAllSubscriptions() async { if (_subscriptions.isEmpty) return; final subs = List>.from(_subscriptions.values); _subscriptions.clear(); + _queueSubscriptionNames.clear(); for (final sub in subs) { try { await sub.cancel(); @@ -2961,10 +3142,15 @@ class Worker { try { final fetched = await store.list(namespace); for (final entry in fetched) { + if (_isQueuePauseTaskId(entry.taskId)) { + _applyQueuePauseEntry(entry, clock: now); + continue; + } _applyRevocationEntry(entry, clock: now); } await store.pruneExpired(namespace, now); _pruneExpiredLocalRevocations(now); + _pruneExpiredQueuePauses(now); } on Object catch (error, stack) { stemLogger.warning( 'Failed to synchronize revokes: $error', @@ -2984,6 +3170,58 @@ class Worker { remove.forEach(_revocations.remove); } + /// Drops expired queue pause entries from local state. + void _pruneExpiredQueuePauses(DateTime now) { + final remove = []; + _queuePauses.forEach((key, value) { + if (value.isExpired(now)) { + remove.add(key); + } + }); + remove.forEach(_queuePauses.remove); + } + + bool _isQueuePauseTaskId(String taskId) => + taskId.startsWith(_queuePauseTaskPrefix); + + String _queuePauseTaskId(String queue) => '$_queuePauseTaskPrefix$queue'; + + String? _queueNameFromPauseTaskId(String taskId) { + if (!_isQueuePauseTaskId(taskId)) return null; + final queue = taskId.substring(_queuePauseTaskPrefix.length).trim(); + return queue.isEmpty ? null : queue; + } + + void _applyQueuePauseEntry(RevokeEntry entry, {DateTime? clock}) { + final queueName = _queueNameFromPauseTaskId(entry.taskId); + if (queueName == null) return; + final now = clock ?? DateTime.now().toUtc(); + if (entry.isExpired(now)) { + _queuePauses.remove(queueName); + return; + } + final current = _queuePauses[queueName]; + if (current == null || entry.version >= current.version) { + _queuePauses[queueName] = entry; + } + } + + bool _isQueuePaused(String queueName) { + final entry = _queuePauses[queueName]; + if (entry == null) return false; + if (entry.isExpired(DateTime.now().toUtc())) { + _queuePauses.remove(queueName); + return false; + } + return true; + } + + List _pausedQueueNames() { + _pruneExpiredQueuePauses(DateTime.now().toUtc()); + final queues = _queuePauses.keys.toList()..sort(); + return queues; + } + RevokeEntry? _revocationFor(String taskId) { final entry = _revocations[taskId]; if (entry == null) return null; @@ -3488,6 +3726,94 @@ class Worker { }; } + List _extractQueueTargets(Map payload) { + final queues = {}; + final queue = (payload['queue'] as String?)?.trim(); + if (queue != null && queue.isNotEmpty) { + queues.add(queue); + } + final rawQueues = payload['queues']; + if (rawQueues is List) { + for (final value in rawQueues) { + final queueName = value.toString().trim(); + if (queueName.isNotEmpty) { + queues.add(queueName); + } + } + } + return queues.toList()..sort(); + } + + Future> _processQueuePauseCommand( + ControlCommandMessage command, { + required bool paused, + }) async { + final payload = command.payload; + final namespaceOverride = (payload['namespace'] as String?)?.trim(); + if (namespaceOverride != null && + namespaceOverride.isNotEmpty && + namespaceOverride != namespace) { + return { + 'queues': const [], + 'updated': 0, + 'paused': _pausedQueueNames(), + 'namespace': namespace, + }; + } + final queues = _extractQueueTargets(payload); + if (queues.isEmpty) { + throw StateError('Queue control command requires at least one queue.'); + } + + final now = DateTime.now().toUtc(); + final requester = (payload['requester'] as String?)?.trim(); + final reason = (payload['reason'] as String?)?.trim(); + final baseVersion = generateRevokeVersion(); + final entries = []; + for (var i = 0; i < queues.length; i += 1) { + entries.add( + RevokeEntry( + namespace: namespace, + taskId: _queuePauseTaskId(queues[i]), + version: baseVersion + i, + issuedAt: now, + terminate: false, + reason: reason, + requestedBy: requester, + expiresAt: paused ? null : now, + ), + ); + } + + final store = revokeStore; + if (store != null) { + try { + await store.upsertAll(entries); + await store.pruneExpired(namespace, now); + } on Object catch (error, stack) { + stemLogger.warning( + 'Failed to persist queue pause state: $error', + Context(_logContext({'stack': stack.toString()})), + ); + throw StateError('Failed to persist queue pause state: $error'); + } + } + + for (final entry in entries) { + _applyQueuePauseEntry(entry, clock: now); + } + _pruneExpiredQueuePauses(now); + await _refreshQueueSubscriptions(); + + return { + 'queues': queues, + 'updated': queues.length, + 'persisted': store != null, + 'paused': _pausedQueueNames(), + 'namespace': namespace, + }; + } + /// Starts the control-plane subscription for revoke and shutdown commands. void _startControlPlane() { final controlQueues = { @@ -3605,6 +3931,39 @@ class Worker { }, ); } + case 'queue_pause': + final result = await _processQueuePauseCommand( + command, + paused: true, + ); + reply = ControlReplyMessage( + requestId: command.requestId, + workerId: _workerIdentifier, + status: 'ok', + payload: result, + ); + case 'queue_resume': + final result = await _processQueuePauseCommand( + command, + paused: false, + ); + reply = ControlReplyMessage( + requestId: command.requestId, + workerId: _workerIdentifier, + status: 'ok', + payload: result, + ); + case 'queue_status': + reply = ControlReplyMessage( + requestId: command.requestId, + workerId: _workerIdentifier, + status: 'ok', + payload: { + 'namespace': namespace, + 'paused': _pausedQueueNames(), + 'subscriptions': _subscriptionMetadata(), + }, + ); case 'shutdown': final mode = _parseShutdownMode(command.payload['mode'] as String?); final summary = await _handleShutdownRequest(mode); @@ -3698,6 +4057,7 @@ class Worker { 'prefetch': prefetch, 'inflight': _inflight, 'queues': queues, + 'pausedQueues': _pausedQueueNames(), 'active': activeTasks, 'subscriptions': _subscriptionMetadata(), 'lastLeaseRenewalMsAgo': _lastLeaseRenewal == null @@ -3739,6 +4099,7 @@ class Worker { 'timestamp': now.toIso8601String(), 'inflight': _inflight, 'active': active, + 'pausedQueues': _pausedQueueNames(), if (includeRevoked) 'revoked': revoked, }; } @@ -3935,7 +4296,7 @@ class Worker { /// An event emitted during worker operation. /// /// Provides details about task lifecycle, errors, and progress. -class WorkerEvent { +class WorkerEvent implements StemEvent { /// Creates a worker event. /// /// [type] indicates the event kind. [envelope] provides task details for @@ -3944,17 +4305,21 @@ class WorkerEvent { /// percentage. [data] holds additional event-specific information. WorkerEvent({ required this.type, + DateTime? timestamp, this.envelope, this.envelopeId, this.error, this.stackTrace, this.progress, this.data, - }); + }) : timestamp = timestamp ?? DateTime.now(); /// The type of event. final WorkerEventType type; + /// Time when the event was created. + final DateTime timestamp; + /// The envelope associated with the event, if applicable. final Envelope? envelope; @@ -3972,6 +4337,24 @@ class WorkerEvent { /// Additional data for the event. final Map? data; + + @override + String get eventName => 'worker.${type.name}'; + + @override + DateTime get occurredAt => timestamp; + + @override + Map get attributes => { + if (envelope != null) 'envelopeId': envelope!.id, + if (envelope != null) 'taskName': envelope!.name, + if (envelope != null) 'queue': envelope!.queue, + if (envelopeId != null) 'trackedEnvelopeId': envelopeId, + if (error != null) 'error': error.toString(), + if (stackTrace != null) 'stackTrace': stackTrace.toString(), + if (progress != null) 'progress': progress, + if (data != null) 'data': data, + }; } /// Types of events a worker can emit. diff --git a/packages/stem/lib/src/workflow/runtime/workflow_introspection.dart b/packages/stem/lib/src/workflow/runtime/workflow_introspection.dart index 00b6ff27..223944ac 100644 --- a/packages/stem/lib/src/workflow/runtime/workflow_introspection.dart +++ b/packages/stem/lib/src/workflow/runtime/workflow_introspection.dart @@ -1,3 +1,5 @@ +import 'package:stem/src/core/stem_event.dart'; + /// Enumerates workflow step event types emitted by the runtime. enum WorkflowStepEventType { /// Step execution started. @@ -14,7 +16,7 @@ enum WorkflowStepEventType { } /// Step-level execution event emitted by the workflow runtime. -class WorkflowStepEvent { +class WorkflowStepEvent implements StemEvent { /// Creates a workflow step execution event. WorkflowStepEvent({ required this.runId, @@ -54,6 +56,23 @@ class WorkflowStepEvent { /// Optional metadata associated with the event. final Map? metadata; + + @override + String get eventName => 'workflow.step.${type.name}'; + + @override + DateTime get occurredAt => timestamp; + + @override + Map get attributes => { + 'runId': runId, + 'workflow': workflow, + 'stepId': stepId, + if (iteration != null) 'iteration': iteration, + if (result != null) 'result': result, + if (error != null) 'error': error, + if (metadata != null) 'metadata': metadata, + }; } /// Sink for workflow step execution events. diff --git a/packages/stem/lib/stem.dart b/packages/stem/lib/stem.dart index 339ed7e1..b534e288 100644 --- a/packages/stem/lib/stem.dart +++ b/packages/stem/lib/stem.dart @@ -92,8 +92,10 @@ export 'src/core/contracts.dart'; export 'src/core/encoder_keys.dart'; export 'src/core/envelope.dart'; export 'src/core/function_task_handler.dart'; +export 'src/core/queue_events.dart'; export 'src/core/retry.dart'; export 'src/core/stem.dart'; +export 'src/core/stem_event.dart'; export 'src/core/task_invocation.dart'; export 'src/core/task_payload_encoder.dart'; export 'src/core/task_result.dart'; diff --git a/packages/stem/test/unit/canvas/canvas_test.dart b/packages/stem/test/unit/canvas/canvas_test.dart index fa416056..5182f0da 100644 --- a/packages/stem/test/unit/canvas/canvas_test.dart +++ b/packages/stem/test/unit/canvas/canvas_test.dart @@ -79,6 +79,29 @@ void main() { final status = await _waitForSuccess(backend, result.callbackTaskId); expect(status.payload, equals(5)); }); + + test( + 'submitBatch returns stable id and terminal lifecycle summary', + () async { + final batch = await canvas.submitBatch([ + task('echo', args: {'value': 8}), + task('echo', args: {'value': 13}), + ]); + + expect(batch.batchId, isNotEmpty); + expect(batch.taskIds, hasLength(2)); + + final status = await _waitForBatchTerminal(canvas, batch.batchId); + expect(status.state, BatchLifecycleState.succeeded); + expect(status.expected, equals(2)); + expect(status.completed, equals(2)); + expect(status.succeededCount, equals(2)); + expect(status.failedCount, equals(0)); + expect(status.cancelledCount, equals(0)); + expect(status.failedTaskIds, isEmpty); + expect(status.meta['stem.batch'], isTrue); + }, + ); }); } @@ -100,6 +123,24 @@ Future _waitForSuccess( } } +Future _waitForBatchTerminal( + Canvas canvas, + String batchId, { + Duration timeout = const Duration(seconds: 2), +}) async { + final start = DateTime.now(); + while (true) { + final status = await canvas.inspectBatch(batchId); + if (status != null && status.isTerminal) { + return status; + } + if (DateTime.now().difference(start) > timeout) { + throw TimeoutException('Batch $batchId did not complete in time'); + } + await Future.delayed(const Duration(milliseconds: 50)); + } +} + class _EchoTask implements TaskHandler { @override String get name => 'echo'; diff --git a/packages/stem/test/unit/core/contracts_test.dart b/packages/stem/test/unit/core/contracts_test.dart index 48102303..c1c62716 100644 --- a/packages/stem/test/unit/core/contracts_test.dart +++ b/packages/stem/test/unit/core/contracts_test.dart @@ -124,6 +124,10 @@ void main() { 'maxRetries': 5, 'softTimeLimitMs': 1000, 'hardTimeLimitMs': '2000', + 'groupRateLimit': '25/m', + 'groupRateKey': 'tenant:acme', + 'groupRateKeyHeader': 'x-tenant', + 'groupRateLimiterFailureMode': 'failClosed', 'unique': true, 'uniqueForMs': 5000, 'priority': 3, @@ -144,6 +148,13 @@ void main() { expect(options.maxRetries, equals(5)); expect(options.softTimeLimit, equals(const Duration(milliseconds: 1000))); expect(options.hardTimeLimit, equals(const Duration(milliseconds: 2000))); + expect(options.groupRateLimit, equals('25/m')); + expect(options.groupRateKey, equals('tenant:acme')); + expect(options.groupRateKeyHeader, equals('x-tenant')); + expect( + options.groupRateLimiterFailureMode, + equals(RateLimiterFailureMode.failClosed), + ); expect(options.unique, isTrue); expect(options.uniqueFor, equals(const Duration(milliseconds: 5000))); expect(options.priority, equals(3)); diff --git a/packages/stem/test/unit/core/queue_events_test.dart b/packages/stem/test/unit/core/queue_events_test.dart new file mode 100644 index 00000000..29e739ae --- /dev/null +++ b/packages/stem/test/unit/core/queue_events_test.dart @@ -0,0 +1,136 @@ +import 'dart:async'; + +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + group('QueueEvents', () { + late InMemoryBroker broker; + late QueueEventsProducer producer; + + setUp(() { + broker = InMemoryBroker( + namespace: 'queue-events-${DateTime.now().microsecondsSinceEpoch}', + ); + producer = QueueEventsProducer(broker: broker); + }); + + tearDown(() async { + await broker.close(); + }); + + test('receives custom events for the subscribed queue', () async { + final listener = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-listener', + ); + await listener.start(); + addTearDown(listener.close); + + final received = listener + .on('order.created') + .first + .timeout( + const Duration(seconds: 5), + ); + + final eventId = await producer.emit( + 'orders', + 'order.created', + payload: const {'orderId': 'o-1'}, + headers: const {'x-source': 'test'}, + meta: const {'tenant': 'acme'}, + ); + + final event = await received; + expect(event.id, eventId); + expect(event.queue, 'orders'); + expect(event.name, 'order.created'); + expect(event.payload['orderId'], 'o-1'); + expect(event.headers['x-source'], 'test'); + expect(event.meta['tenant'], 'acme'); + }); + + test('ignores events from other queues', () async { + final listener = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-listener', + ); + await listener.start(); + addTearDown(listener.close); + + final events = []; + final sub = listener.events.listen(events.add); + addTearDown(sub.cancel); + + await producer.emit( + 'billing', + 'invoice.created', + payload: const {'invoiceId': 'i-1'}, + ); + + await Future.delayed(const Duration(milliseconds: 200)); + expect(events, isEmpty); + }); + + test('fans out to multiple listeners on the same queue', () async { + final listenerA = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-a', + ); + final listenerB = QueueEvents( + broker: broker, + queue: 'orders', + consumerName: 'orders-b', + ); + await listenerA.start(); + await listenerB.start(); + addTearDown(listenerA.close); + addTearDown(listenerB.close); + + final firstA = listenerA + .on('order.updated') + .first + .timeout( + const Duration(seconds: 5), + ); + final firstB = listenerB + .on('order.updated') + .first + .timeout( + const Duration(seconds: 5), + ); + + await producer.emit( + 'orders', + 'order.updated', + payload: const {'orderId': 'o-1', 'status': 'paid'}, + ); + + final results = await Future.wait([firstA, firstB]); + expect(results, hasLength(2)); + expect(results[0].payload['status'], 'paid'); + expect(results[1].payload['status'], 'paid'); + }); + + test('validates queue and event names', () async { + expect( + () => producer.emit('', 'evt'), + throwsA(isA()), + ); + expect( + () => producer.emit('queue', ' '), + throwsA(isA()), + ); + final listener = QueueEvents(broker: broker, queue: 'queue'); + expect( + () => listener.on(''), + throwsA(isA()), + ); + await listener.close(); + }); + }); +} diff --git a/packages/stem/test/unit/core/stem_event_test.dart b/packages/stem/test/unit/core/stem_event_test.dart new file mode 100644 index 00000000..8ad552d9 --- /dev/null +++ b/packages/stem/test/unit/core/stem_event_test.dart @@ -0,0 +1,46 @@ +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + group('StemEvent', () { + test('WorkerEvent implements StemEvent contract', () { + final event = WorkerEvent(type: WorkerEventType.completed); + + expect(event, isA()); + expect(event.eventName, 'worker.completed'); + expect(event.occurredAt, isA()); + expect(event.attributes, isA>()); + }); + + test('QueueCustomEvent implements StemEvent contract', () { + final event = QueueCustomEvent( + id: 'evt-1', + queue: 'orders', + name: 'order.created', + payload: const {'id': 'o-1'}, + emittedAt: DateTime.utc(2026, 2, 24, 15), + ); + + expect(event, isA()); + expect(event.eventName, 'order.created'); + expect(event.occurredAt, DateTime.utc(2026, 2, 24, 15)); + expect(event.attributes['queue'], 'orders'); + }); + + test('WorkflowStepEvent implements StemEvent contract', () { + final event = WorkflowStepEvent( + runId: 'run-1', + workflow: 'checkout', + stepId: 'charge', + type: WorkflowStepEventType.started, + timestamp: DateTime.utc(2026, 2, 24, 16), + ); + + expect(event, isA()); + expect(event.eventName, 'workflow.step.started'); + expect(event.occurredAt, DateTime.utc(2026, 2, 24, 16)); + expect(event.attributes['runId'], 'run-1'); + expect(event.attributes['stepId'], 'charge'); + }); + }); +} diff --git a/packages/stem/test/unit/signals/signal_test.dart b/packages/stem/test/unit/signals/signal_test.dart index 56643d09..f6c6fce1 100644 --- a/packages/stem/test/unit/signals/signal_test.dart +++ b/packages/stem/test/unit/signals/signal_test.dart @@ -1,9 +1,10 @@ +import 'package:stem/src/core/stem_event.dart'; import 'package:stem/src/signals/signal.dart'; import 'package:test/test.dart'; void main() { test('SignalContext cancellation stops remaining handlers', () async { - final signal = Signal(name: 'test'); + final signal = Signal<_TestEvent>(name: 'test'); final calls = []; signal @@ -15,46 +16,50 @@ void main() { calls.add('second'); }); - await signal.emit(1); + await signal.emit(const _TestEvent(1)); expect(calls, ['first']); }); test('Signal handlers respect priority and once', () async { - final signal = Signal(name: 'priority'); + final signal = Signal<_TestEvent>(name: 'priority'); final calls = []; signal ..connect((payload, context) { - calls.add(1); + calls.add(payload.value); }, priority: 1) ..connect((payload, context) { - calls.add(2); + calls.add(payload.value * 2); }, priority: 5) ..connect( (payload, context) { - calls.add(3); + calls.add(payload.value * 3); }, once: true, priority: 3, ); - await signal.emit(1); - await signal.emit(1); + await signal.emit(const _TestEvent(1)); + await signal.emit(const _TestEvent(1)); expect(calls, [2, 3, 1, 2, 1]); }); test('Signal filters can be combined and negated', () { - final even = SignalFilter.where((payload, _) => payload.isEven); - final gtFive = SignalFilter.where((payload, _) => payload > 5); + final even = SignalFilter<_TestEvent>.where( + (payload, _) => payload.value.isEven, + ); + final gtFive = SignalFilter<_TestEvent>.where( + (payload, _) => payload.value > 5, + ); final combined = even.and(gtFive); final context = SignalContext(name: 'filter'); - expect(combined.matches(6, context), isTrue); - expect(combined.matches(4, context), isFalse); - expect(combined.negate().matches(4, context), isTrue); + expect(combined.matches(const _TestEvent(6), context), isTrue); + expect(combined.matches(const _TestEvent(4), context), isFalse); + expect(combined.negate().matches(const _TestEvent(4), context), isTrue); }); test('Signal dispatch reports errors without throwing', () async { @@ -62,7 +67,7 @@ void main() { StackTrace? reportedStack; final signal = - Signal( + Signal<_TestEvent>( name: 'errors', config: SignalDispatchConfig( onError: (name, error, stackTrace) { @@ -74,12 +79,48 @@ void main() { throw StateError('boom'); }); - await signal.emit(1); + await signal.emit(const _TestEvent(1)); expect(reportedError, isA()); expect(reportedStack, isNotNull); }); + test( + 'SignalContext.event references the emitted StemEvent payload', + () async { + final signal = Signal<_TestEvent>(name: 'event-payload'); + _TestEvent? captured; + SignalContext? capturedContext; + + signal.connect((event, context) { + captured = event; + capturedContext = context; + }); + + const payload = _TestEvent(42); + await signal.emit(payload, sender: 'tester'); + + expect(captured, isNotNull); + expect(captured, same(payload)); + expect(captured!.eventName, 'test-event'); + expect(captured!.attributes['value'], 42); + expect(capturedContext, isNotNull); + expect(capturedContext!.event, same(payload)); + expect(capturedContext!.sender, 'tester'); + }, + ); + + test('Signal hasListeners tracks typed listeners', () { + final signal = Signal<_TestEvent>(name: 'listeners'); + expect(signal.hasListeners, isFalse); + + final sub = signal.connect((event, context) {}); + expect(signal.hasListeners, isTrue); + + sub.cancel(); + expect(signal.hasListeners, isFalse); + }); + test('SignalDispatchConfig copyWith preserves defaults', () { const config = SignalDispatchConfig(); final updated = config.copyWith(enabled: false); @@ -88,3 +129,18 @@ void main() { expect(updated.onError, isNull); }); } + +class _TestEvent implements StemEvent { + const _TestEvent(this.value); + + final int value; + + @override + String get eventName => 'test-event'; + + @override + DateTime get occurredAt => DateTime.fromMillisecondsSinceEpoch(value * 1000); + + @override + Map get attributes => {'value': value}; +} diff --git a/packages/stem/test/unit/signals/stem_signals_test.dart b/packages/stem/test/unit/signals/stem_signals_test.dart index 162a1b35..738e98bc 100644 --- a/packages/stem/test/unit/signals/stem_signals_test.dart +++ b/packages/stem/test/unit/signals/stem_signals_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/src/control/control_messages.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/signals/emitter.dart'; import 'package:stem/src/signals/payloads.dart'; @@ -53,6 +54,7 @@ void main() { final failureSub = StemSignals.onTaskFailure( (payload, _) => failures.add(payload.envelope.name), taskName: 'target.task', + workerId: 'worker-1', ); final readySub = StemSignals.onWorkerReady( (payload, _) => workerEvents.add(payload.worker.id), @@ -75,6 +77,11 @@ void main() { worker, error: StateError('fail'), ); + await emitter.taskFailed( + envelope, + otherWorker, + error: StateError('fail'), + ); await emitter.workerReady(worker); await emitter.workerReady(otherWorker); @@ -83,6 +90,102 @@ void main() { expect(workerEvents, ['worker-1']); }); + test( + 'StemSignals supports worker-scoped lifecycle and control subscriptions', + () async { + const workerA = WorkerInfo( + id: 'worker-a', + queues: ['default'], + broadcasts: [], + ); + const workerB = WorkerInfo( + id: 'worker-b', + queues: ['default'], + broadcasts: [], + ); + + final lifecycle = []; + final control = []; + final subs = [ + StemSignals.onWorkerInit( + (payload, _) => lifecycle.add(payload.worker.id), + workerId: workerA.id, + ), + StemSignals.onWorkerStopping( + (payload, _) => lifecycle.add(payload.worker.id), + workerId: workerA.id, + ), + StemSignals.onWorkerShutdown( + (payload, _) => lifecycle.add(payload.worker.id), + workerId: workerA.id, + ), + StemSignals.onControlCommandReceived( + (payload, _) => + control.add('${payload.worker.id}:${payload.command.type}'), + workerId: workerA.id, + commandType: 'ping', + ), + StemSignals.onControlCommandCompleted( + (payload, _) => control.add( + '${payload.worker.id}:${payload.command.type}:${payload.status}', + ), + workerId: workerA.id, + commandType: 'ping', + ), + ]; + addTearDown(() { + for (final sub in subs) { + sub.cancel(); + } + }); + + const emitter = StemSignalEmitter(); + await emitter.workerInit(workerA); + await emitter.workerInit(workerB); + await emitter.workerStopping(workerA); + await emitter.workerStopping(workerB); + await emitter.workerShutdown(workerA); + await emitter.workerShutdown(workerB); + + final pingA = ControlCommandMessage( + requestId: 'req-a', + type: 'ping', + targets: ['*'], + ); + final pauseA = ControlCommandMessage( + requestId: 'req-b', + type: 'pause', + targets: ['*'], + ); + final pingB = ControlCommandMessage( + requestId: 'req-c', + type: 'ping', + targets: ['*'], + ); + await emitter.controlCommandReceived(workerA, pingA); + await emitter.controlCommandReceived(workerA, pauseA); + await emitter.controlCommandReceived(workerB, pingB); + await emitter.controlCommandCompleted( + workerA, + pingA, + status: 'ok', + ); + await emitter.controlCommandCompleted( + workerA, + pauseA, + status: 'ok', + ); + await emitter.controlCommandCompleted( + workerB, + pingB, + status: 'ok', + ); + + expect(lifecycle, equals(['worker-a', 'worker-a', 'worker-a'])); + expect(control, equals(['worker-a:ping', 'worker-a:ping:ok'])); + }, + ); + test('StemSignals forwards errors to configured reporter', () async { Object? capturedError; String? capturedSignal; @@ -106,4 +209,53 @@ void main() { expect(capturedSignal, StemSignals.afterTaskPublishName); expect(capturedError, isA()); }); + + test('emitted payloads use canonical signal names', () async { + String? workerEventName; + String? workflowStartedName; + String? workflowResumedName; + + final workerSub = StemSignals.workerReady.connect((payload, _) { + workerEventName = payload.eventName; + }); + final startedSub = StemSignals.workflowRunStarted.connect((payload, _) { + workflowStartedName = payload.eventName; + }); + final resumedSub = StemSignals.workflowRunResumed.connect((payload, _) { + workflowResumedName = payload.eventName; + }); + + addTearDown(() { + workerSub.cancel(); + startedSub.cancel(); + resumedSub.cancel(); + }); + + const emitter = StemSignalEmitter(); + const worker = WorkerInfo( + id: 'worker-events', + queues: ['default'], + broadcasts: [], + ); + + await emitter.workerReady(worker); + await emitter.workflowRunStarted( + WorkflowRunPayload( + runId: 'run-1', + workflow: 'wf', + status: WorkflowRunStatus.running, + ), + ); + await emitter.workflowRunResumed( + WorkflowRunPayload( + runId: 'run-2', + workflow: 'wf', + status: WorkflowRunStatus.running, + ), + ); + + expect(workerEventName, StemSignals.workerReadyName); + expect(workflowStartedName, StemSignals.workflowRunStartedName); + expect(workflowResumedName, StemSignals.workflowRunResumedName); + }); } diff --git a/packages/stem/test/unit/worker/worker_test.dart b/packages/stem/test/unit/worker/worker_test.dart index 6277ac62..fc9b9508 100644 --- a/packages/stem/test/unit/worker/worker_test.dart +++ b/packages/stem/test/unit/worker/worker_test.dart @@ -1288,6 +1288,283 @@ void main() { broker.dispose(); }); + test('shares group limiter keys across task types', () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + final backend = InMemoryResultBackend(); + final limiter = _ScenarioRateLimiter((key, attempt) { + if (key == 'group:acme' && attempt == 2) { + return const RateLimitDecision( + allowed: false, + retryAfter: Duration(milliseconds: 25), + ); + } + return const RateLimitDecision(allowed: true); + }); + final registry = SimpleTaskRegistry() + ..register( + FunctionTaskHandler( + name: 'tasks.group.a', + options: const TaskOptions( + groupRateLimit: '1/s', + groupRateKeyHeader: 'tenant', + ), + entrypoint: (context, args) async => null, + ), + ) + ..register( + FunctionTaskHandler( + name: 'tasks.group.b', + options: const TaskOptions( + groupRateLimit: '1/s', + groupRateKeyHeader: 'tenant', + ), + entrypoint: (context, args) async => null, + ), + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + rateLimiter: limiter, + consumerName: 'group-limit-worker', + concurrency: 1, + prefetchMultiplier: 1, + ); + final events = []; + final sub = worker.events.listen(events.add); + + await worker.start(); + final stem = Stem(broker: broker, registry: registry, backend: backend); + final firstId = await stem.enqueue( + 'tasks.group.a', + headers: const {'tenant': 'acme'}, + ); + final secondId = await stem.enqueue( + 'tasks.group.b', + headers: const {'tenant': 'acme'}, + ); + + await _waitFor( + () => + events + .where((event) => event.type == WorkerEventType.completed) + .length >= + 2, + timeout: const Duration(seconds: 3), + ); + + expect(await backend.get(firstId), isNotNull); + expect((await backend.get(firstId))?.state, TaskState.succeeded); + expect((await backend.get(secondId))?.state, TaskState.succeeded); + expect( + limiter.keys.where((key) => key == 'group:acme').length, + greaterThanOrEqualTo(2), + ); + expect( + events.any( + (event) => + event.type == WorkerEventType.retried && + event.data?['groupRateLimited'] == true, + ), + isTrue, + ); + + await sub.cancel(); + await worker.shutdown(); + broker.dispose(); + }); + + test( + 'group limiter fail-open continues execution on limiter errors', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + final backend = InMemoryResultBackend(); + final limiter = _ScenarioRateLimiter((key, attempt) { + throw StateError('limiter unavailable'); + }); + final registry = SimpleTaskRegistry() + ..register( + FunctionTaskHandler( + name: 'tasks.group.failopen', + options: const TaskOptions( + groupRateLimit: '10/m', + groupRateKeyHeader: 'tenant', + groupRateLimiterFailureMode: RateLimiterFailureMode.failOpen, + ), + entrypoint: (context, args) async => null, + ), + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + rateLimiter: limiter, + consumerName: 'group-fail-open-worker', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await worker.start(); + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue( + 'tasks.group.failopen', + headers: const {'tenant': 'acme'}, + ); + + await _waitForTaskState(backend, taskId, TaskState.succeeded); + expect((await backend.get(taskId))?.state, TaskState.succeeded); + + await worker.shutdown(); + broker.dispose(); + }, + ); + + test( + 'group limiter fail-closed requeues while limiter is unavailable', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + final backend = InMemoryResultBackend(); + final limiter = _ScenarioRateLimiter((key, attempt) { + throw StateError('limiter unavailable'); + }); + var executed = 0; + final registry = SimpleTaskRegistry() + ..register( + FunctionTaskHandler( + name: 'tasks.group.failclosed', + options: const TaskOptions( + groupRateLimit: '10/m', + groupRateKeyHeader: 'tenant', + groupRateLimiterFailureMode: RateLimiterFailureMode.failClosed, + ), + entrypoint: (context, args) async { + executed += 1; + return null; + }, + ), + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + rateLimiter: limiter, + consumerName: 'group-fail-closed-worker', + concurrency: 1, + prefetchMultiplier: 1, + retryStrategy: const _FixedRetryStrategy( + Duration(milliseconds: 120), + ), + ); + + await worker.start(); + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue( + 'tasks.group.failclosed', + headers: const {'tenant': 'acme'}, + ); + + await Future.delayed(const Duration(milliseconds: 220)); + final status = await backend.get(taskId); + expect(status?.state, TaskState.retried); + expect(executed, equals(0)); + + await worker.shutdown(); + broker.dispose(); + }, + ); + + test('queue pause persists across restarts until resumed', () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + final backend = InMemoryResultBackend(); + final revokeStore = InMemoryRevokeStore(); + final registry = SimpleTaskRegistry()..register(_SuccessTask()); + + final workerA = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'pause-worker-a', + concurrency: 1, + prefetchMultiplier: 1, + revokeStore: revokeStore, + ); + await workerA.start(); + + final pauseReply = await _sendControlCommand( + broker: broker, + namespace: workerA.namespace, + queue: ControlQueueNames.worker( + workerA.namespace, + workerA.consumerName!, + ), + type: 'queue_pause', + payload: const { + 'queues': ['default'], + }, + ); + expect(pauseReply.status, equals('ok')); + await workerA.shutdown(); + + final workerB = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'pause-worker-b', + concurrency: 1, + prefetchMultiplier: 1, + revokeStore: revokeStore, + ); + final events = []; + final sub = workerB.events.listen(events.add); + await workerB.start(); + + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue('tasks.success'); + await Future.delayed(const Duration(milliseconds: 180)); + final pausedStatus = await backend.get(taskId); + expect(pausedStatus?.state, TaskState.queued); + + final resumeReply = await _sendControlCommand( + broker: broker, + namespace: workerB.namespace, + queue: ControlQueueNames.worker( + workerB.namespace, + workerB.consumerName!, + ), + type: 'queue_resume', + payload: const { + 'queues': ['default'], + }, + ); + expect(resumeReply.status, equals('ok')); + + await _waitFor( + () => events.any( + (event) => + event.type == WorkerEventType.completed && + event.envelope?.id == taskId, + ), + timeout: const Duration(seconds: 3), + ); + expect((await backend.get(taskId))?.state, TaskState.succeeded); + + await sub.cancel(); + await workerB.shutdown(); + broker.dispose(); + }); + test('emits control command signals', () async { StemSignals.configure(configuration: const StemSignalConfiguration()); @@ -1452,6 +1729,97 @@ Future _waitForCallbackSuccess( } } +Future _waitForTaskState( + ResultBackend backend, + String taskId, + TaskState expected, { + Duration timeout = const Duration(seconds: 3), +}) async { + final deadline = DateTime.now().add(timeout); + while (true) { + final status = await backend.get(taskId); + if (status?.state == expected) { + return; + } + if (DateTime.now().isAfter(deadline)) { + throw TimeoutException( + 'Task $taskId did not reach state ${expected.name}', + ); + } + await Future.delayed(const Duration(milliseconds: 20)); + } +} + +Future _sendControlCommand({ + required Broker broker, + required String namespace, + required String queue, + required String type, + Map payload = const {}, +}) async { + final requestId = generateEnvelopeId(); + final replyQueue = ControlQueueNames.reply(namespace, requestId); + final completer = Completer(); + + late final StreamSubscription subscription; + subscription = broker + .consume( + RoutingSubscription.singleQueue(replyQueue), + prefetch: 1, + consumerName: 'worker-test-control-$requestId', + ) + .listen((delivery) async { + final reply = controlReplyFromEnvelope(delivery.envelope); + await broker.ack(delivery); + if (!completer.isCompleted) { + completer.complete(reply); + } + }); + + final command = ControlCommandMessage( + requestId: requestId, + type: type, + targets: const ['*'], + payload: payload, + ); + await broker.publish(command.toEnvelope(queue: queue)); + try { + return await completer.future.timeout(const Duration(seconds: 2)); + } finally { + await subscription.cancel(); + } +} + +class _ScenarioRateLimiter implements RateLimiter { + _ScenarioRateLimiter(this._decision); + + final RateLimitDecision Function(String key, int attempt) _decision; + final Map _attempts = {}; + final List keys = []; + + @override + Future acquire( + String key, { + int tokens = 1, + Duration? interval, + Map? meta, + }) async { + final attempt = (_attempts[key] ?? 0) + 1; + _attempts[key] = attempt; + keys.add(key); + return _decision(key, attempt); + } +} + +class _FixedRetryStrategy implements RetryStrategy { + const _FixedRetryStrategy(this.delay); + + final Duration delay; + + @override + Duration nextDelay(int attempt, Object error, StackTrace stackTrace) => delay; +} + class _FlakyTask implements TaskHandler { int _attempts = 0; diff --git a/packages/stem_adapter_tests/README.md b/packages/stem_adapter_tests/README.md index 7ef22f56..ca90372b 100644 --- a/packages/stem_adapter_tests/README.md +++ b/packages/stem_adapter_tests/README.md @@ -26,6 +26,11 @@ void main() { factory: ResultBackendContractFactory(create: createBackend), ); + runQueueEventsContractTests( + adapterName: 'my-adapter', + factory: QueueEventsContractFactory(create: createBroker), + ); + final workflowFactory = WorkflowStoreContractFactory( create: createWorkflowStore, ); @@ -64,6 +69,12 @@ all other contract assertions active. | `verifyWorkerHeartbeats` | `true` | Heartbeat CRUD tests | Verifies heartbeat set/get/list/update behavior. | | `verifyHeartbeatExpiry` | `true` | Heartbeat expiry tests | Verifies heartbeat TTL expiration behavior independently from heartbeat CRUD checks. | +### QueueEventsContractCapabilities + +| Flag | Default | Affects | Behavior when enabled | +|---|---|---|---| +| `verifyFanout` | `true` | Multi-listener fan-out tests | Verifies custom queue events reach all active listeners on the same queue scope. | + ### WorkflowStoreContractCapabilities | Flag | Default | Affects | Behavior when enabled | diff --git a/packages/stem_adapter_tests/lib/src/contract_capabilities.dart b/packages/stem_adapter_tests/lib/src/contract_capabilities.dart index 8fa8ccc8..e60de2a2 100644 --- a/packages/stem_adapter_tests/lib/src/contract_capabilities.dart +++ b/packages/stem_adapter_tests/lib/src/contract_capabilities.dart @@ -81,3 +81,25 @@ class LockStoreContractCapabilities { /// Whether lock renewal behavior should be verified. final bool verifyRenewSemantics; } + +/// Feature capability flags for revoke store contract tests. +class RevokeStoreContractCapabilities { + /// Creates revoke store contract capability flags. + const RevokeStoreContractCapabilities({ + this.verifyPruneExpired = true, + }); + + /// Whether expiry pruning behavior should be verified. + final bool verifyPruneExpired; +} + +/// Feature capability flags for queue events contract tests. +class QueueEventsContractCapabilities { + /// Creates queue events contract capability flags. + const QueueEventsContractCapabilities({ + this.verifyFanout = true, + }); + + /// Whether multi-listener fan-out behavior should be verified. + final bool verifyFanout; +} diff --git a/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart b/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart new file mode 100644 index 00000000..69ec9b4b --- /dev/null +++ b/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart @@ -0,0 +1,251 @@ +import 'dart:async'; + +import 'package:stem/stem.dart'; +import 'package:stem_adapter_tests/src/contract_capabilities.dart'; +import 'package:test/test.dart'; + +/// Settings that tune the queue events contract test suite. +class QueueEventsContractSettings { + /// Creates queue events contract settings. + const QueueEventsContractSettings({ + this.settleDelay = const Duration(milliseconds: 150), + this.timeout = const Duration(seconds: 5), + this.capabilities = const QueueEventsContractCapabilities(), + }); + + /// Delay used to let subscriptions become active before assertions. + final Duration settleDelay; + + /// Timeout used when waiting for event deliveries. + final Duration timeout; + + /// Feature capability flags for optional contract assertions. + final QueueEventsContractCapabilities capabilities; +} + +/// Factory hooks used by the queue events contract test suite. +class QueueEventsContractFactory { + /// Creates a queue events contract factory. + const QueueEventsContractFactory({ + required this.create, + this.dispose, + this.additionalBrokerFactory, + this.additionalDispose, + }); + + /// Creates a broker instance used by producers/listeners. + final Future Function() create; + + /// Optional disposer invoked after each test. + final FutureOr Function(Broker broker)? dispose; + + /// Optional second broker factory used for fan-out verification. + final Future Function()? additionalBrokerFactory; + + /// Optional disposer for brokers created by [additionalBrokerFactory]. + final FutureOr Function(Broker broker)? additionalDispose; +} + +/// Runs contract tests for queue custom events/listeners. +void runQueueEventsContractTests({ + required String adapterName, + required QueueEventsContractFactory factory, + QueueEventsContractSettings settings = const QueueEventsContractSettings(), +}) { + group('$adapterName queue events contract', () { + Broker? broker; + + setUp(() async { + broker = await factory.create(); + }); + + tearDown(() async { + final current = broker; + if (current != null && factory.dispose != null) { + await factory.dispose!(current); + } + broker = null; + }); + + test('emits and receives custom queue events', () async { + final current = broker!; + final queueName = _queueName('events'); + const eventName = 'order.created'; + final producer = QueueEventsProducer(broker: current); + final listener = QueueEvents( + broker: current, + queue: queueName, + consumerName: 'listener-${DateTime.now().microsecondsSinceEpoch}', + ); + await listener.start(); + addTearDown(listener.close); + await Future.delayed(settings.settleDelay); + + final next = listener.on(eventName).first.timeout(settings.timeout); + final eventId = await producer.emit( + queueName, + eventName, + payload: const {'id': 'ord-1'}, + ); + + final received = await next; + expect(received.id, eventId); + expect(received.queue, queueName); + expect(received.name, eventName); + expect(received.payload['id'], 'ord-1'); + }); + + test('does not deliver events from other queues', () async { + final current = broker!; + final producer = QueueEventsProducer(broker: current); + final queueA = _queueName('a'); + final queueB = _queueName('b'); + final listener = QueueEvents( + broker: current, + queue: queueA, + consumerName: 'listener-${DateTime.now().microsecondsSinceEpoch}', + ); + await listener.start(); + addTearDown(listener.close); + await Future.delayed(settings.settleDelay); + + final events = []; + final sub = listener.events.listen(events.add); + addTearDown(sub.cancel); + + await producer.emit( + queueB, + 'invoice.created', + payload: const {'id': 'i-1'}, + ); + await Future.delayed(settings.settleDelay); + + expect(events, isEmpty); + }); + + test('on(eventName) only emits matching event names', () async { + final current = broker!; + final producer = QueueEventsProducer(broker: current); + final queueName = _queueName('filter'); + final listener = QueueEvents( + broker: current, + queue: queueName, + consumerName: 'listener-${DateTime.now().microsecondsSinceEpoch}', + ); + await listener.start(); + addTearDown(listener.close); + await Future.delayed(settings.settleDelay); + + final matchFuture = listener + .on('order.completed') + .first + .timeout( + settings.timeout, + ); + + await producer.emit( + queueName, + 'order.created', + payload: const {'id': 'ord-filter-1'}, + ); + await producer.emit( + queueName, + 'order.completed', + payload: const {'id': 'ord-filter-2'}, + ); + + final matched = await matchFuture; + expect(matched.name, 'order.completed'); + expect(matched.payload['id'], 'ord-filter-2'); + }); + + test('preserves headers and metadata through event delivery', () async { + final current = broker!; + final producer = QueueEventsProducer(broker: current); + final queueName = _queueName('headers-meta'); + final listener = QueueEvents( + broker: current, + queue: queueName, + consumerName: 'listener-${DateTime.now().microsecondsSinceEpoch}', + ); + await listener.start(); + addTearDown(listener.close); + await Future.delayed(settings.settleDelay); + + final next = listener.events.first.timeout(settings.timeout); + await producer.emit( + queueName, + 'invoice.settled', + payload: const {'invoiceId': 'inv-1'}, + headers: const {'x-trace-id': 'trace-123'}, + meta: const {'tenant': 'acme'}, + ); + + final received = await next; + expect(received.headers['x-trace-id'], 'trace-123'); + expect(received.meta['tenant'], 'acme'); + expect(received.payload['invoiceId'], 'inv-1'); + }); + + test( + 'fans out queue events to multiple listeners', + () async { + final primary = broker!; + final secondary = factory.additionalBrokerFactory == null + ? primary + : await factory.additionalBrokerFactory!(); + if (!identical(primary, secondary)) { + addTearDown(() async { + if (factory.additionalDispose != null) { + await factory.additionalDispose!(secondary); + } + }); + } + + final producer = QueueEventsProducer(broker: primary); + final queueName = _queueName('fanout'); + const eventName = 'order.updated'; + final listenerA = QueueEvents( + broker: primary, + queue: queueName, + consumerName: 'listener-a-${DateTime.now().microsecondsSinceEpoch}', + ); + final listenerB = QueueEvents( + broker: secondary, + queue: queueName, + consumerName: 'listener-b-${DateTime.now().microsecondsSinceEpoch}', + ); + await listenerA.start(); + await listenerB.start(); + addTearDown(listenerA.close); + addTearDown(listenerB.close); + await Future.delayed(settings.settleDelay); + + final nextA = listenerA.on(eventName).first.timeout(settings.timeout); + final nextB = listenerB.on(eventName).first.timeout(settings.timeout); + + await producer.emit( + queueName, + eventName, + payload: const {'status': 'paid'}, + ); + final received = await Future.wait([nextA, nextB]); + expect(received, hasLength(2)); + expect(received[0].payload['status'], 'paid'); + expect(received[1].payload['status'], 'paid'); + }, + skip: _skipUnless( + settings.capabilities.verifyFanout, + 'Adapter disabled queue-event fanout capability checks.', + ), + ); + }); +} + +String _queueName(String suffix) => + 'contract-queue-events-$suffix-' + '${DateTime.now().microsecondsSinceEpoch}-${_counter++}'; + +int _counter = 0; + +Object _skipUnless(bool enabled, String reason) => enabled ? false : reason; diff --git a/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart b/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart new file mode 100644 index 00000000..22979765 --- /dev/null +++ b/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart @@ -0,0 +1,194 @@ +import 'dart:async'; + +import 'package:stem/stem.dart'; +import 'package:stem_adapter_tests/src/contract_capabilities.dart'; +import 'package:test/test.dart'; + +/// Settings that tune the revoke store contract test suite. +class RevokeStoreContractSettings { + /// Creates revoke store contract settings. + const RevokeStoreContractSettings({ + this.capabilities = const RevokeStoreContractCapabilities(), + }); + + /// Feature capability flags for optional contract assertions. + final RevokeStoreContractCapabilities capabilities; +} + +/// Factory hooks used by the revoke store contract test suite. +class RevokeStoreContractFactory { + /// Creates a revoke store contract factory. + const RevokeStoreContractFactory({required this.create, this.dispose}); + + /// Creates a fresh revoke store instance for each test case. + final Future Function() create; + + /// Optional disposer invoked after each test. + final FutureOr Function(RevokeStore store)? dispose; +} + +/// Runs contract tests covering the required RevokeStore semantics. +void runRevokeStoreContractTests({ + required String adapterName, + required RevokeStoreContractFactory factory, + RevokeStoreContractSettings settings = const RevokeStoreContractSettings(), +}) { + group('$adapterName revoke store contract', () { + RevokeStore? store; + + setUp(() async { + store = await factory.create(); + }); + + tearDown(() async { + final current = store; + if (current != null && factory.dispose != null) { + await factory.dispose!(current); + } + store = null; + }); + + test( + 'upsert/list stores entries by namespace and version ordering', + () async { + final current = store!; + final now = DateTime.utc(2026, 2, 24, 10, 45); + const namespace = 'stem-contract'; + final entries = [ + RevokeEntry( + namespace: namespace, + taskId: 'task-2', + version: 2, + issuedAt: now, + ), + RevokeEntry( + namespace: namespace, + taskId: 'task-1', + version: 1, + issuedAt: now, + ), + ]; + + final applied = await current.upsertAll(entries); + expect( + applied.map((entry) => entry.taskId), + containsAll(['task-1', 'task-2']), + ); + + final listed = await current.list(namespace); + expect(listed.map((entry) => entry.version).toList(), equals([1, 2])); + expect(listed.every((entry) => entry.namespace == namespace), isTrue); + }, + ); + + test('upsert preserves newer entries and ignores stale versions', () async { + final current = store!; + final now = DateTime.utc(2026, 2, 24, 11); + const namespace = 'stem-contract'; + const taskId = 'task-stale'; + final first = RevokeEntry( + namespace: namespace, + taskId: taskId, + version: 10, + issuedAt: now, + reason: 'newer', + ); + final stale = RevokeEntry( + namespace: namespace, + taskId: taskId, + version: 9, + issuedAt: now.subtract(const Duration(minutes: 1)), + reason: 'stale', + ); + + await current.upsertAll([first]); + final applied = await current.upsertAll([stale]); + expect(applied.single.version, 10); + expect(applied.single.reason, 'newer'); + + final listed = await current.list(namespace); + expect(listed.single.version, 10); + expect(listed.single.reason, 'newer'); + }); + + test('upsert replaces entries when a higher version is provided', () async { + final current = store!; + final now = DateTime.utc(2026, 2, 24, 11, 5); + const namespace = 'stem-contract'; + const taskId = 'task-upgrade'; + await current.upsertAll([ + RevokeEntry( + namespace: namespace, + taskId: taskId, + version: 4, + issuedAt: now, + ), + ]); + + final applied = await current.upsertAll([ + RevokeEntry( + namespace: namespace, + taskId: taskId, + version: 5, + issuedAt: now.add(const Duration(seconds: 30)), + terminate: true, + reason: 'superseded', + ), + ]); + + expect(applied.single.version, 5); + expect(applied.single.terminate, isTrue); + expect(applied.single.reason, 'superseded'); + }); + + test( + 'pruneExpired removes expired records only within target namespace', + () async { + final current = store!; + final now = DateTime.utc(2026, 2, 24, 11, 10); + const namespaceA = 'stem-contract-a'; + const namespaceB = 'stem-contract-b'; + await current.upsertAll([ + RevokeEntry( + namespace: namespaceA, + taskId: 'expired', + version: 1, + issuedAt: now.subtract(const Duration(minutes: 2)), + expiresAt: now.subtract(const Duration(seconds: 1)), + ), + RevokeEntry( + namespace: namespaceA, + taskId: 'active', + version: 2, + issuedAt: now, + expiresAt: now.add(const Duration(minutes: 5)), + ), + RevokeEntry( + namespace: namespaceB, + taskId: 'other-namespace', + version: 3, + issuedAt: now.subtract(const Duration(minutes: 1)), + expiresAt: now.subtract(const Duration(seconds: 1)), + ), + ]); + + final pruned = await current.pruneExpired(namespaceA, now); + expect(pruned, 1); + + final listedA = await current.list(namespaceA); + expect(listedA.map((entry) => entry.taskId), equals(['active'])); + final listedB = await current.list(namespaceB); + expect( + listedB.map((entry) => entry.taskId), + equals(['other-namespace']), + ); + }, + skip: _skipUnless( + settings.capabilities.verifyPruneExpired, + 'Adapter disabled pruneExpired capability checks.', + ), + ); + }); +} + +Object _skipUnless(bool enabled, String reason) => enabled ? false : reason; diff --git a/packages/stem_adapter_tests/lib/stem_adapter_tests.dart b/packages/stem_adapter_tests/lib/stem_adapter_tests.dart index 30e88772..db47f0fc 100644 --- a/packages/stem_adapter_tests/lib/stem_adapter_tests.dart +++ b/packages/stem_adapter_tests/lib/stem_adapter_tests.dart @@ -1,6 +1,8 @@ export 'src/broker_contract_suite.dart'; export 'src/contract_capabilities.dart'; export 'src/lock_store_contract_suite.dart'; +export 'src/queue_events_contract_suite.dart'; export 'src/result_backend_contract_suite.dart'; +export 'src/revoke_store_contract_suite.dart'; export 'src/workflow_script_facade_suite.dart'; export 'src/workflow_store_contract_suite.dart'; diff --git a/packages/stem_adapter_tests/test/contract_suite_exports_test.dart b/packages/stem_adapter_tests/test/contract_suite_exports_test.dart index 78744fd4..ab99942d 100644 --- a/packages/stem_adapter_tests/test/contract_suite_exports_test.dart +++ b/packages/stem_adapter_tests/test/contract_suite_exports_test.dart @@ -6,6 +6,8 @@ void main() { expect(runBrokerContractTests, isNotNull); expect(runResultBackendContractTests, isNotNull); expect(runLockStoreContractTests, isNotNull); + expect(runQueueEventsContractTests, isNotNull); + expect(runRevokeStoreContractTests, isNotNull); expect(runWorkflowStoreContractTests, isNotNull); }); } diff --git a/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart b/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart new file mode 100644 index 00000000..42c3b829 --- /dev/null +++ b/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart @@ -0,0 +1,16 @@ +import 'package:stem/stem.dart'; +import 'package:stem_adapter_tests/stem_adapter_tests.dart'; + +void main() { + final namespace = 'adapter-tests-${DateTime.now().microsecondsSinceEpoch}'; + + runQueueEventsContractTests( + adapterName: 'in-memory', + factory: QueueEventsContractFactory( + create: () async => InMemoryBroker(namespace: namespace), + dispose: (broker) => broker.close(), + additionalBrokerFactory: () async => InMemoryBroker(namespace: namespace), + additionalDispose: (broker) => broker.close(), + ), + ); +} diff --git a/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart b/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart new file mode 100644 index 00000000..6e082c5e --- /dev/null +++ b/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart @@ -0,0 +1,12 @@ +import 'package:stem/stem.dart'; +import 'package:stem_adapter_tests/stem_adapter_tests.dart'; + +void main() { + runRevokeStoreContractTests( + adapterName: 'in-memory', + factory: RevokeStoreContractFactory( + create: () async => InMemoryRevokeStore(), + dispose: (store) => store.close(), + ), + ); +} diff --git a/packages/stem_cli/lib/src/cli/revoke_store_factory.dart b/packages/stem_cli/lib/src/cli/revoke_store_factory.dart index 5f44682a..cb8f5b3e 100644 --- a/packages/stem_cli/lib/src/cli/revoke_store_factory.dart +++ b/packages/stem_cli/lib/src/cli/revoke_store_factory.dart @@ -4,6 +4,7 @@ import 'package:stem/stem.dart'; // import 'package:stem_cloud_worker/stem_cloud_worker.dart'; import 'package:stem_postgres/stem_postgres.dart'; import 'package:stem_redis/stem_redis.dart'; +import 'package:stem_sqlite/stem_sqlite.dart'; /// Creates a [RevokeStore] based on configuration and URL overrides. class RevokeStoreFactory { @@ -43,6 +44,8 @@ class RevokeStoreFactory { namespace: namespace, tls: config.tls, ); + case 'sqlite': + return SqliteRevokeStore.connect(candidate, namespace: namespace); case 'postgres': case 'postgresql': case 'postgres+ssl': diff --git a/packages/stem_cli/lib/src/cli/worker.dart b/packages/stem_cli/lib/src/cli/worker.dart index 54c287d6..280290d1 100644 --- a/packages/stem_cli/lib/src/cli/worker.dart +++ b/packages/stem_cli/lib/src/cli/worker.dart @@ -20,6 +20,8 @@ class WorkerCommand extends Command { addSubcommand(WorkerStatsCommand(dependencies)); addSubcommand(WorkerStatusCommand(dependencies)); addSubcommand(WorkerShutdownCommand(dependencies)); + addSubcommand(WorkerPauseCommand(dependencies)); + addSubcommand(WorkerResumeCommand(dependencies)); addSubcommand(WorkerHealthcheckCommand(dependencies)); addSubcommand(WorkerDiagnoseCommand(dependencies)); addSubcommand(WorkerMultiCommand(dependencies)); @@ -927,6 +929,202 @@ class WorkerShutdownCommand extends Command { } } +abstract class _WorkerQueueControlCommand extends Command { + _WorkerQueueControlCommand( + this.dependencies, { + required this.commandType, + required this.commandName, + required this.commandDescription, + required this.defaultReason, + }) { + argParser + ..addMultiOption( + 'queue', + abbr: 'q', + help: 'Queue name to control (repeatable).', + valueHelp: 'queue', + ) + ..addMultiOption( + 'worker', + abbr: 'w', + help: 'Target worker identifier (repeatable).', + ) + ..addOption( + 'namespace', + defaultsTo: 'stem', + help: 'Control namespace used for worker identifiers.', + ) + ..addOption( + 'timeout', + defaultsTo: '5s', + help: 'Wait duration for replies (e.g. 3s, 1m).', + valueHelp: 'duration', + ) + ..addOption( + 'reason', + help: 'Optional human-readable reason for audit logs.', + ) + ..addOption( + 'requester', + help: 'Requester identifier (defaults to stem-cli).', + ) + ..addFlag( + 'json', + defaultsTo: false, + negatable: false, + help: 'Emit replies as JSON instead of text.', + ); + } + + final StemCommandDependencies dependencies; + final String commandType; + final String commandName; + final String commandDescription; + final String defaultReason; + + @override + String get name => commandName; + + @override + String get description => commandDescription; + + @override + Future run() async { + final args = argResults!; + final queues = + ((args['queue'] as List?) ?? const []) + .cast() + .map((value) => value.trim()) + .where((value) => value.isNotEmpty) + .toSet() + .toList() + ..sort(); + if (queues.isEmpty) { + dependencies.err.writeln('At least one --queue must be provided.'); + return 64; + } + + final namespaceInput = (args['namespace'] as String?)?.trim(); + final namespace = namespaceInput == null || namespaceInput.isEmpty + ? 'stem' + : namespaceInput; + final timeout = + ObservabilityConfig.parseDuration(args['timeout'] as String?) ?? + const Duration(seconds: 5); + final requester = (args['requester'] as String?)?.trim(); + final reason = (args['reason'] as String?)?.trim() ?? defaultReason; + final jsonOutput = args['json'] as bool? ?? false; + final targets = ((args['worker'] as List?) ?? const []) + .cast() + .map((value) => value.trim()) + .where((value) => value.isNotEmpty) + .toSet(); + + late CliContext ctx; + try { + ctx = await dependencies.createCliContext(); + } catch (error, stack) { + dependencies.err.writeln('Failed to initialize Stem context: $error'); + dependencies.err.writeln(stack); + return 70; + } + + try { + final requestId = generateEnvelopeId(); + final command = ControlCommandMessage( + requestId: requestId, + type: commandType, + targets: targets.isEmpty ? const ['*'] : targets.toList(), + timeoutMs: timeout.inMilliseconds, + payload: { + 'namespace': namespace, + 'queues': queues, + 'reason': reason, + if (requester != null && requester.isNotEmpty) 'requester': requester, + }, + ); + + await _publishControlCommand( + ctx, + namespace: namespace, + targets: targets, + command: command, + ); + + final replies = await _collectControlReplies( + ctx, + namespace: namespace, + requestId: requestId, + expectedWorkers: targets.isEmpty ? null : targets.length, + timeout: timeout, + ); + + if (jsonOutput) { + dependencies.out.writeln( + jsonEncode(replies.map((reply) => reply.toMap()).toList()), + ); + } else { + if (targets.isNotEmpty) { + final missing = targets.difference( + replies.map((reply) => reply.workerId).toSet(), + ); + if (missing.isNotEmpty) { + dependencies.err.writeln('No reply from: ${missing.join(', ')}'); + } + } + if (replies.isEmpty) { + dependencies.out.writeln( + 'No replies received within ${timeout.inMilliseconds}ms.', + ); + return 70; + } + dependencies.out.writeln('Worker | Status | Updated | Paused'); + dependencies.out.writeln( + '--------------+--------+---------+----------------', + ); + final ordered = [...replies] + ..sort((a, b) => a.workerId.compareTo(b.workerId)); + for (final reply in ordered) { + final updated = (reply.payload['updated'] ?? '-').toString(); + final paused = (reply.payload['paused'] as List?)?.join(', ') ?? '-'; + dependencies.out.writeln( + '${reply.workerId.padRight(14)}| ' + '${reply.status.padRight(6)} | ' + '${updated.padRight(7)} | ' + '$paused', + ); + } + } + + return replies.isEmpty ? 70 : 0; + } finally { + await ctx.dispose(); + } + } +} + +class WorkerPauseCommand extends _WorkerQueueControlCommand { + WorkerPauseCommand(StemCommandDependencies dependencies) + : super( + dependencies, + commandType: 'queue_pause', + commandName: 'pause', + commandDescription: 'Pause one or more queues for targeted workers.', + defaultReason: 'queue paused by stem-cli', + ); +} + +class WorkerResumeCommand extends _WorkerQueueControlCommand { + WorkerResumeCommand(StemCommandDependencies dependencies) + : super( + dependencies, + commandType: 'queue_resume', + commandName: 'resume', + commandDescription: 'Resume one or more paused queues.', + defaultReason: 'queue resumed by stem-cli', + ); +} + class WorkerStatusCommand extends Command { WorkerStatusCommand(this.dependencies) { argParser diff --git a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart index 0cb4bd2c..9757559f 100644 --- a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart @@ -108,6 +108,72 @@ void main() { broker.dispose(); }); + test('pause/resume commands stop and restart queue consumption', () async { + final broker = InMemoryBroker(); + final backend = InMemoryResultBackend(); + final revokeStore = InMemoryRevokeStore(); + final registry = SimpleTaskRegistry()..register(_FastTask()); + + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + queue: 'default', + consumerName: 'worker-pause', + concurrency: 1, + prefetchMultiplier: 1, + revokeStore: revokeStore, + ); + await worker.start(); + + final pauseOut = StringBuffer(); + final pauseErr = StringBuffer(); + final pauseCode = await runStemCli( + ['worker', 'pause', '--worker', 'worker-pause', '--queue', 'default'], + out: pauseOut, + err: pauseErr, + contextBuilder: () async => CliContext( + broker: broker, + backend: backend, + revokeStore: revokeStore, + routing: RoutingRegistry(RoutingConfig.legacy()), + dispose: () async {}, + registry: registry, + ), + ); + expect(pauseCode, equals(0)); + + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue('tasks.fast'); + await Future.delayed(const Duration(milliseconds: 180)); + final pausedStatus = await backend.get(taskId); + expect(pausedStatus?.state, TaskState.queued); + + final resumeOut = StringBuffer(); + final resumeErr = StringBuffer(); + final resumeCode = await runStemCli( + ['worker', 'resume', '--worker', 'worker-pause', '--queue', 'default'], + out: resumeOut, + err: resumeErr, + contextBuilder: () async => CliContext( + broker: broker, + backend: backend, + revokeStore: revokeStore, + routing: RoutingRegistry(RoutingConfig.legacy()), + dispose: () async {}, + registry: registry, + ), + ); + expect(resumeCode, equals(0)); + + await _waitFor( + () async => (await backend.get(taskId))?.state == TaskState.succeeded, + ); + + await worker.shutdown(); + broker.dispose(); + }); + test('includes active task metadata', () async { final broker = InMemoryBroker(); final backend = InMemoryResultBackend(); @@ -323,7 +389,7 @@ void main() { expect(code, equals(0)); await _waitFor( - () => events.any( + () async => events.any( (event) => event.type == WorkerEventType.revoked && event.envelope?.id == taskId, @@ -395,15 +461,32 @@ class _LoopingTask implements TaskHandler { } Future _waitFor( - bool Function() predicate, { + Future Function() predicate, { Duration timeout = const Duration(seconds: 2), Duration pollInterval = const Duration(milliseconds: 10), }) async { final deadline = DateTime.now().add(timeout); - while (!predicate()) { + while (!await predicate()) { if (DateTime.now().isAfter(deadline)) { throw TimeoutException('Condition not met within $timeout'); } await Future.delayed(pollInterval); } } + +class _FastTask implements TaskHandler { + @override + String get name => 'tasks.fast'; + + @override + TaskOptions get options => const TaskOptions(maxRetries: 0); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async {} +} diff --git a/packages/stem_cli/test/unit/cli/revoke_store_factory_test.dart b/packages/stem_cli/test/unit/cli/revoke_store_factory_test.dart index 3590803f..9725f5af 100644 --- a/packages/stem_cli/test/unit/cli/revoke_store_factory_test.dart +++ b/packages/stem_cli/test/unit/cli/revoke_store_factory_test.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + import 'package:stem/stem.dart' hide RevokeStoreFactory; import 'package:stem_cli/src/cli/revoke_store_factory.dart'; // import 'package:stem_cloud_worker/stem_cloud_worker.dart'; @@ -19,5 +21,43 @@ void main() { // expect(store, isA()); await store.close(); }, skip: true); + + test('supports sqlite revoke store URLs', () async { + final tempDir = await Directory.systemTemp.createTemp( + 'stem_cli_revoke_sqlite', + ); + final dbPath = '${tempDir.path}/revokes.db'; + final config = StemConfig( + brokerUrl: 'redis://localhost:6379', + revokeStoreUrl: 'sqlite://$dbPath', + ); + + final store = await RevokeStoreFactory.create(config: config); + try { + expect(store.runtimeType.toString(), contains('SqliteRevokeStore')); + } finally { + await store.close(); + await tempDir.delete(recursive: true); + } + }); + + test('falls back to sqlite result backend URL for revoke store', () async { + final tempDir = await Directory.systemTemp.createTemp( + 'stem_cli_revoke_sqlite_fallback', + ); + final dbPath = '${tempDir.path}/fallback.db'; + final config = StemConfig( + brokerUrl: 'redis://localhost:6379', + resultBackendUrl: 'sqlite://$dbPath', + ); + + final store = await RevokeStoreFactory.create(config: config); + try { + expect(store.runtimeType.toString(), contains('SqliteRevokeStore')); + } finally { + await store.close(); + await tempDir.delete(recursive: true); + } + }); }); } diff --git a/packages/stem_postgres/test/integration/brokers/postgres_broker_integration_test.dart b/packages/stem_postgres/test/integration/brokers/postgres_broker_integration_test.dart index 27d08b5a..d9c1bb36 100644 --- a/packages/stem_postgres/test/integration/brokers/postgres_broker_integration_test.dart +++ b/packages/stem_postgres/test/integration/brokers/postgres_broker_integration_test.dart @@ -56,6 +56,28 @@ Future main() async { ), ); + runQueueEventsContractTests( + adapterName: 'Postgres', + factory: QueueEventsContractFactory( + create: () async => PostgresBroker.fromDataSource( + dataSource, + defaultVisibilityTimeout: const Duration(seconds: 1), + pollInterval: const Duration(milliseconds: 50), + sweeperInterval: const Duration(milliseconds: 200), + runMigrations: false, + ), + dispose: (broker) => (broker as PostgresBroker).close(), + additionalBrokerFactory: () async => PostgresBroker.fromDataSource( + dataSource, + defaultVisibilityTimeout: const Duration(seconds: 1), + pollInterval: const Duration(milliseconds: 50), + sweeperInterval: const Duration(milliseconds: 200), + runMigrations: false, + ), + additionalDispose: (broker) => (broker as PostgresBroker).close(), + ), + ); + test('namespace isolates queue data', () async { final namespaceA = 'broker-ns-a-${DateTime.now().microsecondsSinceEpoch}'; final namespaceB = 'broker-ns-b-${DateTime.now().microsecondsSinceEpoch}'; diff --git a/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart b/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart index fb913c3b..40dbd7b6 100644 --- a/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart +++ b/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart @@ -67,6 +67,45 @@ void main() { ), ); + runQueueEventsContractTests( + adapterName: 'Redis', + factory: QueueEventsContractFactory( + create: () async { + final namespace = _uniqueNamespace(); + contractNamespace = namespace; + return RedisStreamsBroker.connect( + redisUrl, + namespace: namespace, + defaultVisibilityTimeout: const Duration(seconds: 1), + claimInterval: const Duration(milliseconds: 200), + blockTime: const Duration(milliseconds: 100), + ); + }, + dispose: (broker) async { + if (broker is _NoCloseBroker) { + return; + } + await _safeCloseRedisBroker(broker as RedisStreamsBroker); + }, + additionalBrokerFactory: () async { + final namespace = contractNamespace; + if (namespace == null || namespace.isEmpty) { + throw StateError( + 'Redis queue-events contract requires primary broker namespace.', + ); + } + final broker = await RedisStreamsBroker.connect( + redisUrl, + namespace: namespace, + defaultVisibilityTimeout: const Duration(seconds: 1), + claimInterval: const Duration(milliseconds: 200), + blockTime: const Duration(milliseconds: 100), + ); + return _NoCloseBroker(broker); + }, + ), + ); + test('namespace isolates queue data', () async { final namespaceA = _uniqueNamespace(); final namespaceB = _uniqueNamespace(); diff --git a/packages/stem_sqlite/lib/orm_registry.g.dart b/packages/stem_sqlite/lib/orm_registry.g.dart index 30554f75..9c991f6f 100644 --- a/packages/stem_sqlite/lib/orm_registry.g.dart +++ b/packages/stem_sqlite/lib/orm_registry.g.dart @@ -5,6 +5,7 @@ import 'src/models/stem_dead_letter.dart'; import 'src/models/stem_group.dart'; import 'src/models/stem_group_result.dart'; import 'src/models/stem_queue_job.dart'; +import 'src/models/stem_revoke_entry.dart'; import 'src/models/stem_task_result.dart'; import 'src/models/stem_worker_heartbeat.dart'; import 'src/models/stem_workflow_run.dart'; @@ -16,6 +17,7 @@ final List> _$ormModelDefinitions = [ StemGroupOrmDefinition.definition, StemGroupResultOrmDefinition.definition, StemQueueJobOrmDefinition.definition, + StemRevokeEntryOrmDefinition.definition, StemTaskResultOrmDefinition.definition, StemWorkerHeartbeatOrmDefinition.definition, StemWorkflowRunOrmDefinition.definition, @@ -29,11 +31,12 @@ ModelRegistry buildOrmRegistry() => ModelRegistry() ..registerTypeAlias(_$ormModelDefinitions[1]) ..registerTypeAlias(_$ormModelDefinitions[2]) ..registerTypeAlias(_$ormModelDefinitions[3]) - ..registerTypeAlias(_$ormModelDefinitions[4]) - ..registerTypeAlias(_$ormModelDefinitions[5]) - ..registerTypeAlias(_$ormModelDefinitions[6]) - ..registerTypeAlias(_$ormModelDefinitions[7]) - ..registerTypeAlias(_$ormModelDefinitions[8]); + ..registerTypeAlias(_$ormModelDefinitions[4]) + ..registerTypeAlias(_$ormModelDefinitions[5]) + ..registerTypeAlias(_$ormModelDefinitions[6]) + ..registerTypeAlias(_$ormModelDefinitions[7]) + ..registerTypeAlias(_$ormModelDefinitions[8]) + ..registerTypeAlias(_$ormModelDefinitions[9]); List> get generatedOrmModelDefinitions => List.unmodifiable(_$ormModelDefinitions); @@ -45,11 +48,12 @@ extension GeneratedOrmModels on ModelRegistry { registerTypeAlias(_$ormModelDefinitions[1]); registerTypeAlias(_$ormModelDefinitions[2]); registerTypeAlias(_$ormModelDefinitions[3]); - registerTypeAlias(_$ormModelDefinitions[4]); - registerTypeAlias(_$ormModelDefinitions[5]); - registerTypeAlias(_$ormModelDefinitions[6]); - registerTypeAlias(_$ormModelDefinitions[7]); - registerTypeAlias(_$ormModelDefinitions[8]); + registerTypeAlias(_$ormModelDefinitions[4]); + registerTypeAlias(_$ormModelDefinitions[5]); + registerTypeAlias(_$ormModelDefinitions[6]); + registerTypeAlias(_$ormModelDefinitions[7]); + registerTypeAlias(_$ormModelDefinitions[8]); + registerTypeAlias(_$ormModelDefinitions[9]); return this; } } diff --git a/packages/stem_sqlite/lib/src/control/sqlite_revoke_store.dart b/packages/stem_sqlite/lib/src/control/sqlite_revoke_store.dart new file mode 100644 index 00000000..40fad005 --- /dev/null +++ b/packages/stem_sqlite/lib/src/control/sqlite_revoke_store.dart @@ -0,0 +1,168 @@ +import 'dart:io'; + +import 'package:ormed/ormed.dart'; +import 'package:stem/stem.dart'; + +import 'package:stem_sqlite/src/connection.dart'; +import 'package:stem_sqlite/src/models/models.dart'; + +/// SQLite-backed implementation of [RevokeStore]. +class SqliteRevokeStore implements RevokeStore { + SqliteRevokeStore._(this._connections, {required this.namespace}); + + /// Creates a revoke store using an existing [DataSource]. + /// + /// The caller remains responsible for disposing the [DataSource]. + static Future fromDataSource( + DataSource dataSource, { + String namespace = 'stem', + }) async { + final resolvedNamespace = namespace.trim().isEmpty + ? 'stem' + : namespace.trim(); + final connections = await SqliteConnections.openWithDataSource(dataSource); + return SqliteRevokeStore._(connections, namespace: resolvedNamespace); + } + + /// Opens a SQLite revoke store from an existing database [file]. + static Future open( + File file, { + String namespace = 'stem', + }) async { + final resolvedNamespace = namespace.trim().isEmpty + ? 'stem' + : namespace.trim(); + final connections = await SqliteConnections.open(file); + return SqliteRevokeStore._(connections, namespace: resolvedNamespace); + } + + /// Connects to SQLite via a connection string. + /// + /// Accepts `sqlite:///path/to/db.sqlite`, `file:///path/to/db.sqlite`, and + /// direct file paths. + static Future connect( + String connectionString, { + String namespace = 'stem', + }) async { + final uri = Uri.parse(connectionString); + late final File file; + switch (uri.scheme) { + case 'sqlite': + final path = uri.path.isNotEmpty ? uri.path : uri.host; + if (path.isEmpty) { + throw StateError( + 'SQLite URL must include a file path ' + '(e.g. sqlite:///tmp/stem.db).', + ); + } + file = File(path); + case 'file': + file = File(uri.toFilePath()); + case '': + file = File(connectionString); + default: + throw StateError( + 'Unsupported sqlite revoke store scheme: ${uri.scheme}', + ); + } + return open(file, namespace: namespace); + } + + final SqliteConnections _connections; + + /// Namespace used when incoming entries omit namespace. + final String namespace; + + @override + Future close() => _connections.close(); + + @override + Future> list(String namespace) async { + final rows = await _connections.context + .query() + .whereEquals('namespace', namespace) + .orderBy('version') + .get(); + return rows.map(_toRevokeEntry).toList(growable: false); + } + + @override + Future pruneExpired(String namespace, DateTime clock) async { + return _connections.runInTransaction((txn) async { + final expired = await txn + .query() + .whereEquals('namespace', namespace) + .whereNotNull('expiresAt') + .where('expiresAt', clock, PredicateOperator.lessThanOrEqual) + .get(); + if (expired.isEmpty) { + return 0; + } + + for (final row in expired) { + await txn.repository().delete( + StemRevokeEntryPartial(namespace: row.namespace, taskId: row.taskId), + ); + } + return expired.length; + }); + } + + @override + Future> upsertAll(List entries) async { + if (entries.isEmpty) { + return const []; + } + return _connections.runInTransaction((txn) async { + final applied = []; + for (final entry in entries) { + final targetNamespace = entry.namespace.trim().isEmpty + ? namespace + : entry.namespace; + final existing = await txn + .query() + .whereEquals('namespace', targetNamespace) + .whereEquals('taskId', entry.taskId) + .firstOrNull(); + + if (existing == null || entry.version > existing.version) { + final model = StemRevokeEntry( + namespace: targetNamespace, + taskId: entry.taskId, + version: entry.version, + issuedAt: entry.issuedAt, + terminate: entry.terminate ? 1 : 0, + reason: entry.reason, + requestedBy: entry.requestedBy, + expiresAt: entry.expiresAt, + ); + await txn.repository().upsert( + model, + uniqueBy: ['namespace', 'taskId'], + ); + applied.add( + entry.copyWith( + namespace: targetNamespace, + ), + ); + } else { + applied.add(_toRevokeEntry(existing)); + } + } + return applied; + }); + } + + RevokeEntry _toRevokeEntry(StemRevokeEntry row) { + return RevokeEntry( + namespace: row.namespace, + taskId: row.taskId, + version: row.version, + issuedAt: row.issuedAt, + terminate: row.terminate == 1, + reason: row.reason, + requestedBy: row.requestedBy, + expiresAt: row.expiresAt, + ); + } +} diff --git a/packages/stem_sqlite/lib/src/database/migrations.dart b/packages/stem_sqlite/lib/src/database/migrations.dart index c15fd642..102ac469 100644 --- a/packages/stem_sqlite/lib/src/database/migrations.dart +++ b/packages/stem_sqlite/lib/src/database/migrations.dart @@ -7,6 +7,7 @@ import 'package:stem_sqlite/src/database/migrations/m_20251222070816_create_stem import 'package:stem_sqlite/src/database/migrations/m_20251231120000_create_workflow_tables.dart'; import 'package:stem_sqlite/src/database/migrations/m_20251231161000_add_namespace_scoping.dart'; import 'package:stem_sqlite/src/database/migrations/m_20260116120000_add_workflow_run_leases.dart'; +import 'package:stem_sqlite/src/database/migrations/m_20260224103000_add_revoke_store.dart'; final List _entries = [ MigrationEntry( @@ -37,6 +38,13 @@ final List _entries = [ ), migration: const AddWorkflowRunLeases(), ), + MigrationEntry( + id: MigrationId( + DateTime.utc(2026, 2, 24, 10, 30), + 'm_20260224103000_add_revoke_store', + ), + migration: const AddRevokeStore(), + ), ]; /// Build migration descriptors sorted by timestamp. diff --git a/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart b/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart new file mode 100644 index 00000000..e2dc8e05 --- /dev/null +++ b/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart @@ -0,0 +1,35 @@ +import 'package:ormed/migrations.dart'; + +/// Adds a revoke store table for worker control state. +class AddRevokeStore extends Migration { + /// Creates the migration. + const AddRevokeStore(); + + @override + void up(SchemaBuilder schema) { + schema.create('stem_revokes', (table) { + table + ..text('namespace') + ..text('task_id') + ..integer('version') + ..timestamp('issued_at'); + table.integer('terminate').defaultValue(0); + table.text('reason').nullable(); + table.text('requested_by').nullable(); + table.timestamp('expires_at').nullable(); + table + ..timestampsTz() + ..primary([ + 'namespace', + 'task_id', + ], name: 'stem_revokes_primary') + ..index(['namespace'], name: 'stem_revokes_namespace_idx') + ..index(['expires_at'], name: 'stem_revokes_expires_at_idx'); + }); + } + + @override + void down(SchemaBuilder schema) { + schema.drop('stem_revokes', ifExists: true); + } +} diff --git a/packages/stem_sqlite/lib/src/models/models.dart b/packages/stem_sqlite/lib/src/models/models.dart index c6563361..28a6c188 100644 --- a/packages/stem_sqlite/lib/src/models/models.dart +++ b/packages/stem_sqlite/lib/src/models/models.dart @@ -2,6 +2,7 @@ export 'stem_dead_letter.dart'; export 'stem_group.dart'; export 'stem_group_result.dart'; export 'stem_queue_job.dart'; +export 'stem_revoke_entry.dart'; export 'stem_task_result.dart'; export 'stem_worker_heartbeat.dart'; export 'stem_workflow_run.dart'; diff --git a/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart b/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart new file mode 100644 index 00000000..3c70903f --- /dev/null +++ b/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart @@ -0,0 +1,47 @@ +import 'package:ormed/ormed.dart'; + +part 'stem_revoke_entry.orm.dart'; + +/// Database model for persisted revoke entries. +@OrmModel(table: 'stem_revokes', primaryKey: ['namespace', 'taskId']) +class StemRevokeEntry extends Model with TimestampsTZ { + /// Creates a revoke entry record. + const StemRevokeEntry({ + required this.namespace, + required this.taskId, + required this.version, + required this.issuedAt, + required this.terminate, + this.reason, + this.requestedBy, + this.expiresAt, + }); + + /// Namespace that owns the revoke record. + final String namespace; + + /// Task identifier for the revoke record. + @OrmField(columnName: 'task_id') + final String taskId; + + /// Monotonic version of the revoke record. + final int version; + + /// Timestamp when the revoke was issued. + @OrmField(columnName: 'issued_at') + final DateTime issuedAt; + + /// Integer flag representing terminate intent (`1` means true). + final int terminate; + + /// Optional human-readable reason. + final String? reason; + + /// Optional caller identity. + @OrmField(columnName: 'requested_by') + final String? requestedBy; + + /// Optional expiration timestamp. + @OrmField(columnName: 'expires_at') + final DateTime? expiresAt; +} diff --git a/packages/stem_sqlite/lib/src/models/stem_revoke_entry.orm.dart b/packages/stem_sqlite/lib/src/models/stem_revoke_entry.orm.dart new file mode 100644 index 00000000..696a75d8 --- /dev/null +++ b/packages/stem_sqlite/lib/src/models/stem_revoke_entry.orm.dart @@ -0,0 +1,996 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND +// dart format width=80 + +part of 'stem_revoke_entry.dart'; + +// ************************************************************************** +// OrmModelGenerator +// ************************************************************************** + +const FieldDefinition _$StemRevokeEntryNamespaceField = FieldDefinition( + name: 'namespace', + columnName: 'namespace', + dartType: 'String', + resolvedType: 'String', + isPrimaryKey: true, + isNullable: false, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryTaskIdField = FieldDefinition( + name: 'taskId', + columnName: 'task_id', + dartType: 'String', + resolvedType: 'String', + isPrimaryKey: true, + isNullable: false, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryVersionField = FieldDefinition( + name: 'version', + columnName: 'version', + dartType: 'int', + resolvedType: 'int', + isPrimaryKey: false, + isNullable: false, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryIssuedAtField = FieldDefinition( + name: 'issuedAt', + columnName: 'issued_at', + dartType: 'DateTime', + resolvedType: 'DateTime', + isPrimaryKey: false, + isNullable: false, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryTerminateField = FieldDefinition( + name: 'terminate', + columnName: 'terminate', + dartType: 'int', + resolvedType: 'int', + isPrimaryKey: false, + isNullable: false, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryReasonField = FieldDefinition( + name: 'reason', + columnName: 'reason', + dartType: 'String', + resolvedType: 'String?', + isPrimaryKey: false, + isNullable: true, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryRequestedByField = FieldDefinition( + name: 'requestedBy', + columnName: 'requested_by', + dartType: 'String', + resolvedType: 'String?', + isPrimaryKey: false, + isNullable: true, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryExpiresAtField = FieldDefinition( + name: 'expiresAt', + columnName: 'expires_at', + dartType: 'DateTime', + resolvedType: 'DateTime?', + isPrimaryKey: false, + isNullable: true, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryCreatedAtField = FieldDefinition( + name: 'createdAt', + columnName: 'created_at', + dartType: 'DateTime', + resolvedType: 'DateTime?', + isPrimaryKey: false, + isNullable: true, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +const FieldDefinition _$StemRevokeEntryUpdatedAtField = FieldDefinition( + name: 'updatedAt', + columnName: 'updated_at', + dartType: 'DateTime', + resolvedType: 'DateTime?', + isPrimaryKey: false, + isNullable: true, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + +Map _encodeStemRevokeEntryUntracked( + Object model, + ValueCodecRegistry registry, +) { + final m = model as StemRevokeEntry; + return { + 'namespace': registry.encodeField( + _$StemRevokeEntryNamespaceField, + m.namespace, + ), + 'task_id': registry.encodeField(_$StemRevokeEntryTaskIdField, m.taskId), + 'version': registry.encodeField(_$StemRevokeEntryVersionField, m.version), + 'issued_at': registry.encodeField( + _$StemRevokeEntryIssuedAtField, + m.issuedAt, + ), + 'terminate': registry.encodeField( + _$StemRevokeEntryTerminateField, + m.terminate, + ), + 'reason': registry.encodeField(_$StemRevokeEntryReasonField, m.reason), + 'requested_by': registry.encodeField( + _$StemRevokeEntryRequestedByField, + m.requestedBy, + ), + 'expires_at': registry.encodeField( + _$StemRevokeEntryExpiresAtField, + m.expiresAt, + ), + }; +} + +final ModelDefinition<$StemRevokeEntry> _$StemRevokeEntryDefinition = + ModelDefinition( + modelName: 'StemRevokeEntry', + tableName: 'stem_revokes', + fields: const [ + _$StemRevokeEntryNamespaceField, + _$StemRevokeEntryTaskIdField, + _$StemRevokeEntryVersionField, + _$StemRevokeEntryIssuedAtField, + _$StemRevokeEntryTerminateField, + _$StemRevokeEntryReasonField, + _$StemRevokeEntryRequestedByField, + _$StemRevokeEntryExpiresAtField, + _$StemRevokeEntryCreatedAtField, + _$StemRevokeEntryUpdatedAtField, + ], + relations: const [], + softDeleteColumn: 'deleted_at', + metadata: ModelAttributesMetadata( + hidden: const [], + visible: const [], + fillable: const [], + guarded: const [], + casts: const {}, + appends: const [], + touches: const [], + timestamps: true, + softDeletes: false, + softDeleteColumn: 'deleted_at', + ), + untrackedToMap: _encodeStemRevokeEntryUntracked, + codec: _$StemRevokeEntryCodec(), + ); + +extension StemRevokeEntryOrmDefinition on StemRevokeEntry { + static ModelDefinition<$StemRevokeEntry> get definition => + _$StemRevokeEntryDefinition; +} + +class StemRevokeEntries { + const StemRevokeEntries._(); + + /// Starts building a query for [$StemRevokeEntry]. + /// + /// {@macro ormed.query} + static Query<$StemRevokeEntry> query([String? connection]) => + Model.query<$StemRevokeEntry>(connection: connection); + + static Future<$StemRevokeEntry?> find(Object id, {String? connection}) => + Model.find<$StemRevokeEntry>(id, connection: connection); + + static Future<$StemRevokeEntry> findOrFail(Object id, {String? connection}) => + Model.findOrFail<$StemRevokeEntry>(id, connection: connection); + + static Future> all({String? connection}) => + Model.all<$StemRevokeEntry>(connection: connection); + + static Future count({String? connection}) => + Model.count<$StemRevokeEntry>(connection: connection); + + static Future anyExist({String? connection}) => + Model.anyExist<$StemRevokeEntry>(connection: connection); + + static Query<$StemRevokeEntry> where( + String column, + String operator, + dynamic value, { + String? connection, + }) => Model.where<$StemRevokeEntry>( + column, + operator, + value, + connection: connection, + ); + + static Query<$StemRevokeEntry> whereIn( + String column, + List values, { + String? connection, + }) => Model.whereIn<$StemRevokeEntry>(column, values, connection: connection); + + static Query<$StemRevokeEntry> orderBy( + String column, { + String direction = "asc", + String? connection, + }) => Model.orderBy<$StemRevokeEntry>( + column, + direction: direction, + connection: connection, + ); + + static Query<$StemRevokeEntry> limit(int count, {String? connection}) => + Model.limit<$StemRevokeEntry>(count, connection: connection); + + /// Creates a [Repository] for [$StemRevokeEntry]. + /// + /// {@macro ormed.repository} + static Repository<$StemRevokeEntry> repo([String? connection]) => + Model.repository<$StemRevokeEntry>(connection: connection); + + /// Builds a tracked model from a column/value map. + static $StemRevokeEntry fromMap( + Map data, { + ValueCodecRegistry? registry, + }) => _$StemRevokeEntryDefinition.fromMap(data, registry: registry); + + /// Converts a tracked model to a column/value map. + static Map toMap( + $StemRevokeEntry model, { + ValueCodecRegistry? registry, + }) => _$StemRevokeEntryDefinition.toMap(model, registry: registry); +} + +class StemRevokeEntryModelFactory { + const StemRevokeEntryModelFactory._(); + + static ModelDefinition<$StemRevokeEntry> get definition => + _$StemRevokeEntryDefinition; + + static ModelCodec<$StemRevokeEntry> get codec => definition.codec; + + static StemRevokeEntry fromMap( + Map data, { + ValueCodecRegistry? registry, + }) => definition.fromMap(data, registry: registry); + + static Map toMap( + StemRevokeEntry model, { + ValueCodecRegistry? registry, + }) => definition.toMap(model.toTracked(), registry: registry); + + static void registerWith(ModelRegistry registry) => + registry.register(definition); + + static ModelFactoryConnection withConnection( + QueryContext context, + ) => ModelFactoryConnection( + definition: definition, + context: context, + ); + + static ModelFactoryBuilder factory({ + GeneratorProvider? generatorProvider, + }) => ModelFactoryRegistry.factoryFor( + generatorProvider: generatorProvider, + ); +} + +class _$StemRevokeEntryCodec extends ModelCodec<$StemRevokeEntry> { + const _$StemRevokeEntryCodec(); + @override + Map encode( + $StemRevokeEntry model, + ValueCodecRegistry registry, + ) { + return { + 'namespace': registry.encodeField( + _$StemRevokeEntryNamespaceField, + model.namespace, + ), + 'task_id': registry.encodeField( + _$StemRevokeEntryTaskIdField, + model.taskId, + ), + 'version': registry.encodeField( + _$StemRevokeEntryVersionField, + model.version, + ), + 'issued_at': registry.encodeField( + _$StemRevokeEntryIssuedAtField, + model.issuedAt, + ), + 'terminate': registry.encodeField( + _$StemRevokeEntryTerminateField, + model.terminate, + ), + 'reason': registry.encodeField( + _$StemRevokeEntryReasonField, + model.reason, + ), + 'requested_by': registry.encodeField( + _$StemRevokeEntryRequestedByField, + model.requestedBy, + ), + 'expires_at': registry.encodeField( + _$StemRevokeEntryExpiresAtField, + model.expiresAt, + ), + if (model.hasAttribute('created_at')) + 'created_at': registry.encodeField( + _$StemRevokeEntryCreatedAtField, + model.getAttribute('created_at'), + ), + if (model.hasAttribute('updated_at')) + 'updated_at': registry.encodeField( + _$StemRevokeEntryUpdatedAtField, + model.getAttribute('updated_at'), + ), + }; + } + + @override + $StemRevokeEntry decode( + Map data, + ValueCodecRegistry registry, + ) { + final String stemRevokeEntryNamespaceValue = + registry.decodeField( + _$StemRevokeEntryNamespaceField, + data['namespace'], + ) ?? + (throw StateError( + 'Field namespace on StemRevokeEntry cannot be null.', + )); + final String stemRevokeEntryTaskIdValue = + registry.decodeField( + _$StemRevokeEntryTaskIdField, + data['task_id'], + ) ?? + (throw StateError('Field taskId on StemRevokeEntry cannot be null.')); + final int stemRevokeEntryVersionValue = + registry.decodeField( + _$StemRevokeEntryVersionField, + data['version'], + ) ?? + (throw StateError('Field version on StemRevokeEntry cannot be null.')); + final DateTime stemRevokeEntryIssuedAtValue = + registry.decodeField( + _$StemRevokeEntryIssuedAtField, + data['issued_at'], + ) ?? + (throw StateError('Field issuedAt on StemRevokeEntry cannot be null.')); + final int stemRevokeEntryTerminateValue = + registry.decodeField( + _$StemRevokeEntryTerminateField, + data['terminate'], + ) ?? + (throw StateError( + 'Field terminate on StemRevokeEntry cannot be null.', + )); + final String? stemRevokeEntryReasonValue = registry.decodeField( + _$StemRevokeEntryReasonField, + data['reason'], + ); + final String? stemRevokeEntryRequestedByValue = registry + .decodeField( + _$StemRevokeEntryRequestedByField, + data['requested_by'], + ); + final DateTime? stemRevokeEntryExpiresAtValue = registry + .decodeField( + _$StemRevokeEntryExpiresAtField, + data['expires_at'], + ); + final DateTime? stemRevokeEntryCreatedAtValue = registry + .decodeField( + _$StemRevokeEntryCreatedAtField, + data['created_at'], + ); + final DateTime? stemRevokeEntryUpdatedAtValue = registry + .decodeField( + _$StemRevokeEntryUpdatedAtField, + data['updated_at'], + ); + final model = $StemRevokeEntry( + namespace: stemRevokeEntryNamespaceValue, + taskId: stemRevokeEntryTaskIdValue, + version: stemRevokeEntryVersionValue, + issuedAt: stemRevokeEntryIssuedAtValue, + terminate: stemRevokeEntryTerminateValue, + reason: stemRevokeEntryReasonValue, + requestedBy: stemRevokeEntryRequestedByValue, + expiresAt: stemRevokeEntryExpiresAtValue, + ); + model._attachOrmRuntimeMetadata({ + 'namespace': stemRevokeEntryNamespaceValue, + 'task_id': stemRevokeEntryTaskIdValue, + 'version': stemRevokeEntryVersionValue, + 'issued_at': stemRevokeEntryIssuedAtValue, + 'terminate': stemRevokeEntryTerminateValue, + 'reason': stemRevokeEntryReasonValue, + 'requested_by': stemRevokeEntryRequestedByValue, + 'expires_at': stemRevokeEntryExpiresAtValue, + if (data.containsKey('created_at')) + 'created_at': stemRevokeEntryCreatedAtValue, + if (data.containsKey('updated_at')) + 'updated_at': stemRevokeEntryUpdatedAtValue, + }); + return model; + } +} + +/// Insert DTO for [StemRevokeEntry]. +/// +/// Auto-increment/DB-generated fields are omitted by default. +class StemRevokeEntryInsertDto implements InsertDto<$StemRevokeEntry> { + const StemRevokeEntryInsertDto({ + this.namespace, + this.taskId, + this.version, + this.issuedAt, + this.terminate, + this.reason, + this.requestedBy, + this.expiresAt, + }); + final String? namespace; + final String? taskId; + final int? version; + final DateTime? issuedAt; + final int? terminate; + final String? reason; + final String? requestedBy; + final DateTime? expiresAt; + + @override + Map toMap() { + return { + if (namespace != null) 'namespace': namespace, + if (taskId != null) 'task_id': taskId, + if (version != null) 'version': version, + if (issuedAt != null) 'issued_at': issuedAt, + if (terminate != null) 'terminate': terminate, + if (reason != null) 'reason': reason, + if (requestedBy != null) 'requested_by': requestedBy, + if (expiresAt != null) 'expires_at': expiresAt, + }; + } + + static const _StemRevokeEntryInsertDtoCopyWithSentinel _copyWithSentinel = + _StemRevokeEntryInsertDtoCopyWithSentinel(); + StemRevokeEntryInsertDto copyWith({ + Object? namespace = _copyWithSentinel, + Object? taskId = _copyWithSentinel, + Object? version = _copyWithSentinel, + Object? issuedAt = _copyWithSentinel, + Object? terminate = _copyWithSentinel, + Object? reason = _copyWithSentinel, + Object? requestedBy = _copyWithSentinel, + Object? expiresAt = _copyWithSentinel, + }) { + return StemRevokeEntryInsertDto( + namespace: identical(namespace, _copyWithSentinel) + ? this.namespace + : namespace as String?, + taskId: identical(taskId, _copyWithSentinel) + ? this.taskId + : taskId as String?, + version: identical(version, _copyWithSentinel) + ? this.version + : version as int?, + issuedAt: identical(issuedAt, _copyWithSentinel) + ? this.issuedAt + : issuedAt as DateTime?, + terminate: identical(terminate, _copyWithSentinel) + ? this.terminate + : terminate as int?, + reason: identical(reason, _copyWithSentinel) + ? this.reason + : reason as String?, + requestedBy: identical(requestedBy, _copyWithSentinel) + ? this.requestedBy + : requestedBy as String?, + expiresAt: identical(expiresAt, _copyWithSentinel) + ? this.expiresAt + : expiresAt as DateTime?, + ); + } +} + +class _StemRevokeEntryInsertDtoCopyWithSentinel { + const _StemRevokeEntryInsertDtoCopyWithSentinel(); +} + +/// Update DTO for [StemRevokeEntry]. +/// +/// All fields are optional; only provided entries are used in SET clauses. +class StemRevokeEntryUpdateDto implements UpdateDto<$StemRevokeEntry> { + const StemRevokeEntryUpdateDto({ + this.namespace, + this.taskId, + this.version, + this.issuedAt, + this.terminate, + this.reason, + this.requestedBy, + this.expiresAt, + }); + final String? namespace; + final String? taskId; + final int? version; + final DateTime? issuedAt; + final int? terminate; + final String? reason; + final String? requestedBy; + final DateTime? expiresAt; + + @override + Map toMap() { + return { + if (namespace != null) 'namespace': namespace, + if (taskId != null) 'task_id': taskId, + if (version != null) 'version': version, + if (issuedAt != null) 'issued_at': issuedAt, + if (terminate != null) 'terminate': terminate, + if (reason != null) 'reason': reason, + if (requestedBy != null) 'requested_by': requestedBy, + if (expiresAt != null) 'expires_at': expiresAt, + }; + } + + static const _StemRevokeEntryUpdateDtoCopyWithSentinel _copyWithSentinel = + _StemRevokeEntryUpdateDtoCopyWithSentinel(); + StemRevokeEntryUpdateDto copyWith({ + Object? namespace = _copyWithSentinel, + Object? taskId = _copyWithSentinel, + Object? version = _copyWithSentinel, + Object? issuedAt = _copyWithSentinel, + Object? terminate = _copyWithSentinel, + Object? reason = _copyWithSentinel, + Object? requestedBy = _copyWithSentinel, + Object? expiresAt = _copyWithSentinel, + }) { + return StemRevokeEntryUpdateDto( + namespace: identical(namespace, _copyWithSentinel) + ? this.namespace + : namespace as String?, + taskId: identical(taskId, _copyWithSentinel) + ? this.taskId + : taskId as String?, + version: identical(version, _copyWithSentinel) + ? this.version + : version as int?, + issuedAt: identical(issuedAt, _copyWithSentinel) + ? this.issuedAt + : issuedAt as DateTime?, + terminate: identical(terminate, _copyWithSentinel) + ? this.terminate + : terminate as int?, + reason: identical(reason, _copyWithSentinel) + ? this.reason + : reason as String?, + requestedBy: identical(requestedBy, _copyWithSentinel) + ? this.requestedBy + : requestedBy as String?, + expiresAt: identical(expiresAt, _copyWithSentinel) + ? this.expiresAt + : expiresAt as DateTime?, + ); + } +} + +class _StemRevokeEntryUpdateDtoCopyWithSentinel { + const _StemRevokeEntryUpdateDtoCopyWithSentinel(); +} + +/// Partial projection for [StemRevokeEntry]. +/// +/// All fields are nullable; intended for subset SELECTs. +class StemRevokeEntryPartial implements PartialEntity<$StemRevokeEntry> { + const StemRevokeEntryPartial({ + this.namespace, + this.taskId, + this.version, + this.issuedAt, + this.terminate, + this.reason, + this.requestedBy, + this.expiresAt, + }); + + /// Creates a partial from a database row map. + /// + /// The [row] keys should be column names (snake_case). + /// Missing columns will result in null field values. + factory StemRevokeEntryPartial.fromRow(Map row) { + return StemRevokeEntryPartial( + namespace: row['namespace'] as String?, + taskId: row['task_id'] as String?, + version: row['version'] as int?, + issuedAt: row['issued_at'] as DateTime?, + terminate: row['terminate'] as int?, + reason: row['reason'] as String?, + requestedBy: row['requested_by'] as String?, + expiresAt: row['expires_at'] as DateTime?, + ); + } + + final String? namespace; + final String? taskId; + final int? version; + final DateTime? issuedAt; + final int? terminate; + final String? reason; + final String? requestedBy; + final DateTime? expiresAt; + + @override + $StemRevokeEntry toEntity() { + // Basic required-field check: non-nullable fields must be present. + final String? namespaceValue = namespace; + if (namespaceValue == null) { + throw StateError('Missing required field: namespace'); + } + final String? taskIdValue = taskId; + if (taskIdValue == null) { + throw StateError('Missing required field: taskId'); + } + final int? versionValue = version; + if (versionValue == null) { + throw StateError('Missing required field: version'); + } + final DateTime? issuedAtValue = issuedAt; + if (issuedAtValue == null) { + throw StateError('Missing required field: issuedAt'); + } + final int? terminateValue = terminate; + if (terminateValue == null) { + throw StateError('Missing required field: terminate'); + } + return $StemRevokeEntry( + namespace: namespaceValue, + taskId: taskIdValue, + version: versionValue, + issuedAt: issuedAtValue, + terminate: terminateValue, + reason: reason, + requestedBy: requestedBy, + expiresAt: expiresAt, + ); + } + + @override + Map toMap() { + return { + if (namespace != null) 'namespace': namespace, + if (taskId != null) 'task_id': taskId, + if (version != null) 'version': version, + if (issuedAt != null) 'issued_at': issuedAt, + if (terminate != null) 'terminate': terminate, + if (reason != null) 'reason': reason, + if (requestedBy != null) 'requested_by': requestedBy, + if (expiresAt != null) 'expires_at': expiresAt, + }; + } + + static const _StemRevokeEntryPartialCopyWithSentinel _copyWithSentinel = + _StemRevokeEntryPartialCopyWithSentinel(); + StemRevokeEntryPartial copyWith({ + Object? namespace = _copyWithSentinel, + Object? taskId = _copyWithSentinel, + Object? version = _copyWithSentinel, + Object? issuedAt = _copyWithSentinel, + Object? terminate = _copyWithSentinel, + Object? reason = _copyWithSentinel, + Object? requestedBy = _copyWithSentinel, + Object? expiresAt = _copyWithSentinel, + }) { + return StemRevokeEntryPartial( + namespace: identical(namespace, _copyWithSentinel) + ? this.namespace + : namespace as String?, + taskId: identical(taskId, _copyWithSentinel) + ? this.taskId + : taskId as String?, + version: identical(version, _copyWithSentinel) + ? this.version + : version as int?, + issuedAt: identical(issuedAt, _copyWithSentinel) + ? this.issuedAt + : issuedAt as DateTime?, + terminate: identical(terminate, _copyWithSentinel) + ? this.terminate + : terminate as int?, + reason: identical(reason, _copyWithSentinel) + ? this.reason + : reason as String?, + requestedBy: identical(requestedBy, _copyWithSentinel) + ? this.requestedBy + : requestedBy as String?, + expiresAt: identical(expiresAt, _copyWithSentinel) + ? this.expiresAt + : expiresAt as DateTime?, + ); + } +} + +class _StemRevokeEntryPartialCopyWithSentinel { + const _StemRevokeEntryPartialCopyWithSentinel(); +} + +/// Generated tracked model class for [StemRevokeEntry]. +/// +/// This class extends the user-defined [StemRevokeEntry] model and adds +/// attribute tracking, change detection, and relationship management. +/// Instances of this class are returned by queries and repositories. +/// +/// **Do not instantiate this class directly.** Use queries, repositories, +/// or model factories to create tracked model instances. +class $StemRevokeEntry extends StemRevokeEntry + with ModelAttributes, TimestampsTZImpl + implements OrmEntity { + /// Internal constructor for [$StemRevokeEntry]. + $StemRevokeEntry({ + required String namespace, + required String taskId, + required int version, + required DateTime issuedAt, + required int terminate, + String? reason, + String? requestedBy, + DateTime? expiresAt, + }) : super( + namespace: namespace, + taskId: taskId, + version: version, + issuedAt: issuedAt, + terminate: terminate, + reason: reason, + requestedBy: requestedBy, + expiresAt: expiresAt, + ) { + _attachOrmRuntimeMetadata({ + 'namespace': namespace, + 'task_id': taskId, + 'version': version, + 'issued_at': issuedAt, + 'terminate': terminate, + 'reason': reason, + 'requested_by': requestedBy, + 'expires_at': expiresAt, + }); + } + + /// Creates a tracked model instance from a user-defined model instance. + factory $StemRevokeEntry.fromModel(StemRevokeEntry model) { + return $StemRevokeEntry( + namespace: model.namespace, + taskId: model.taskId, + version: model.version, + issuedAt: model.issuedAt, + terminate: model.terminate, + reason: model.reason, + requestedBy: model.requestedBy, + expiresAt: model.expiresAt, + ); + } + + $StemRevokeEntry copyWith({ + String? namespace, + String? taskId, + int? version, + DateTime? issuedAt, + int? terminate, + String? reason, + String? requestedBy, + DateTime? expiresAt, + }) { + return $StemRevokeEntry( + namespace: namespace ?? this.namespace, + taskId: taskId ?? this.taskId, + version: version ?? this.version, + issuedAt: issuedAt ?? this.issuedAt, + terminate: terminate ?? this.terminate, + reason: reason ?? this.reason, + requestedBy: requestedBy ?? this.requestedBy, + expiresAt: expiresAt ?? this.expiresAt, + ); + } + + /// Builds a tracked model from a column/value map. + static $StemRevokeEntry fromMap( + Map data, { + ValueCodecRegistry? registry, + }) => _$StemRevokeEntryDefinition.fromMap(data, registry: registry); + + /// Converts this tracked model to a column/value map. + Map toMap({ValueCodecRegistry? registry}) => + _$StemRevokeEntryDefinition.toMap(this, registry: registry); + + /// Tracked getter for [namespace]. + @override + String get namespace => getAttribute('namespace') ?? super.namespace; + + /// Tracked setter for [namespace]. + set namespace(String value) => setAttribute('namespace', value); + + /// Tracked getter for [taskId]. + @override + String get taskId => getAttribute('task_id') ?? super.taskId; + + /// Tracked setter for [taskId]. + set taskId(String value) => setAttribute('task_id', value); + + /// Tracked getter for [version]. + @override + int get version => getAttribute('version') ?? super.version; + + /// Tracked setter for [version]. + set version(int value) => setAttribute('version', value); + + /// Tracked getter for [issuedAt]. + @override + DateTime get issuedAt => + getAttribute('issued_at') ?? super.issuedAt; + + /// Tracked setter for [issuedAt]. + set issuedAt(DateTime value) => setAttribute('issued_at', value); + + /// Tracked getter for [terminate]. + @override + int get terminate => getAttribute('terminate') ?? super.terminate; + + /// Tracked setter for [terminate]. + set terminate(int value) => setAttribute('terminate', value); + + /// Tracked getter for [reason]. + @override + String? get reason => getAttribute('reason') ?? super.reason; + + /// Tracked setter for [reason]. + set reason(String? value) => setAttribute('reason', value); + + /// Tracked getter for [requestedBy]. + @override + String? get requestedBy => + getAttribute('requested_by') ?? super.requestedBy; + + /// Tracked setter for [requestedBy]. + set requestedBy(String? value) => setAttribute('requested_by', value); + + /// Tracked getter for [expiresAt]. + @override + DateTime? get expiresAt => + getAttribute('expires_at') ?? super.expiresAt; + + /// Tracked setter for [expiresAt]. + set expiresAt(DateTime? value) => setAttribute('expires_at', value); + + void _attachOrmRuntimeMetadata(Map values) { + replaceAttributes(values); + attachModelDefinition(_$StemRevokeEntryDefinition); + } +} + +class _StemRevokeEntryCopyWithSentinel { + const _StemRevokeEntryCopyWithSentinel(); +} + +extension StemRevokeEntryOrmExtension on StemRevokeEntry { + static const _StemRevokeEntryCopyWithSentinel _copyWithSentinel = + _StemRevokeEntryCopyWithSentinel(); + StemRevokeEntry copyWith({ + Object? namespace = _copyWithSentinel, + Object? taskId = _copyWithSentinel, + Object? version = _copyWithSentinel, + Object? issuedAt = _copyWithSentinel, + Object? terminate = _copyWithSentinel, + Object? reason = _copyWithSentinel, + Object? requestedBy = _copyWithSentinel, + Object? expiresAt = _copyWithSentinel, + }) { + return StemRevokeEntry( + namespace: identical(namespace, _copyWithSentinel) + ? this.namespace + : namespace as String, + taskId: identical(taskId, _copyWithSentinel) + ? this.taskId + : taskId as String, + version: identical(version, _copyWithSentinel) + ? this.version + : version as int, + issuedAt: identical(issuedAt, _copyWithSentinel) + ? this.issuedAt + : issuedAt as DateTime, + terminate: identical(terminate, _copyWithSentinel) + ? this.terminate + : terminate as int, + reason: identical(reason, _copyWithSentinel) + ? this.reason + : reason as String?, + requestedBy: identical(requestedBy, _copyWithSentinel) + ? this.requestedBy + : requestedBy as String?, + expiresAt: identical(expiresAt, _copyWithSentinel) + ? this.expiresAt + : expiresAt as DateTime?, + ); + } + + /// Converts this model to a column/value map. + Map toMap({ValueCodecRegistry? registry}) => + _$StemRevokeEntryDefinition.toMap(this, registry: registry); + + /// Builds a model from a column/value map. + static StemRevokeEntry fromMap( + Map data, { + ValueCodecRegistry? registry, + }) => _$StemRevokeEntryDefinition.fromMap(data, registry: registry); + + /// The Type of the generated ORM-managed model class. + /// Use this when you need to specify the tracked model type explicitly, + /// for example in generic type parameters. + static Type get trackedType => $StemRevokeEntry; + + /// Converts this immutable model to a tracked ORM-managed model. + /// The tracked model supports attribute tracking, change detection, + /// and persistence operations like save() and touch(). + $StemRevokeEntry toTracked() { + return $StemRevokeEntry.fromModel(this); + } +} + +extension StemRevokeEntryPredicateFields on PredicateBuilder { + PredicateField get namespace => + PredicateField(this, 'namespace'); + PredicateField get taskId => + PredicateField(this, 'taskId'); + PredicateField get version => + PredicateField(this, 'version'); + PredicateField get issuedAt => + PredicateField(this, 'issuedAt'); + PredicateField get terminate => + PredicateField(this, 'terminate'); + PredicateField get reason => + PredicateField(this, 'reason'); + PredicateField get requestedBy => + PredicateField(this, 'requestedBy'); + PredicateField get expiresAt => + PredicateField(this, 'expiresAt'); +} + +void registerStemRevokeEntryEventHandlers(EventBus bus) { + // No event handlers registered for StemRevokeEntry. +} diff --git a/packages/stem_sqlite/lib/src/stack/sqlite_adapter.dart b/packages/stem_sqlite/lib/src/stack/sqlite_adapter.dart index c924cbd5..2267459d 100644 --- a/packages/stem_sqlite/lib/src/stack/sqlite_adapter.dart +++ b/packages/stem_sqlite/lib/src/stack/sqlite_adapter.dart @@ -81,7 +81,10 @@ class StemSqliteAdapter implements StemStoreAdapter { LockStoreFactory? lockStoreFactory(Uri uri) => null; @override - RevokeStoreFactory? revokeStoreFactory(Uri uri) => null; + RevokeStoreFactory? revokeStoreFactory(Uri uri) { + final file = _fileFromUri(uri); + return sqliteRevokeStoreFactory(file); + } } File _fileFromUri(Uri uri) { diff --git a/packages/stem_sqlite/lib/src/workflow/sqlite_factories.dart b/packages/stem_sqlite/lib/src/workflow/sqlite_factories.dart index ebebe180..9d989a09 100644 --- a/packages/stem_sqlite/lib/src/workflow/sqlite_factories.dart +++ b/packages/stem_sqlite/lib/src/workflow/sqlite_factories.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:stem/stem.dart'; import 'package:stem_sqlite/src/backend/sqlite_result_backend.dart'; import 'package:stem_sqlite/src/broker/sqlite_broker.dart'; +import 'package:stem_sqlite/src/control/sqlite_revoke_store.dart'; import 'package:stem_sqlite/src/workflow/sqlite_workflow_store.dart'; /// Creates a [StemBrokerFactory] backed by SQLite. @@ -62,3 +63,18 @@ WorkflowStoreFactory sqliteWorkflowStoreFactory(File file) { }, ); } + +/// Creates a [RevokeStoreFactory] backed by SQLite. +RevokeStoreFactory sqliteRevokeStoreFactory( + File file, { + String namespace = 'stem', +}) { + return RevokeStoreFactory( + create: () async => SqliteRevokeStore.open(file, namespace: namespace), + dispose: (store) async { + if (store is SqliteRevokeStore) { + await store.close(); + } + }, + ); +} diff --git a/packages/stem_sqlite/lib/stem_sqlite.dart b/packages/stem_sqlite/lib/stem_sqlite.dart index d60d79c6..fe60dd33 100644 --- a/packages/stem_sqlite/lib/stem_sqlite.dart +++ b/packages/stem_sqlite/lib/stem_sqlite.dart @@ -2,6 +2,7 @@ export 'orm_registry.g.dart'; export 'src/backend/sqlite_result_backend.dart' show SqliteResultBackend; export 'src/broker/sqlite_broker.dart' show SqliteBroker; export 'src/connection.dart' show SqliteConnections; +export 'src/control/sqlite_revoke_store.dart' show SqliteRevokeStore; // Exported for compatibility with the deprecated StemSqliteDatabase API. // ignore: deprecated_member_use_from_same_package export 'src/database.dart' show StemSqliteDatabase; @@ -12,5 +13,6 @@ export 'src/workflow/sqlite_factories.dart' show sqliteBrokerFactory, sqliteResultBackendFactory, + sqliteRevokeStoreFactory, sqliteWorkflowStoreFactory; export 'src/workflow/sqlite_workflow_store.dart' show SqliteWorkflowStore; diff --git a/packages/stem_sqlite/test/broker/sqlite_broker_test.dart b/packages/stem_sqlite/test/broker/sqlite_broker_test.dart index 883ed47f..f2b2f2dd 100644 --- a/packages/stem_sqlite/test/broker/sqlite_broker_test.dart +++ b/packages/stem_sqlite/test/broker/sqlite_broker_test.dart @@ -52,6 +52,26 @@ void main() { ), ); + runQueueEventsContractTests( + adapterName: 'SQLite', + factory: QueueEventsContractFactory( + create: () async => SqliteBroker.open( + dbFile, + defaultVisibilityTimeout: const Duration(milliseconds: 200), + pollInterval: const Duration(milliseconds: 25), + sweeperInterval: const Duration(milliseconds: 75), + ), + dispose: (broker) => (broker as SqliteBroker).close(), + additionalBrokerFactory: () async => SqliteBroker.open( + dbFile, + defaultVisibilityTimeout: const Duration(milliseconds: 200), + pollInterval: const Duration(milliseconds: 25), + sweeperInterval: const Duration(milliseconds: 75), + ), + additionalDispose: (broker) => (broker as SqliteBroker).close(), + ), + ); + test('fromDataSource runs migrations', () async { ensureSqliteDriverRegistration(); final dataSource = DataSource( diff --git a/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart b/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart new file mode 100644 index 00000000..f68a5025 --- /dev/null +++ b/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart @@ -0,0 +1,105 @@ +import 'dart:io'; + +import 'package:ormed/ormed.dart'; +import 'package:ormed_sqlite/ormed_sqlite.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_adapter_tests/stem_adapter_tests.dart'; +import 'package:stem_sqlite/stem_sqlite.dart'; +import 'package:test/test.dart'; + +void main() { + late Directory tempDir; + late File dbFile; + + setUp(() async { + tempDir = Directory.systemTemp.createTempSync('stem_sqlite_revoke_store'); + dbFile = File('${tempDir.path}/revoke.db'); + }); + + tearDown(() async { + if (dbFile.existsSync()) { + await dbFile.delete(); + } + await tempDir.delete(recursive: true); + }); + + runRevokeStoreContractTests( + adapterName: 'SQLite', + factory: RevokeStoreContractFactory( + create: () async => SqliteRevokeStore.open(dbFile), + dispose: (store) => (store as SqliteRevokeStore).close(), + ), + ); + + test('fromDataSource runs migrations', () async { + ensureSqliteDriverRegistration(); + final dataSource = DataSource( + DataSourceOptions( + driver: SqliteDriverAdapter.file(dbFile.path), + registry: buildOrmRegistry(), + database: dbFile.path, + ), + ); + final store = await SqliteRevokeStore.fromDataSource(dataSource); + try { + final now = DateTime.utc(2026, 2, 24, 12, 30); + await store.upsertAll([ + RevokeEntry( + namespace: 'stem', + taskId: 'from-datasource', + version: 1, + issuedAt: now, + ), + ]); + + final listed = await store.list('stem'); + expect(listed.map((entry) => entry.taskId), contains('from-datasource')); + } finally { + await store.close(); + await dataSource.dispose(); + } + }); + + test('connect supports sqlite urls', () async { + final store = await SqliteRevokeStore.connect('sqlite://${dbFile.path}'); + try { + final now = DateTime.utc(2026, 2, 24, 12, 45); + await store.upsertAll([ + RevokeEntry( + namespace: 'stem', + taskId: 'connect-sqlite', + version: 1, + issuedAt: now, + ), + ]); + final listed = await store.list('stem'); + expect(listed.map((entry) => entry.taskId), contains('connect-sqlite')); + } finally { + await store.close(); + } + }); + + test('adapter resolves revoke store factory', () async { + const adapter = StemSqliteAdapter(); + final factory = adapter.revokeStoreFactory( + Uri.parse('sqlite://${dbFile.path}'), + ); + expect(factory, isNotNull); + final store = await factory!.create(); + try { + final now = DateTime.utc(2026, 2, 24, 13); + await store.upsertAll([ + RevokeEntry( + namespace: 'stem', + taskId: 'adapter-revoke', + version: 1, + issuedAt: now, + ), + ]); + final listed = await store.list('stem'); + expect(listed.map((entry) => entry.taskId), contains('adapter-revoke')); + } finally { + await factory.dispose(store); + } + }); +} From ecb4f22addf14fb3689699afc51ea0b548aff350 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 12:27:32 -0500 Subject: [PATCH 03/23] fix(brokers): allow broadcast-only subscriptions for queue events --- .../lib/src/brokers/postgres_broker.dart | 48 +++++++++++-------- .../lib/src/brokers/redis_broker.dart | 24 ++++++---- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/packages/stem_postgres/lib/src/brokers/postgres_broker.dart b/packages/stem_postgres/lib/src/brokers/postgres_broker.dart index c387542d..4426f3fa 100644 --- a/packages/stem_postgres/lib/src/brokers/postgres_broker.dart +++ b/packages/stem_postgres/lib/src/brokers/postgres_broker.dart @@ -221,11 +221,6 @@ class PostgresBroker implements Broker { 'queues=${subscription.queues})', _logContext({'queues': subscription.queues}), ); - if (subscription.queues.isEmpty) { - throw ArgumentError( - 'RoutingSubscription must specify at least one queue.', - ); - } if (subscription.queues.length > 1) { throw UnsupportedError( 'PostgresBroker currently supports consuming a single queue per ' @@ -233,21 +228,29 @@ class PostgresBroker implements Broker { ); } - final queue = subscription.queues.first; + final queue = subscription.queues.isEmpty + ? null + : subscription.queues.single; final group = consumerGroup ?? 'default'; final consumer = consumerName ?? const Uuid().v7(); - final locker = _encodeLocker(queue, group, consumer); final broadcastChannels = subscription.broadcastChannels; + if (queue == null && broadcastChannels.isEmpty) { + throw ArgumentError( + 'PostgresBroker requires at least one queue or broadcast channel.', + ); + } + final locker = _encodeLocker(queue ?? '__broadcast__', group, consumer); late _ConsumerRunner runner; late StreamController controller; controller = StreamController( onListen: () => runner.start(), onCancel: () { + final queueLabel = queue ?? ''; stemLogger.debug( - 'Consumer stream canceled (queue=$queue, worker=$consumer)', + 'Consumer stream canceled (queue=$queueLabel, worker=$consumer)', _logContext({ - 'queue': queue, + 'queue': queueLabel, 'worker': consumer, }), ); @@ -806,7 +809,7 @@ class _ConsumerRunner { final PostgresBroker broker; final StreamController controller; - final String queue; + final String? queue; final String locker; final int prefetch; final List broadcastChannels; @@ -818,17 +821,19 @@ class _ConsumerRunner { void start() { if (_started) return; _started = true; + final queueLabel = queue ?? ''; stemLogger.debug( - 'Consumer runner started (queue=$queue, worker=$workerId)', - broker._logContext({'queue': queue, 'worker': workerId}), + 'Consumer runner started (queue=$queueLabel, worker=$workerId)', + broker._logContext({'queue': queueLabel, 'worker': workerId}), ); unawaited(_loop()); } void stop() { + final queueLabel = queue ?? ''; stemLogger.debug( - 'Consumer runner stopped (queue=$queue, worker=$workerId)', - broker._logContext({'queue': queue, 'worker': workerId}), + 'Consumer runner stopped (queue=$queueLabel, worker=$workerId)', + broker._logContext({'queue': queueLabel, 'worker': workerId}), ); _stopped = true; } @@ -840,10 +845,12 @@ class _ConsumerRunner { !broker._closed) { try { final jobs = <_QueuedJob>[]; - for (var i = 0; i < prefetch; i++) { - final job = await broker._claimNextJob(queue, locker); - if (job == null) break; - jobs.add(job); + if (queue != null) { + for (var i = 0; i < prefetch; i++) { + final job = await broker._claimNextJob(queue!, locker); + if (job == null) break; + jobs.add(job); + } } final broadcasts = broadcastChannels.isEmpty ? const [] @@ -869,11 +876,12 @@ class _ConsumerRunner { controller.add(delivery); } } on Object catch (error, stack) { + final queueLabel = queue ?? ''; stemLogger.warning( - 'Consumer loop error (queue=$queue, worker=$workerId): ' + 'Consumer loop error (queue=$queueLabel, worker=$workerId): ' '$error\n$stack', broker._logContext({ - 'queue': queue, + 'queue': queueLabel, 'worker': workerId, 'error': error.toString(), 'stack': stack.toString(), diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 1b16cbcc..9412562a 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -479,22 +479,27 @@ class RedisStreamsBroker implements Broker { String? consumerGroup, String? consumerName, }) { - if (subscription.queues.isEmpty) { - throw ArgumentError( - 'RoutingSubscription must specify at least one queue.', - ); - } if (subscription.queues.length > 1) { throw UnsupportedError( 'RedisStreamsBroker currently supports consuming a single queue ' 'per subscription.', ); } - final queue = subscription.queues.first; + final queue = subscription.queues.isEmpty + ? null + : subscription.queues.single; final consumer = consumerName ?? const Uuid().v7(); - final group = consumerGroup ?? _groupKey(queue); - final streamKeys = _priorityStreamKeys(queue); final broadcastChannels = subscription.broadcastChannels; + if (queue == null && broadcastChannels.isEmpty) { + throw ArgumentError( + 'RedisStreamsBroker requires at least one queue or broadcast channel.', + ); + } + final group = + consumerGroup ?? (queue == null ? '__broadcast__' : _groupKey(queue)); + final streamKeys = queue == null + ? const [] + : _priorityStreamKeys(queue); final claimTimerKeys = {}; RedisConnection? consumerConnection; Command? consumerCommand; @@ -566,6 +571,9 @@ class RedisStreamsBroker implements Broker { } Future loop() async { + if (queue == null || streamKeys.isEmpty) { + return; + } while (!controller.isClosed && !_closed) { for (final stream in streamKeys) { await _ensureGroupForStream(queue, stream); From b7684ebbb32ae713adaf1bd4bcf4ccc4fcd99b77 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 12:42:03 -0500 Subject: [PATCH 04/23] fix(stem): address retry timing, batch idempotency, and flaky tests --- packages/stem/lib/src/canvas/canvas.dart | 63 ++++++++- packages/stem/lib/src/signals/emitter.dart | 2 + packages/stem/lib/src/signals/payloads.dart | 11 +- .../stem/test/unit/canvas/canvas_test.dart | 50 +++++++ .../stem/test/unit/signals/payloads_test.dart | 6 + .../test/unit/cli/cli_worker_stats_test.dart | 128 +++++++++++------- .../lib/src/brokers/redis_broker.dart | 2 +- 7 files changed, 205 insertions(+), 57 deletions(-) diff --git a/packages/stem/lib/src/canvas/canvas.dart b/packages/stem/lib/src/canvas/canvas.dart index 52acf161..9e4061bd 100644 --- a/packages/stem/lib/src/canvas/canvas.dart +++ b/packages/stem/lib/src/canvas/canvas.dart @@ -424,6 +424,35 @@ class Canvas { throw ArgumentError('Batch must include at least one task'); } final id = batchId ?? _generateId('batch'); + final existing = await backend.getGroup(id); + if (existing != null) { + if (existing.meta['stem.batch'] != true) { + throw StateError('Group "$id" already exists and is not a batch'); + } + return BatchSubmission( + batchId: id, + taskIds: _batchTaskIdsFromGroup(existing), + ); + } + + final normalizedSignatures = >[]; + final taskIds = []; + for (final signature in signatures) { + final raw = signature(); + final grouped = raw.copyWith( + headers: {...raw.headers, 'stem-group-id': id}, + meta: {...raw.meta, 'groupId': id}, + ); + taskIds.add(grouped.id); + normalizedSignatures.add( + TaskSignature.custom( + signature.name, + () => grouped, + decode: signature.decode, + ), + ); + } + final createdAt = DateTime.now().toUtc().toIso8601String(); await backend.initGroup( GroupDescriptor( @@ -434,11 +463,11 @@ class Canvas { 'stem.batch': true, 'stem.batch.createdAt': createdAt, 'stem.batch.taskCount': signatures.length, + 'stem.batch.taskIds': taskIds, }, ), ); - final dispatch = await group(signatures, groupId: id); - final taskIds = List.from(dispatch.taskIds); + final dispatch = await group(normalizedSignatures, groupId: id); await dispatch.dispose(); return BatchSubmission(batchId: id, taskIds: taskIds); } @@ -683,7 +712,9 @@ class Canvas { } BatchStatus _buildBatchStatus(GroupStatus status) { - final entries = status.results.entries.toList(growable: false); + final entries = status.results.entries + .where((entry) => entry.value.state.isTerminal) + .toList(growable: false); final succeeded = entries .where((entry) => entry.value.state == TaskState.succeeded) .length; @@ -702,11 +733,11 @@ class Canvas { state = BatchLifecycleState.pending; } else if (completed < status.expected) { state = BatchLifecycleState.running; - } else if (failed == 0 && cancelled == 0) { + } else if (succeeded == completed) { state = BatchLifecycleState.succeeded; - } else if (succeeded == 0 && cancelled == 0) { + } else if (failed == completed) { state = BatchLifecycleState.failed; - } else if (succeeded == 0 && failed == 0) { + } else if (cancelled == completed) { state = BatchLifecycleState.cancelled; } else { state = BatchLifecycleState.partial; @@ -726,6 +757,26 @@ class Canvas { ); } + List _batchTaskIdsFromGroup(GroupStatus status) { + final taskIds = {}; + final rawTaskIds = status.meta['stem.batch.taskIds']; + if (rawTaskIds is List) { + for (final rawTaskId in rawTaskIds) { + if (rawTaskId is! String) { + continue; + } + final trimmed = rawTaskId.trim(); + if (trimmed.isNotEmpty) { + taskIds.add(trimmed); + } + } + } + if (taskIds.isEmpty) { + taskIds.addAll(status.results.keys.where((id) => id.trim().isNotEmpty)); + } + return taskIds.toList(growable: false)..sort(); + } + (Envelope, TaskPayloadEncoder) _prepareEnvelope(Envelope envelope) { final handler = registry.resolve(envelope.name); final argsEncoder = _resolveArgsEncoder(handler); diff --git a/packages/stem/lib/src/signals/emitter.dart b/packages/stem/lib/src/signals/emitter.dart index e0045e25..a3af95ce 100644 --- a/packages/stem/lib/src/signals/emitter.dart +++ b/packages/stem/lib/src/signals/emitter.dart @@ -101,12 +101,14 @@ class StemSignalEmitter { required DateTime nextRetryAt, String? sender, }) { + final emittedAt = DateTime.now().toUtc(); return StemSignals.taskRetry.emit( TaskRetryPayload( envelope: envelope, worker: worker, reason: reason, nextRetryAt: nextRetryAt, + emittedAt: emittedAt, ), sender: _senderOverride(sender), ); diff --git a/packages/stem/lib/src/signals/payloads.dart b/packages/stem/lib/src/signals/payloads.dart index 016d7ce8..7249c1d4 100644 --- a/packages/stem/lib/src/signals/payloads.dart +++ b/packages/stem/lib/src/signals/payloads.dart @@ -232,12 +232,13 @@ class TaskPostrunPayload implements StemEvent { /// Payload emitted when a task is scheduled for retry. class TaskRetryPayload implements StemEvent { /// Creates a new [TaskRetryPayload] instance. - const TaskRetryPayload({ + TaskRetryPayload({ required this.envelope, required this.worker, required this.reason, required this.nextRetryAt, - }); + DateTime? emittedAt, + }) : emittedAt = (emittedAt ?? DateTime.now()).toUtc(); /// The task envelope to be retried. final Envelope envelope; @@ -251,6 +252,9 @@ class TaskRetryPayload implements StemEvent { /// The scheduled time for the next retry attempt. final DateTime nextRetryAt; + /// The timestamp when the retry signal was emitted. + final DateTime emittedAt; + /// The unique identifier for the task. String get taskId => envelope.id; @@ -264,7 +268,7 @@ class TaskRetryPayload implements StemEvent { String get eventName => 'task-retry'; @override - DateTime get occurredAt => nextRetryAt.toUtc(); + DateTime get occurredAt => emittedAt; @override Map get attributes => { @@ -274,6 +278,7 @@ class TaskRetryPayload implements StemEvent { 'attempt': attempt, 'workerId': worker.id, 'reason': reason.toString(), + 'emittedAt': emittedAt.toIso8601String(), 'nextRetryAt': nextRetryAt.toUtc().toIso8601String(), }; } diff --git a/packages/stem/test/unit/canvas/canvas_test.dart b/packages/stem/test/unit/canvas/canvas_test.dart index 5182f0da..21c4fe15 100644 --- a/packages/stem/test/unit/canvas/canvas_test.dart +++ b/packages/stem/test/unit/canvas/canvas_test.dart @@ -102,6 +102,56 @@ void main() { expect(status.meta['stem.batch'], isTrue); }, ); + + test('submitBatch with an existing batchId is idempotent', () async { + const batchId = 'batch-fixed'; + final first = await canvas.submitBatch([ + task('echo', args: {'value': 1}), + task('echo', args: {'value': 2}), + ], batchId: batchId); + final second = await canvas.submitBatch([ + task('echo', args: {'value': 99}), + ], batchId: batchId); + + expect(second.batchId, equals(first.batchId)); + expect(second.taskIds, equals(first.taskIds)); + + final status = await _waitForBatchTerminal(canvas, batchId); + expect(status.expected, equals(2)); + expect(status.meta['stem.batch.taskCount'], equals(2)); + }); + + test( + 'inspectBatch counts only terminal group entries as completed', + () async { + const batchId = 'batch-non-terminal'; + await backend.initGroup( + GroupDescriptor( + id: batchId, + expected: 2, + meta: const {'stem.batch': true}, + ), + ); + await backend.addGroupResult( + batchId, + TaskStatus(id: 'task-queued', state: TaskState.queued, attempt: 0), + ); + await backend.addGroupResult( + batchId, + TaskStatus( + id: 'task-succeeded', + state: TaskState.succeeded, + attempt: 0, + ), + ); + + final status = await canvas.inspectBatch(batchId); + expect(status, isNotNull); + expect(status!.completed, equals(1)); + expect(status.succeededCount, equals(1)); + expect(status.state, equals(BatchLifecycleState.running)); + }, + ); }); } diff --git a/packages/stem/test/unit/signals/payloads_test.dart b/packages/stem/test/unit/signals/payloads_test.dart index 4a6da998..a71b74fb 100644 --- a/packages/stem/test/unit/signals/payloads_test.dart +++ b/packages/stem/test/unit/signals/payloads_test.dart @@ -55,9 +55,15 @@ void main() { worker: worker, reason: 'boom', nextRetryAt: DateTime.utc(2025), + emittedAt: DateTime.utc(2024), ); expect(retry.taskId, equals('task-1')); expect(retry.taskName, equals('demo.task')); expect(retry.attempt, equals(2)); + expect(retry.occurredAt, equals(DateTime.utc(2024))); + expect( + retry.attributes['nextRetryAt'], + equals(DateTime.utc(2025).toIso8601String()), + ); }); } diff --git a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart index 9757559f..167115f7 100644 --- a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart @@ -112,7 +112,10 @@ void main() { final broker = InMemoryBroker(); final backend = InMemoryResultBackend(); final revokeStore = InMemoryRevokeStore(); - final registry = SimpleTaskRegistry()..register(_FastTask()); + final started = Completer(); + final release = Completer(); + final registry = SimpleTaskRegistry() + ..register(_BlockingTask(started, release)); final worker = Worker( broker: broker, @@ -125,53 +128,65 @@ void main() { revokeStore: revokeStore, ); await worker.start(); + try { + final pauseOut = StringBuffer(); + final pauseErr = StringBuffer(); + final pauseCode = await runStemCli( + ['worker', 'pause', '--worker', 'worker-pause', '--queue', 'default'], + out: pauseOut, + err: pauseErr, + contextBuilder: () async => CliContext( + broker: broker, + backend: backend, + revokeStore: revokeStore, + routing: RoutingRegistry(RoutingConfig.legacy()), + dispose: () async {}, + registry: registry, + ), + ); + expect(pauseCode, equals(0)); + + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue('tasks.blocking'); + await _assertTaskRemainsQueued(backend, taskId); + expect(started.isCompleted, isFalse); + + final resumeOut = StringBuffer(); + final resumeErr = StringBuffer(); + final resumeCode = await runStemCli( + [ + 'worker', + 'resume', + '--worker', + 'worker-pause', + '--queue', + 'default', + ], + out: resumeOut, + err: resumeErr, + contextBuilder: () async => CliContext( + broker: broker, + backend: backend, + revokeStore: revokeStore, + routing: RoutingRegistry(RoutingConfig.legacy()), + dispose: () async {}, + registry: registry, + ), + ); + expect(resumeCode, equals(0)); - final pauseOut = StringBuffer(); - final pauseErr = StringBuffer(); - final pauseCode = await runStemCli( - ['worker', 'pause', '--worker', 'worker-pause', '--queue', 'default'], - out: pauseOut, - err: pauseErr, - contextBuilder: () async => CliContext( - broker: broker, - backend: backend, - revokeStore: revokeStore, - routing: RoutingRegistry(RoutingConfig.legacy()), - dispose: () async {}, - registry: registry, - ), - ); - expect(pauseCode, equals(0)); - - final stem = Stem(broker: broker, registry: registry, backend: backend); - final taskId = await stem.enqueue('tasks.fast'); - await Future.delayed(const Duration(milliseconds: 180)); - final pausedStatus = await backend.get(taskId); - expect(pausedStatus?.state, TaskState.queued); - - final resumeOut = StringBuffer(); - final resumeErr = StringBuffer(); - final resumeCode = await runStemCli( - ['worker', 'resume', '--worker', 'worker-pause', '--queue', 'default'], - out: resumeOut, - err: resumeErr, - contextBuilder: () async => CliContext( - broker: broker, - backend: backend, - revokeStore: revokeStore, - routing: RoutingRegistry(RoutingConfig.legacy()), - dispose: () async {}, - registry: registry, - ), - ); - expect(resumeCode, equals(0)); - - await _waitFor( - () async => (await backend.get(taskId))?.state == TaskState.succeeded, - ); - - await worker.shutdown(); - broker.dispose(); + await _waitFor(() async => started.isCompleted); + release.complete(); + await _waitFor( + () async => (await backend.get(taskId))?.state == TaskState.succeeded, + ); + } finally { + if (!release.isCompleted) { + release.complete(); + } + await worker.shutdown(); + broker.dispose(); + } }); test('includes active task metadata', () async { @@ -474,6 +489,25 @@ Future _waitFor( } } +Future _assertTaskRemainsQueued( + ResultBackend backend, + String taskId, { + Duration holdFor = const Duration(milliseconds: 180), +}) async { + await _waitFor(() async => (await backend.get(taskId))?.state != null); + final deadline = DateTime.now().add(holdFor); + while (DateTime.now().isBefore(deadline)) { + final status = await backend.get(taskId); + if (status?.state != TaskState.queued) { + throw StateError( + 'Expected task $taskId to remain queued while paused. ' + 'Found ${status?.state}.', + ); + } + await Future.delayed(const Duration(milliseconds: 20)); + } +} + class _FastTask implements TaskHandler { @override String get name => 'tasks.fast'; diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 9412562a..2060390f 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -571,7 +571,7 @@ class RedisStreamsBroker implements Broker { } Future loop() async { - if (queue == null || streamKeys.isEmpty) { + if (queue == null) { return; } while (!controller.isClosed && !_closed) { From 5559a24b2280ee27d7b5d1bd8f365964fbae03e5 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 12:44:46 -0500 Subject: [PATCH 05/23] fix(canvas): preserve stored batch task id order --- packages/stem/lib/src/canvas/canvas.dart | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/stem/lib/src/canvas/canvas.dart b/packages/stem/lib/src/canvas/canvas.dart index 9e4061bd..e5d99441 100644 --- a/packages/stem/lib/src/canvas/canvas.dart +++ b/packages/stem/lib/src/canvas/canvas.dart @@ -758,7 +758,8 @@ class Canvas { } List _batchTaskIdsFromGroup(GroupStatus status) { - final taskIds = {}; + final taskIds = []; + final seen = {}; final rawTaskIds = status.meta['stem.batch.taskIds']; if (rawTaskIds is List) { for (final rawTaskId in rawTaskIds) { @@ -767,14 +768,24 @@ class Canvas { } final trimmed = rawTaskId.trim(); if (trimmed.isNotEmpty) { - taskIds.add(trimmed); + if (seen.add(trimmed)) { + taskIds.add(trimmed); + } } } } if (taskIds.isEmpty) { - taskIds.addAll(status.results.keys.where((id) => id.trim().isNotEmpty)); + for (final taskId in status.results.keys) { + final trimmed = taskId.trim(); + if (trimmed.isEmpty) { + continue; + } + if (seen.add(trimmed)) { + taskIds.add(trimmed); + } + } } - return taskIds.toList(growable: false)..sort(); + return List.unmodifiable(taskIds); } (Envelope, TaskPayloadEncoder) _prepareEnvelope(Envelope envelope) { From 937f9805a40c92434e368f2c24ce49b4aa9ab482 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 13:27:28 -0500 Subject: [PATCH 06/23] feat(clock): add shared stem clock abstraction --- packages/stem/lib/src/core/clock.dart | 53 +++++++++++++++++++ .../lib/src/workflow/core/workflow_clock.dart | 9 ++-- packages/stem/lib/stem.dart | 1 + packages/stem/test/unit/core/clock_test.dart | 26 +++++++++ .../unit/workflow/workflow_clock_test.dart | 10 ++-- 5 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 packages/stem/lib/src/core/clock.dart create mode 100644 packages/stem/test/unit/core/clock_test.dart diff --git a/packages/stem/lib/src/core/clock.dart b/packages/stem/lib/src/core/clock.dart new file mode 100644 index 00000000..842a303c --- /dev/null +++ b/packages/stem/lib/src/core/clock.dart @@ -0,0 +1,53 @@ +import 'dart:async'; + +/// Shared clock abstraction used across the Stem ecosystem. +abstract class StemClock { + /// Creates a clock implementation. + const StemClock(); + + /// Returns the current instant. + DateTime now(); +} + +/// Default wall-clock implementation. +class SystemStemClock extends StemClock { + /// Creates a system clock wrapper. + const SystemStemClock(); + + @override + DateTime now() => DateTime.now(); +} + +/// Controllable clock for deterministic testing. +class FakeStemClock extends StemClock { + /// Creates a fake clock initialized to [initial]. + FakeStemClock(DateTime initial) : currentTime = initial; + + /// Current instant returned by [now]. + DateTime currentTime; + + @override + DateTime now() => currentTime; + + /// Advances the fake clock by [duration]. + void advance(Duration duration) { + currentTime = currentTime.add(duration); + } +} + +final Object _zoneClockKey = Object(); +const StemClock _systemClock = SystemStemClock(); + +/// Returns the current instant from the active clock scope. +DateTime stemNow() { + final clock = Zone.current[_zoneClockKey]; + if (clock is StemClock) { + return clock.now(); + } + return _systemClock.now(); +} + +/// Runs [body] using [clock] as the active clock for this zone. +T withStemClock(StemClock clock, T Function() body) { + return runZoned(body, zoneValues: {_zoneClockKey: clock}); +} diff --git a/packages/stem/lib/src/workflow/core/workflow_clock.dart b/packages/stem/lib/src/workflow/core/workflow_clock.dart index 544fd5a6..8bb556bd 100644 --- a/packages/stem/lib/src/workflow/core/workflow_clock.dart +++ b/packages/stem/lib/src/workflow/core/workflow_clock.dart @@ -1,12 +1,11 @@ +import 'package:stem/src/core/clock.dart'; + /// Abstraction over time sources used by the workflow runtime and stores. // Intentionally interface-like for injection and testing. // ignore: one_member_abstracts -abstract class WorkflowClock { +abstract class WorkflowClock extends StemClock { /// Creates a workflow clock implementation. const WorkflowClock(); - - /// Returns the current instant. - DateTime now(); } /// Default clock that proxies to [DateTime.now]. @@ -15,7 +14,7 @@ class SystemWorkflowClock extends WorkflowClock { const SystemWorkflowClock(); @override - DateTime now() => DateTime.now(); + DateTime now() => stemNow(); } /// Controllable clock intended for tests. diff --git a/packages/stem/lib/stem.dart b/packages/stem/lib/stem.dart index b534e288..143f8729 100644 --- a/packages/stem/lib/stem.dart +++ b/packages/stem/lib/stem.dart @@ -87,6 +87,7 @@ export 'src/control/file_revoke_store.dart'; export 'src/control/in_memory_revoke_store.dart'; export 'src/control/revoke_store.dart'; export 'src/core/chord_metadata.dart'; +export 'src/core/clock.dart'; export 'src/core/config.dart'; export 'src/core/contracts.dart'; export 'src/core/encoder_keys.dart'; diff --git a/packages/stem/test/unit/core/clock_test.dart b/packages/stem/test/unit/core/clock_test.dart new file mode 100644 index 00000000..00027029 --- /dev/null +++ b/packages/stem/test/unit/core/clock_test.dart @@ -0,0 +1,26 @@ +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + group('clock abstraction', () { + test('stemNow uses active scoped clock', () { + final fake = FakeStemClock(DateTime.utc(2025, 1, 1, 12)); + + final now = withStemClock(fake, stemNow); + + expect(now, DateTime.utc(2025, 1, 1, 12)); + }); + + test('withStemClock propagates through async boundaries', () async { + final fake = FakeStemClock(DateTime.utc(2025, 1, 1, 12)); + + final now = await withStemClock(fake, () async { + await Future.delayed(Duration.zero); + fake.advance(const Duration(seconds: 30)); + return stemNow(); + }); + + expect(now, DateTime.utc(2025, 1, 1, 12, 0, 30)); + }); + }); +} diff --git a/packages/stem/test/unit/workflow/workflow_clock_test.dart b/packages/stem/test/unit/workflow/workflow_clock_test.dart index 46a25042..82dfd94c 100644 --- a/packages/stem/test/unit/workflow/workflow_clock_test.dart +++ b/packages/stem/test/unit/workflow/workflow_clock_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/stem.dart'; import 'package:stem/src/workflow/core/workflow_clock.dart'; import 'package:test/test.dart'; @@ -11,11 +12,12 @@ void main() { expect(clock.now(), DateTime.parse('2025-01-01T00:00:05Z')); }); - test('SystemWorkflowClock returns current time', () { + test('SystemWorkflowClock respects scoped Stem clock overrides', () { const clock = SystemWorkflowClock(); - final now = DateTime.now(); - final clockNow = clock.now(); + final fake = FakeStemClock(DateTime.parse('2025-01-01T00:00:10Z')); - expect(clockNow.isAfter(now.subtract(const Duration(seconds: 1))), isTrue); + final clockNow = withStemClock(fake, clock.now); + + expect(clockNow, DateTime.parse('2025-01-01T00:00:10Z')); }); } From ab5069130ccaae29061d3aa5d6e730065d59a76d Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 13:27:43 -0500 Subject: [PATCH 07/23] refactor(stem): use shared clock across runtime modules --- .../lib/src/backend/in_memory_backend.dart | 21 ++--- .../stem/lib/src/bootstrap/workflow_app.dart | 5 +- .../lib/src/brokers/in_memory_broker.dart | 17 ++-- packages/stem/lib/src/canvas/canvas.dart | 3 +- .../stem/lib/src/control/revoke_store.dart | 3 +- packages/stem/lib/src/core/contracts.dart | 3 +- packages/stem/lib/src/core/envelope.dart | 3 +- packages/stem/lib/src/core/queue_events.dart | 3 +- packages/stem/lib/src/core/stem.dart | 7 +- .../stem/lib/src/observability/metrics.dart | 7 +- packages/stem/lib/src/scheduler/beat.dart | 9 ++- .../src/scheduler/in_memory_lock_store.dart | 9 ++- .../scheduler/in_memory_schedule_store.dart | 3 +- packages/stem/lib/src/signals/emitter.dart | 3 +- packages/stem/lib/src/signals/payloads.dart | 13 ++-- packages/stem/lib/src/signals/signal.dart | 3 +- packages/stem/lib/src/testing/fake_stem.dart | 5 +- packages/stem/lib/src/worker/worker.dart | 77 +++++++++---------- .../stem/lib/src/workflow/core/run_state.dart | 3 +- .../src/workflow/core/workflow_watcher.dart | 4 +- 20 files changed, 110 insertions(+), 91 deletions(-) diff --git a/packages/stem/lib/src/backend/in_memory_backend.dart b/packages/stem/lib/src/backend/in_memory_backend.dart index 04a9fb37..a635f0af 100644 --- a/packages/stem/lib/src/backend/in_memory_backend.dart +++ b/packages/stem/lib/src/backend/in_memory_backend.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:stem/src/core/chord_metadata.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/observability/heartbeat.dart'; +import 'package:stem/src/core/clock.dart'; /// Simple in-memory result backend used for tests and local development. class InMemoryResultBackend implements ResultBackend { @@ -56,7 +57,7 @@ class InMemoryResultBackend implements ResultBackend { Map meta = const {}, Duration? ttl, }) async { - final now = DateTime.now(); + final now = stemNow(); final existing = _entries[taskId]; final createdAt = existing?.createdAt ?? now; final status = TaskStatus( @@ -70,7 +71,7 @@ class InMemoryResultBackend implements ResultBackend { _entries[taskId] = _Entry( status: status, - expiresAt: DateTime.now().add(ttl ?? defaultTtl), + expiresAt: stemNow().add(ttl ?? defaultTtl), createdAt: createdAt, updatedAt: now, ); @@ -84,7 +85,7 @@ class InMemoryResultBackend implements ResultBackend { Future get(String taskId) async { final entry = _entries[taskId]; if (entry == null) return null; - if (entry.expiresAt.isBefore(DateTime.now())) { + if (entry.expiresAt.isBefore(stemNow())) { _remove(taskId); return null; } @@ -158,7 +159,7 @@ class InMemoryResultBackend implements ResultBackend { Future initGroup(GroupDescriptor descriptor) async { _groups[descriptor.id] = _GroupEntry( descriptor: descriptor, - expiresAt: DateTime.now().add(descriptor.ttl ?? groupDefaultTtl), + expiresAt: stemNow().add(descriptor.ttl ?? groupDefaultTtl), ); _claimedChords.remove(descriptor.id); _scheduleGroupExpiry(descriptor.id, descriptor.ttl ?? groupDefaultTtl); @@ -183,7 +184,7 @@ class InMemoryResultBackend implements ResultBackend { Future getGroup(String groupId) async { final group = _groups[groupId]; if (group == null) return null; - if (group.expiresAt.isBefore(DateTime.now())) { + if (group.expiresAt.isBefore(stemNow())) { _removeGroup(groupId); return null; } @@ -200,7 +201,7 @@ class InMemoryResultBackend implements ResultBackend { Future expire(String taskId, Duration ttl) async { final entry = _entries[taskId]; if (entry == null) return; - entry.expiresAt = DateTime.now().add(ttl); + entry.expiresAt = stemNow().add(ttl); _scheduleExpiry(taskId, ttl); } @@ -230,7 +231,7 @@ class InMemoryResultBackend implements ResultBackend { @override /// Stores or refreshes a worker heartbeat entry. Future setWorkerHeartbeat(WorkerHeartbeat heartbeat) async { - final expiresAt = DateTime.now().add(heartbeatTtl); + final expiresAt = stemNow().add(heartbeatTtl); _heartbeats[heartbeat.workerId] = _HeartbeatEntry( heartbeat: heartbeat, expiresAt: expiresAt, @@ -243,7 +244,7 @@ class InMemoryResultBackend implements ResultBackend { Future getWorkerHeartbeat(String workerId) async { final entry = _heartbeats[workerId]; if (entry == null) return null; - if (entry.expiresAt.isBefore(DateTime.now())) { + if (entry.expiresAt.isBefore(stemNow())) { _removeHeartbeat(workerId); return null; } @@ -330,7 +331,7 @@ class InMemoryResultBackend implements ResultBackend { /// Evicts expired worker heartbeats before listing. void _pruneExpiredHeartbeats() { - final now = DateTime.now(); + final now = stemNow(); _heartbeats.entries .where((entry) => entry.value.expiresAt.isBefore(now)) .map((entry) => entry.key) @@ -340,7 +341,7 @@ class InMemoryResultBackend implements ResultBackend { /// Evicts expired task status entries before listing. void _pruneExpired() { - final now = DateTime.now(); + final now = stemNow(); _entries.entries .where((entry) => entry.value.expiresAt.isBefore(now)) .map((entry) => entry.key) diff --git a/packages/stem/lib/src/bootstrap/workflow_app.dart b/packages/stem/lib/src/bootstrap/workflow_app.dart index 06a10f68..d1b7f5f2 100644 --- a/packages/stem/lib/src/bootstrap/workflow_app.dart +++ b/packages/stem/lib/src/bootstrap/workflow_app.dart @@ -17,6 +17,7 @@ import 'package:stem/src/workflow/core/workflow_store.dart'; import 'package:stem/src/workflow/runtime/workflow_introspection.dart'; import 'package:stem/src/workflow/runtime/workflow_registry.dart'; import 'package:stem/src/workflow/runtime/workflow_runtime.dart'; +import 'package:stem/src/core/clock.dart'; /// Helper that bootstraps a workflow runtime on top of [StemApp]. /// @@ -146,7 +147,7 @@ class StemWorkflowApp { Duration? timeout, T Function(Object? payload)? decode, }) async { - final startedAt = DateTime.now(); + final startedAt = stemNow(); while (true) { final state = await store.get(runId); if (state == null) { @@ -155,7 +156,7 @@ class StemWorkflowApp { if (state.isTerminal) { return _buildResult(state, decode, timedOut: false); } - if (timeout != null && DateTime.now().difference(startedAt) >= timeout) { + if (timeout != null && stemNow().difference(startedAt) >= timeout) { return _buildResult(state, decode, timedOut: true); } await Future.delayed(pollInterval); diff --git a/packages/stem/lib/src/brokers/in_memory_broker.dart b/packages/stem/lib/src/brokers/in_memory_broker.dart index 51941b32..ed710e3a 100644 --- a/packages/stem/lib/src/brokers/in_memory_broker.dart +++ b/packages/stem/lib/src/brokers/in_memory_broker.dart @@ -6,6 +6,7 @@ import 'package:collection/collection.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:uuid/uuid.dart'; +import 'package:stem/src/core/clock.dart'; /// In-memory broker for testing and local development. class InMemoryBroker implements Broker { @@ -19,11 +20,11 @@ class InMemoryBroker implements Broker { _namespaceRefs[namespace] = (_namespaceRefs[namespace] ?? 0) + 1; _delayedTimer = Timer.periodic( delayedInterval, - (_) => _drainDelayed(DateTime.now()), + (_) => _drainDelayed(stemNow()), ); _claimTimer = Timer.periodic( claimInterval, - (_) => _reclaimExpired(DateTime.now()), + (_) => _reclaimExpired(stemNow()), ); } @@ -117,7 +118,7 @@ class InMemoryBroker implements Broker { priority: resolvedRoute.priority ?? envelope.priority, ); - if (msg.notBefore != null && msg.notBefore!.isAfter(DateTime.now())) { + if (msg.notBefore != null && msg.notBefore!.isAfter(stemNow())) { state.addDelayed(msg); } else { state.enqueue(msg); @@ -315,7 +316,7 @@ class InMemoryBroker implements Broker { if (dryRun || selected.isEmpty) { return DeadLetterReplayResult(entries: selected, dryRun: true); } - final now = DateTime.now(); + final now = stemNow(); for (final entry in selected) { state.deadLetters.remove(entry); final replayEnvelope = entry.envelope.copyWith( @@ -444,7 +445,7 @@ class _QueueState { if (_cancelledConsumers.contains(consumer)) { throw _ConsumerCancelled(consumer); } - moveDue(DateTime.now()); + moveDue(stemNow()); final inFlight = _consumerInFlight[consumer] ?? 0; if (inFlight < prefetch && _ready.isNotEmpty) { @@ -454,7 +455,7 @@ class _QueueState { envelope.visibilityTimeout ?? defaultVisibilityTimeout; final expiresAt = visibility == Duration.zero ? null - : DateTime.now().add(visibility); + : stemNow().add(visibility); final delivery = Delivery( envelope: envelope, receipt: receipt, @@ -509,7 +510,7 @@ class _QueueState { envelope: envelope, reason: reason, meta: meta ?? const {}, - deadAt: DateTime.now(), + deadAt: stemNow(), ), ); } @@ -545,7 +546,7 @@ class _QueueState { void extendLease(String receipt, Duration by) { final entry = _pending[receipt]; if (entry == null) return; - entry.leaseExpiresAt = DateTime.now().add(by); + entry.leaseExpiresAt = stemNow().add(by); } /// Clears all queues, in-flight deliveries, and dead letters. diff --git a/packages/stem/lib/src/canvas/canvas.dart b/packages/stem/lib/src/canvas/canvas.dart index e5d99441..260851eb 100644 --- a/packages/stem/lib/src/canvas/canvas.dart +++ b/packages/stem/lib/src/canvas/canvas.dart @@ -8,6 +8,7 @@ import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/task_payload_encoder.dart'; import 'package:stem/src/core/task_result.dart'; import 'package:uuid/uuid.dart'; +import 'package:stem/src/core/clock.dart'; /// Describes a task to schedule along with optional decoder metadata. class TaskSignature { @@ -453,7 +454,7 @@ class Canvas { ); } - final createdAt = DateTime.now().toUtc().toIso8601String(); + final createdAt = stemNow().toUtc().toIso8601String(); await backend.initGroup( GroupDescriptor( id: id, diff --git a/packages/stem/lib/src/control/revoke_store.dart b/packages/stem/lib/src/control/revoke_store.dart index d4fc4338..e6e4bdaa 100644 --- a/packages/stem/lib/src/control/revoke_store.dart +++ b/packages/stem/lib/src/control/revoke_store.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:convert'; +import 'package:stem/src/core/clock.dart'; /// Represents a persisted revoke entry for a task. class RevokeEntry { @@ -120,4 +121,4 @@ abstract class RevokeStore { /// Generates a monotonically increasing revoke version based on UTC time. /// /// Useful for ordering revocation updates across distributed callers. -int generateRevokeVersion() => DateTime.now().toUtc().microsecondsSinceEpoch; +int generateRevokeVersion() => stemNow().toUtc().microsecondsSinceEpoch; diff --git a/packages/stem/lib/src/core/contracts.dart b/packages/stem/lib/src/core/contracts.dart index c74ae271..3fa5a147 100644 --- a/packages/stem/lib/src/core/contracts.dart +++ b/packages/stem/lib/src/core/contracts.dart @@ -38,6 +38,7 @@ import 'package:stem/src/core/task_invocation.dart'; import 'package:stem/src/core/task_payload_encoder.dart'; import 'package:stem/src/observability/heartbeat.dart'; import 'package:stem/src/scheduler/schedule_spec.dart'; +import 'package:stem/src/core/clock.dart'; /// Subscription describing the queues and broadcast channels a worker should /// consume from. @@ -2002,7 +2003,7 @@ class TaskEnqueueBuilder { /// Sets a relative delay before execution. TaskEnqueueBuilder delay(Duration duration) { - _notBefore = DateTime.now().add(duration); + _notBefore = stemNow().add(duration); return this; } diff --git a/packages/stem/lib/src/core/envelope.dart b/packages/stem/lib/src/core/envelope.dart index 6477df03..357200fb 100644 --- a/packages/stem/lib/src/core/envelope.dart +++ b/packages/stem/lib/src/core/envelope.dart @@ -33,6 +33,7 @@ library; import 'dart:convert'; import 'package:uuid/uuid.dart'; +import 'package:stem/src/core/clock.dart'; /// Target classification for routing operations. enum RoutingTargetType { @@ -190,7 +191,7 @@ class Envelope { Map? meta, }) : id = id ?? generateEnvelopeId(), headers = Map.unmodifiable(headers ?? const {}), - enqueuedAt = enqueuedAt ?? DateTime.now(), + enqueuedAt = enqueuedAt ?? stemNow(), meta = Map.unmodifiable(meta ?? const {}); /// Builds an envelope from persisted JSON. diff --git a/packages/stem/lib/src/core/queue_events.dart b/packages/stem/lib/src/core/queue_events.dart index f5004de6..13684f06 100644 --- a/packages/stem/lib/src/core/queue_events.dart +++ b/packages/stem/lib/src/core/queue_events.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/stem_event.dart'; +import 'package:stem/src/core/clock.dart'; const String _queueEventEnvelopeName = '__stem.queue.event__'; const String _queueEventChannelPrefix = 'stem:events'; @@ -99,7 +100,7 @@ class QueueEventsProducer { ); } - final emittedAt = DateTime.now().toUtc(); + final emittedAt = stemNow().toUtc(); final envelope = Envelope( name: _queueEventEnvelopeName, args: { diff --git a/packages/stem/lib/src/core/stem.dart b/packages/stem/lib/src/core/stem.dart index 72129d22..7f25e7ee 100644 --- a/packages/stem/lib/src/core/stem.dart +++ b/packages/stem/lib/src/core/stem.dart @@ -71,6 +71,7 @@ import 'package:stem/src/routing/routing_config.dart'; import 'package:stem/src/routing/routing_registry.dart'; import 'package:stem/src/security/signing.dart'; import 'package:stem/src/signals/emitter.dart'; +import 'package:stem/src/core/clock.dart'; /// Facade used by producer applications to enqueue tasks. class Stem implements TaskEnqueuer { @@ -300,7 +301,7 @@ class Stem implements TaskEnqueuer { ); return envelope.id; } - final expiresAt = claim.computeExpiry(DateTime.now()); + final expiresAt = claim.computeExpiry(stemNow()); envelope = envelope.copyWith( meta: { ...envelope.meta, @@ -462,7 +463,7 @@ class Stem implements TaskEnqueuer { return enqueueOptions.eta; } if (enqueueOptions.countdown != null) { - return DateTime.now().add(enqueueOptions.countdown!); + return stemNow().add(enqueueOptions.countdown!); } return notBefore; } @@ -641,7 +642,7 @@ class Stem implements TaskEnqueuer { } duplicates.add({ 'taskId': duplicate.id, - 'timestamp': DateTime.now().toIso8601String(), + 'timestamp': stemNow().toIso8601String(), 'headers': duplicate.headers, 'meta': duplicate.meta, }); diff --git a/packages/stem/lib/src/observability/metrics.dart b/packages/stem/lib/src/observability/metrics.dart index 57a90ccd..4b891608 100644 --- a/packages/stem/lib/src/observability/metrics.dart +++ b/packages/stem/lib/src/observability/metrics.dart @@ -7,6 +7,7 @@ import 'package:dartastic_opentelemetry/dartastic_opentelemetry.dart' as dotel; import 'package:dartastic_opentelemetry_api/dartastic_opentelemetry_api.dart' as dotel_api; import 'package:meta/meta.dart'; +import 'package:stem/src/core/clock.dart'; /// Known metric aggregation types supported by the exporters. enum MetricType { @@ -31,7 +32,7 @@ class MetricEvent { DateTime? timestamp, this.unit, }) : tags = Map.unmodifiable(tags), - timestamp = (timestamp ?? DateTime.now()).toUtc(), + timestamp = (timestamp ?? stemNow()).toUtc(), attributes = tags.isEmpty ? dotel.Attributes.of(const {}) : dotel.Attributes.of(Map.from(tags)); @@ -285,12 +286,12 @@ class _GaugeState { double value = 0; /// Timestamp when [value] was last updated. - DateTime updatedAt = DateTime.now().toUtc(); + DateTime updatedAt = stemNow().toUtc(); /// Sets the gauge [value] to [newValue]. void update(double newValue) { value = newValue; - updatedAt = DateTime.now().toUtc(); + updatedAt = stemNow().toUtc(); } /// Serializes the gauge aggregate. diff --git a/packages/stem/lib/src/scheduler/beat.dart b/packages/stem/lib/src/scheduler/beat.dart index a083d9fa..3295f7f1 100644 --- a/packages/stem/lib/src/scheduler/beat.dart +++ b/packages/stem/lib/src/scheduler/beat.dart @@ -23,6 +23,7 @@ import 'package:stem/src/observability/logging.dart'; import 'package:stem/src/observability/metrics.dart'; import 'package:stem/src/security/signing.dart'; import 'package:stem/src/signals/emitter.dart'; +import 'package:stem/src/core/clock.dart'; /// Scheduler loop that dispatches due [ScheduleEntry] records. /// @@ -106,7 +107,7 @@ class Beat { /// 4. Dispatches each entry via [_dispatch]. Future _tick() async { if (!_running && _timer != null) return; - final now = DateTime.now(); + final now = stemNow(); final dueEntries = await store.due(now); StemMetrics.instance.setGauge( 'stem.scheduler.due.entries', @@ -214,7 +215,7 @@ class Beat { ? Duration(milliseconds: _random.nextInt(jitter.inMilliseconds + 1)) : Duration.zero; final scheduledFor = baseScheduled.add(jitterDelay); - final startedAt = DateTime.now(); + final startedAt = stemNow(); StemMetrics.instance.increment( 'stem.scheduler.dispatch.attempts', @@ -239,7 +240,7 @@ class Beat { } await broker.publish(envelope); - final executedAt = DateTime.now(); + final executedAt = stemNow(); final duration = executedAt.difference(startedAt); StemMetrics.instance.recordDuration( 'stem.scheduler.dispatch.duration', @@ -293,7 +294,7 @@ class Beat { ), ); try { - final executedAt = DateTime.now(); + final executedAt = stemNow(); await store.markExecuted( entry.id, scheduledFor: scheduledFor, diff --git a/packages/stem/lib/src/scheduler/in_memory_lock_store.dart b/packages/stem/lib/src/scheduler/in_memory_lock_store.dart index 8e435941..97c9dccb 100644 --- a/packages/stem/lib/src/scheduler/in_memory_lock_store.dart +++ b/packages/stem/lib/src/scheduler/in_memory_lock_store.dart @@ -9,6 +9,7 @@ import 'dart:async'; import 'package:stem/src/core/contracts.dart'; import 'package:uuid/uuid.dart'; +import 'package:stem/src/core/clock.dart'; /// In-memory lock store used for tests and local scheduling. /// @@ -23,7 +24,7 @@ class InMemoryLockStore implements LockStore { Duration ttl = const Duration(seconds: 30), String? owner, }) async { - final now = DateTime.now(); + final now = stemNow(); final existing = _locks[key]; // Check if a non-expired lock already exists for this key. @@ -42,7 +43,7 @@ class InMemoryLockStore implements LockStore { Future ownerOf(String key) async { final lock = _locks[key]; if (lock == null) return null; - if (lock.isExpired(DateTime.now())) { + if (lock.isExpired(stemNow())) { _locks.remove(key); return null; } @@ -59,11 +60,11 @@ class InMemoryLockStore implements LockStore { if (lock.owner != owner) { return false; } - if (lock.isExpired(DateTime.now())) { + if (lock.isExpired(stemNow())) { _locks.remove(key); return false; } - lock.expiresAt = DateTime.now().add(ttl); + lock.expiresAt = stemNow().add(ttl); return true; } diff --git a/packages/stem/lib/src/scheduler/in_memory_schedule_store.dart b/packages/stem/lib/src/scheduler/in_memory_schedule_store.dart index 02c370a8..42d19297 100644 --- a/packages/stem/lib/src/scheduler/in_memory_schedule_store.dart +++ b/packages/stem/lib/src/scheduler/in_memory_schedule_store.dart @@ -8,6 +8,7 @@ library; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/scheduler/schedule_calculator.dart'; import 'package:stem/src/scheduler/schedule_spec.dart'; +import 'package:stem/src/core/clock.dart'; /// Simple in-memory schedule store implementation used in tests. /// @@ -57,7 +58,7 @@ class InMemoryScheduleStore implements ScheduleStore { @override /// Inserts or updates a schedule entry, recomputing its next run. Future upsert(ScheduleEntry entry) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); var next = entry.nextRunAt; if (entry.enabled) { next ??= _calculator.nextRun( diff --git a/packages/stem/lib/src/signals/emitter.dart b/packages/stem/lib/src/signals/emitter.dart index a3af95ce..0bc3564e 100644 --- a/packages/stem/lib/src/signals/emitter.dart +++ b/packages/stem/lib/src/signals/emitter.dart @@ -3,6 +3,7 @@ import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/signals/payloads.dart'; import 'package:stem/src/signals/stem_signals.dart'; +import 'package:stem/src/core/clock.dart'; /// Helper used by coordinators, workers, and middleware to emit strongly /// typed Stem signals without duplicating payload construction. @@ -101,7 +102,7 @@ class StemSignalEmitter { required DateTime nextRetryAt, String? sender, }) { - final emittedAt = DateTime.now().toUtc(); + final emittedAt = stemNow().toUtc(); return StemSignals.taskRetry.emit( TaskRetryPayload( envelope: envelope, diff --git a/packages/stem/lib/src/signals/payloads.dart b/packages/stem/lib/src/signals/payloads.dart index 7249c1d4..1c1222da 100644 --- a/packages/stem/lib/src/signals/payloads.dart +++ b/packages/stem/lib/src/signals/payloads.dart @@ -2,6 +2,7 @@ import 'package:stem/src/control/control_messages.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/stem_event.dart'; +import 'package:stem/src/core/clock.dart'; /// Status of a workflow run emitted via signals. enum WorkflowRunStatus { @@ -238,7 +239,7 @@ class TaskRetryPayload implements StemEvent { required this.reason, required this.nextRetryAt, DateTime? emittedAt, - }) : emittedAt = (emittedAt ?? DateTime.now()).toUtc(); + }) : emittedAt = (emittedAt ?? stemNow()).toUtc(); /// The task envelope to be retried. final Envelope envelope; @@ -417,7 +418,7 @@ class WorkerLifecyclePayload implements StemEvent { this.reason, this.signalName = 'worker-lifecycle', DateTime? timestamp, - }) : _occurredAt = (timestamp ?? DateTime.now()).toUtc(); + }) : _occurredAt = (timestamp ?? stemNow()).toUtc(); /// The worker involved in the lifecycle event. final WorkerInfo worker; @@ -479,7 +480,7 @@ class WorkerChildLifecyclePayload implements StemEvent { required this.isolateId, this.signalName = 'worker-child-lifecycle', DateTime? timestamp, - }) : _occurredAt = (timestamp ?? DateTime.now()).toUtc(); + }) : _occurredAt = (timestamp ?? stemNow()).toUtc(); /// The parent worker managing the child isolate. final WorkerInfo worker; @@ -516,7 +517,7 @@ class WorkflowRunPayload implements StemEvent { this.metadata = const {}, this.signalName, DateTime? occurredAt, - }) : _occurredAt = (occurredAt ?? DateTime.now()).toUtc(); + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The unique identifier for the workflow run. final String runId; @@ -681,7 +682,7 @@ class ControlCommandReceivedPayload implements StemEvent { String get eventName => 'control-command-received'; @override - DateTime get occurredAt => DateTime.now().toUtc(); + DateTime get occurredAt => stemNow().toUtc(); @override Map get attributes => { @@ -724,7 +725,7 @@ class ControlCommandCompletedPayload implements StemEvent { String get eventName => 'control-command-completed'; @override - DateTime get occurredAt => DateTime.now().toUtc(); + DateTime get occurredAt => stemNow().toUtc(); @override Map get attributes => { diff --git a/packages/stem/lib/src/signals/signal.dart b/packages/stem/lib/src/signals/signal.dart index 1258678a..a96643a3 100644 --- a/packages/stem/lib/src/signals/signal.dart +++ b/packages/stem/lib/src/signals/signal.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:stem/src/core/stem_event.dart'; +import 'package:stem/src/core/clock.dart'; /// Signature for signal handlers. typedef SignalHandler = @@ -18,7 +19,7 @@ class SignalContext { this.sender, DateTime? timestamp, this.event, - }) : timestamp = timestamp ?? DateTime.now(); + }) : timestamp = timestamp ?? stemNow(); /// Signal identifier. final String name; diff --git a/packages/stem/lib/src/testing/fake_stem.dart b/packages/stem/lib/src/testing/fake_stem.dart index 3ce9bdfe..71c3fa0a 100644 --- a/packages/stem/lib/src/testing/fake_stem.dart +++ b/packages/stem/lib/src/testing/fake_stem.dart @@ -2,6 +2,7 @@ import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/stem.dart' show Stem; import 'package:stem/stem.dart' show Stem; +import 'package:stem/src/core/clock.dart'; /// Record describing an enqueued task captured by [FakeStem]. class RecordedEnqueue { @@ -74,7 +75,7 @@ class FakeStem { options: options, notBefore: call.notBefore, meta: Map.from(call.meta), - enqueuedAt: DateTime.now(), + enqueuedAt: stemNow(), call: call, ), ); @@ -100,7 +101,7 @@ class FakeStem { options: options, notBefore: notBefore, meta: Map.from(meta), - enqueuedAt: DateTime.now(), + enqueuedAt: stemNow(), ), ); return id; diff --git a/packages/stem/lib/src/worker/worker.dart b/packages/stem/lib/src/worker/worker.dart index dde5a086..232ddbe2 100644 --- a/packages/stem/lib/src/worker/worker.dart +++ b/packages/stem/lib/src/worker/worker.dart @@ -124,6 +124,7 @@ import 'package:stem/src/signals/emitter.dart'; import 'package:stem/src/signals/payloads.dart'; import 'package:stem/src/worker/isolate_pool.dart'; import 'package:stem/src/worker/worker_config.dart'; +import 'package:stem/src/core/clock.dart'; /// Shutdown modes for workers. /// @@ -621,7 +622,7 @@ class Worker { _lastScaleUp = null; _lastScaleDown = null; _drainCompleter = null; - _startedAt ??= DateTime.now().toUtc(); + _startedAt ??= stemNow().toUtc(); _startedCount = 0; _completedCount = 0; _failedCount = 0; @@ -919,7 +920,7 @@ class Worker { _startedCount += 1; String? startedAtIso; - final startedAt = DateTime.now().toUtc(); + final startedAt = stemNow().toUtc(); final runningMeta = _statusMeta( envelope, resultEncoder, @@ -1000,7 +1001,7 @@ class Worker { extra: { 'queue': envelope.queue, 'worker': consumerName, - 'completedAt': DateTime.now().toIso8601String(), + 'completedAt': stemNow().toIso8601String(), 'startedAt': startedAtIso, }, ); @@ -1097,7 +1098,7 @@ class Worker { softTimer?.cancel(); final completed = _releaseDelivery(envelope); if (completed != null) { - final duration = DateTime.now().toUtc().difference( + final duration = stemNow().toUtc().difference( completed.startedAt, ); StemMetrics.instance.recordDuration( @@ -1212,9 +1213,9 @@ class Worker { /// /// ```dart /// // TimingMiddleware.onExecute: - /// final start = DateTime.now(); + /// final start = stemNow(); /// await next(); // Inner middleware and handler run here - /// final duration = DateTime.now().difference(start); + /// final duration = stemNow().difference(start); /// log('Task took $duration'); /// ``` /// @@ -1346,7 +1347,7 @@ class Worker { void _scheduleLeaseRenewal(Delivery delivery) { final expiresAt = delivery.leaseExpiresAt; if (expiresAt == null) return; - final remainingMs = expiresAt.difference(DateTime.now()).inMilliseconds; + final remainingMs = expiresAt.difference(stemNow()).inMilliseconds; if (remainingMs <= 0) return; final interval = Duration( milliseconds: (remainingMs ~/ 2).clamp(1000, 30000), @@ -1451,7 +1452,7 @@ class Worker { /// - This method: updates timestamps for observability /// - [_recordLeaseRenewal]: emits metrics to the metrics backend void _noteLeaseRenewal(Delivery delivery) { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); _lastLeaseRenewal = now; final active = _activeDeliveries[delivery.envelope.id]; if (active != null) { @@ -1583,7 +1584,7 @@ class Worker { } final resultsPayload = status.results.values.map((s) => s.payload).toList(); - final dispatchedAt = DateTime.now().toUtc(); + final dispatchedAt = stemNow().toUtc(); final callbackTaskId = (callbackData['id'] as String?) ?? generateEnvelopeId(); @@ -1889,7 +1890,7 @@ class Worker { extra: { 'queue': envelope.queue, 'worker': consumerName, - 'failedAt': DateTime.now().toIso8601String(), + 'failedAt': stemNow().toIso8601String(), 'security': 'signature-invalid', }, ); @@ -2025,13 +2026,13 @@ class Worker { stack, retryPolicy, ); - final nextRunAt = DateTime.now().add(delay); + final nextRunAt = stemNow().add(delay); await broker.nack(delivery, requeue: false); await broker.publish( envelope.copyWith( attempt: envelope.attempt + 1, maxRetries: maxRetries, - notBefore: DateTime.now().add(delay), + notBefore: stemNow().add(delay), ), ); final retriedMeta = _statusMeta( @@ -2078,7 +2079,7 @@ class Worker { extra: { 'queue': envelope.queue, 'worker': consumerName, - 'failedAt': DateTime.now().toIso8601String(), + 'failedAt': stemNow().toIso8601String(), 'startedAt': startedAtIso, }, ); @@ -2166,7 +2167,7 @@ class Worker { extra: { 'queue': envelope.queue, 'worker': consumerName, - 'failedAt': DateTime.now().toIso8601String(), + 'failedAt': stemNow().toIso8601String(), 'retryExhausted': true, }, ); @@ -2202,18 +2203,16 @@ class Worker { final scheduledAt = request.eta ?? - (request.countdown != null - ? DateTime.now().add(request.countdown!) - : null); + (request.countdown != null ? stemNow().add(request.countdown!) : null); final delay = scheduledAt != null - ? scheduledAt.difference(DateTime.now()) + ? scheduledAt.difference(stemNow()) : _computeRetryDelay( envelope.attempt, request, StackTrace.current, policy, ); - final notBefore = scheduledAt ?? DateTime.now().add(delay); + final notBefore = scheduledAt ?? stemNow().add(delay); final updatedMeta = Map.from(envelope.meta); if (request.timeLimit != null) { @@ -2299,7 +2298,7 @@ class Worker { }) async { await broker.nack(delivery, requeue: false); await broker.publish( - envelope.copyWith(notBefore: DateTime.now().add(backoff)), + envelope.copyWith(notBefore: stemNow().add(backoff)), ); final data = { ...extra, @@ -2365,7 +2364,7 @@ class Worker { final envelope = delivery.envelope; final id = envelope.id; final queueName = envelope.queue; - final startedAt = DateTime.now().toUtc(); + final startedAt = stemNow().toUtc(); _activeDeliveries[id] = _ActiveDelivery( queue: queueName, startedAt: startedAt, @@ -2611,7 +2610,7 @@ class Worker { /// Requests cooperative termination for all active tasks. void _requestTerminationForActiveTasks({required String reason}) { if (_activeDeliveries.isEmpty) return; - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); var version = generateRevokeVersion(); for (final active in _activeDeliveries.values) { final entry = RevokeEntry( @@ -2735,7 +2734,7 @@ class Worker { _lastQueueDepth = depth; } final inflight = _inflight; - final now = DateTime.now(); + final now = stemNow(); final configuredMax = autoscaleConfig.maxConcurrency ?? _maxConcurrency; final maxAllowed = configuredMax < _maxConcurrency ? configuredMax @@ -2967,7 +2966,7 @@ class Worker { /// Builds a worker heartbeat payload from current runtime state. WorkerHeartbeat _buildHeartbeat() { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final isolatePool = _isolatePool; final activeIsolates = isolatePool?.activeCount ?? math.min(_inflight, _currentConcurrency); @@ -3138,7 +3137,7 @@ class Worker { Future _syncRevocations() async { final store = revokeStore; if (store == null) return; - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); try { final fetched = await store.list(namespace); for (final entry in fetched) { @@ -3195,7 +3194,7 @@ class Worker { void _applyQueuePauseEntry(RevokeEntry entry, {DateTime? clock}) { final queueName = _queueNameFromPauseTaskId(entry.taskId); if (queueName == null) return; - final now = clock ?? DateTime.now().toUtc(); + final now = clock ?? stemNow().toUtc(); if (entry.isExpired(now)) { _queuePauses.remove(queueName); return; @@ -3209,7 +3208,7 @@ class Worker { bool _isQueuePaused(String queueName) { final entry = _queuePauses[queueName]; if (entry == null) return false; - if (entry.isExpired(DateTime.now().toUtc())) { + if (entry.isExpired(stemNow().toUtc())) { _queuePauses.remove(queueName); return false; } @@ -3217,7 +3216,7 @@ class Worker { } List _pausedQueueNames() { - _pruneExpiredQueuePauses(DateTime.now().toUtc()); + _pruneExpiredQueuePauses(stemNow().toUtc()); final queues = _queuePauses.keys.toList()..sort(); return queues; } @@ -3225,7 +3224,7 @@ class Worker { RevokeEntry? _revocationFor(String taskId) { final entry = _revocations[taskId]; if (entry == null) return null; - if (entry.isExpired(DateTime.now().toUtc())) { + if (entry.isExpired(stemNow().toUtc())) { _revocations.remove(taskId); return null; } @@ -3246,7 +3245,7 @@ class Worker { /// Applies a revocation entry to the local cache. void _applyRevocationEntry(RevokeEntry entry, {DateTime? clock}) { - final now = clock ?? DateTime.now().toUtc(); + final now = clock ?? stemNow().toUtc(); if (entry.isExpired(now)) { _revocations.remove(entry.taskId); return; @@ -3560,7 +3559,7 @@ class Worker { ? value : DateTime.tryParse(value.toString()); if (expiresAt == null) return false; - return DateTime.now().isAfter(expiresAt); + return stemNow().isAfter(expiresAt); } /// Marks expired deliveries and acknowledges them. @@ -3576,7 +3575,7 @@ class Worker { extra: { 'queue': envelope.queue, 'worker': consumerName, - 'expiredAt': DateTime.now().toIso8601String(), + 'expiredAt': stemNow().toIso8601String(), 'stem.expired': true, }, ); @@ -3604,7 +3603,7 @@ class Worker { : namespace; final rawRevocations = (payload['revocations'] as List?) ?? const []; final requester = (payload['requester'] as String?)?.trim(); - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final entries = []; for (final raw in rawRevocations) { @@ -3671,7 +3670,7 @@ class Worker { }; } - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final applied = []; final inflight = []; final ignored = []; @@ -3765,7 +3764,7 @@ class Worker { throw StateError('Queue control command requires at least one queue.'); } - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final requester = (payload['requester'] as String?)?.trim(); final reason = (payload['reason'] as String?)?.trim(); final baseVersion = generateRevokeVersion(); @@ -3886,7 +3885,7 @@ class Worker { workerId: _workerIdentifier, status: 'ok', payload: { - 'timestamp': DateTime.now().toUtc().toIso8601String(), + 'timestamp': stemNow().toUtc().toIso8601String(), 'queue': primaryQueue, 'inflight': _inflight, 'subscriptions': _subscriptionMetadata(), @@ -4028,7 +4027,7 @@ class Worker { /// Collects metrics from the broker, active deliveries, and the isolate pool /// to provide a comprehensive view of worker health. Map _buildStatsSnapshot() { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final activeTasks = _activeDeliveries.entries.map((entry) { final delivery = entry.value; final envelope = delivery.envelope; @@ -4069,7 +4068,7 @@ class Worker { /// Builds a detailed inspect snapshot for control commands. Map _buildInspectSnapshot({bool includeRevoked = true}) { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final active = _activeDeliveries.values.map((delivery) { final envelope = delivery.envelope; final runtime = now.difference(delivery.startedAt); @@ -4312,7 +4311,7 @@ class WorkerEvent implements StemEvent { this.stackTrace, this.progress, this.data, - }) : timestamp = timestamp ?? DateTime.now(); + }) : timestamp = timestamp ?? stemNow(); /// The type of event. final WorkerEventType type; diff --git a/packages/stem/lib/src/workflow/core/run_state.dart b/packages/stem/lib/src/workflow/core/run_state.dart index 3f474b4a..06df3cc5 100644 --- a/packages/stem/lib/src/workflow/core/run_state.dart +++ b/packages/stem/lib/src/workflow/core/run_state.dart @@ -3,6 +3,7 @@ import 'package:stem/src/workflow/core/workflow_status.dart'; import 'package:stem/src/workflow/core/workflow_store.dart' show WorkflowStore; import 'package:stem/src/workflow/workflow.dart' show WorkflowStore; import 'package:stem/stem.dart' show WorkflowStore; +import 'package:stem/src/core/clock.dart'; /// Snapshot of a workflow run persisted by a [WorkflowStore]. /// @@ -37,7 +38,7 @@ class RunState { status: _statusFromJson(json['status']), cursor: _intFromJson(json['cursor']), params: (json['params'] as Map?)?.cast() ?? const {}, - createdAt: _dateFromJson(json['createdAt']) ?? DateTime.now().toUtc(), + createdAt: _dateFromJson(json['createdAt']) ?? stemNow().toUtc(), result: json['result'], waitTopic: json['waitTopic'] as String?, resumeAt: _dateFromJson(json['resumeAt']), diff --git a/packages/stem/lib/src/workflow/core/workflow_watcher.dart b/packages/stem/lib/src/workflow/core/workflow_watcher.dart index 93313d9c..077b2872 100644 --- a/packages/stem/lib/src/workflow/core/workflow_watcher.dart +++ b/packages/stem/lib/src/workflow/core/workflow_watcher.dart @@ -1,3 +1,5 @@ +import 'package:stem/src/core/clock.dart'; + /// Describes a workflow event watcher registered by the runtime. class WorkflowWatcher { /// Creates a watcher entry for a suspended workflow run. @@ -16,7 +18,7 @@ class WorkflowWatcher { runId: json['runId']?.toString() ?? '', stepName: json['stepName']?.toString() ?? '', topic: json['topic']?.toString() ?? '', - createdAt: _dateFromJson(json['createdAt']) ?? DateTime.now().toUtc(), + createdAt: _dateFromJson(json['createdAt']) ?? stemNow().toUtc(), deadline: _dateFromJson(json['deadline']), data: (json['data'] as Map?)?.cast() ?? const {}, ); From 599c43863e8973438251d561fb213b9497992c21 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 13:27:50 -0500 Subject: [PATCH 08/23] refactor(adapters): route backend time reads through stem clock --- .../lib/src/backend/postgres_backend.dart | 22 +++++++++---------- .../lib/src/brokers/postgres_broker.dart | 20 ++++++++--------- .../src/control/postgres_revoke_store.dart | 2 +- .../src/scheduler/postgres_lock_store.dart | 6 ++--- .../scheduler/postgres_schedule_store.dart | 4 ++-- .../lib/src/backend/redis_backend.dart | 2 +- .../lib/src/brokers/redis_broker.dart | 19 +++++++--------- .../src/scheduler/redis_schedule_store.dart | 2 +- .../src/backend/sqlite_result_backend.dart | 22 +++++++++---------- .../lib/src/broker/sqlite_broker.dart | 22 +++++++++---------- 10 files changed, 59 insertions(+), 62 deletions(-) diff --git a/packages/stem_postgres/lib/src/backend/postgres_backend.dart b/packages/stem_postgres/lib/src/backend/postgres_backend.dart index fdffe2cb..c833ae95 100644 --- a/packages/stem_postgres/lib/src/backend/postgres_backend.dart +++ b/packages/stem_postgres/lib/src/backend/postgres_backend.dart @@ -109,7 +109,7 @@ class PostgresResultBackend implements ResultBackend { Future _cleanup() async { if (_closed) return; try { - final now = DateTime.now(); + final now = stemNow(); await _connections.runInTransaction((txn) async { await txn .query() @@ -168,7 +168,7 @@ class PostgresResultBackend implements ResultBackend { meta: meta, ); - final expiresAt = DateTime.now().add(ttl ?? defaultTtl); + final expiresAt = stemNow().add(ttl ?? defaultTtl); await _connections.runInTransaction((txn) async { final model = $StemTaskResult( id: taskId, @@ -188,7 +188,7 @@ class PostgresResultBackend implements ResultBackend { @override Future get(String taskId) async { - final now = DateTime.now(); + final now = stemNow(); final row = await _connections.runInTransaction( (context) async { return context @@ -240,7 +240,7 @@ class PostgresResultBackend implements ResultBackend { if (request.limit <= 0) { return const TaskStatusPage(items: []); } - final now = DateTime.now(); + final now = stemNow(); final matches = []; var scanOffset = 0; final target = request.offset + request.limit; @@ -299,7 +299,7 @@ class PostgresResultBackend implements ResultBackend { @override Future initGroup(GroupDescriptor descriptor) async { - final now = DateTime.now(); + final now = stemNow(); final expiresAt = now.add(descriptor.ttl ?? groupDefaultTtl); await _connections.runInTransaction((txn) async { final repository = txn.repository(); @@ -362,7 +362,7 @@ class PostgresResultBackend implements ResultBackend { @override Future expire(String taskId, Duration ttl) async { - final expiresAt = DateTime.now().add(ttl); + final expiresAt = stemNow().add(ttl); await _context.repository().update( StemTaskResultUpdateDto(expiresAt: expiresAt), @@ -372,7 +372,7 @@ class PostgresResultBackend implements ResultBackend { @override Future setWorkerHeartbeat(WorkerHeartbeat heartbeat) async { - final expiresAt = DateTime.now().add(heartbeatTtl); + final expiresAt = stemNow().add(heartbeatTtl); await _connections.runInTransaction((txn) async { final model = StemWorkerHeartbeat( workerId: heartbeat.workerId, @@ -401,7 +401,7 @@ class PostgresResultBackend implements ResultBackend { @override Future getWorkerHeartbeat(String workerId) async { - final now = DateTime.now(); + final now = stemNow(); final row = await _context .query() .whereEquals('workerId', workerId) @@ -414,7 +414,7 @@ class PostgresResultBackend implements ResultBackend { @override Future> listWorkerHeartbeats() async { - final now = DateTime.now(); + final now = stemNow(); final rows = await _context .query() .whereEquals('namespace', namespace) @@ -457,7 +457,7 @@ class PostgresResultBackend implements ResultBackend { // Removed legacy JSON decoder (not needed with Ormed models) Future _groupExists(String groupId) async { - final now = DateTime.now(); + final now = stemNow(); return _context .query() .whereEquals('id', groupId) @@ -467,7 +467,7 @@ class PostgresResultBackend implements ResultBackend { } Future _readGroup(String groupId) async { - final now = DateTime.now(); + final now = stemNow(); final groupRow = await _context .query() .whereEquals('id', groupId) diff --git a/packages/stem_postgres/lib/src/brokers/postgres_broker.dart b/packages/stem_postgres/lib/src/brokers/postgres_broker.dart index 4426f3fa..3b759649 100644 --- a/packages/stem_postgres/lib/src/brokers/postgres_broker.dart +++ b/packages/stem_postgres/lib/src/brokers/postgres_broker.dart @@ -325,7 +325,7 @@ class PostgresBroker implements Broker { 'queue': delivery.envelope.queue, }), ); - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); await _withDb(() { return _context .query() @@ -356,7 +356,7 @@ class PostgresBroker implements Broker { final entryReason = (reason == null || reason.trim().isEmpty) ? 'unknown' : reason.trim(); - final deadAt = DateTime.now().toUtc(); + final deadAt = stemNow().toUtc(); await _withDb(() async { await _connections.runInTransaction((txn) async { @@ -403,7 +403,7 @@ class PostgresBroker implements Broker { Future extendLease(Delivery delivery, Duration by) async { if (by <= Duration.zero) return; final jobId = _parseReceipt(delivery.receipt); - final leaseUntil = DateTime.now().toUtc().add(by); + final leaseUntil = stemNow().toUtc().add(by); await _withDb(() { return _context.repository().update( StemQueueJobUpdateDto(lockedUntil: leaseUntil), @@ -414,7 +414,7 @@ class PostgresBroker implements Broker { @override Future pendingCount(String queue) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); return _withDb(() { return _context .query() @@ -436,7 +436,7 @@ class PostgresBroker implements Broker { @override Future inflightCount(String queue) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); return _withDb(() { return _context .query() @@ -523,7 +523,7 @@ class PostgresBroker implements Broker { final updatedEnvelope = delay == null ? entry.envelope : entry.envelope.copyWith( - notBefore: DateTime.now().toUtc().add(delay), + notBefore: stemNow().toUtc().add(delay), ); await _insertJob( txn, @@ -587,7 +587,7 @@ class PostgresBroker implements Broker { } Future<_QueuedJob?> _claimNextJob(String queue, String consumerId) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final visibilityUntil = now.add(defaultVisibilityTimeout); return _withDb(() { @@ -707,7 +707,7 @@ class PostgresBroker implements Broker { messageId: messageId, workerId: workerId, namespace: namespace, - acknowledgedAt: DateTime.now().toUtc(), + acknowledgedAt: stemNow().toUtc(), ).toTracked(); await _withDb(() { return _context.repository().upsert( @@ -726,7 +726,7 @@ class PostgresBroker implements Broker { } Future _runSweeperCycle() async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); await _withDb(() async { await _connections.runInTransaction((txn) async { await txn @@ -863,7 +863,7 @@ class _ConsumerRunner { await Future.delayed(broker.pollInterval); continue; } - final leaseExpiresAt = DateTime.now().toUtc().add( + final leaseExpiresAt = stemNow().toUtc().add( broker.defaultVisibilityTimeout, ); for (final delivery in [ diff --git a/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart b/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart index 975d8c63..9bdae55c 100644 --- a/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart +++ b/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart @@ -128,7 +128,7 @@ class PostgresRevokeStore implements RevokeStore { issuedAt: entry.issuedAt, expiresAt: entry.expiresAt, version: existing != null ? entry.version : entry.version, - updatedAt: DateTime.now(), + updatedAt: stemNow(), ); if (existing != null) { diff --git a/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart b/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart index 8fcd88eb..d05af002 100644 --- a/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart +++ b/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart @@ -70,7 +70,7 @@ class PostgresLockStore implements LockStore { String? owner, }) async { final ownerValue = _owner(owner); - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final expiresAt = now.add(ttl); final ctx = _connections.context; final repository = ctx.repository<$StemLock>(); @@ -116,7 +116,7 @@ class PostgresLockStore implements LockStore { } Future _renew(String key, String owner, Duration ttl) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final expiresAt = now.add(ttl); final ctx = _connections.context; @@ -164,7 +164,7 @@ class PostgresLockStore implements LockStore { @override Future ownerOf(String key) async { final ctx = _connections.context; - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final locks = await ctx .query<$StemLock>() .whereEquals('key', key) diff --git a/packages/stem_postgres/lib/src/scheduler/postgres_schedule_store.dart b/packages/stem_postgres/lib/src/scheduler/postgres_schedule_store.dart index 8383c7a2..3d7c991f 100644 --- a/packages/stem_postgres/lib/src/scheduler/postgres_schedule_store.dart +++ b/packages/stem_postgres/lib/src/scheduler/postgres_schedule_store.dart @@ -123,7 +123,7 @@ class PostgresScheduleStore implements ScheduleStore { /// Inserts or updates a schedule [entry] within the configured namespace. @override Future upsert(ScheduleEntry entry) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final ctx = _connections.context; final existing = await ctx @@ -252,7 +252,7 @@ class PostgresScheduleStore implements ScheduleStore { expireAt: entry.expireAt, meta: entry.meta, createdAt: entry.createdAt, - updatedAt: DateTime.now().toUtc(), + updatedAt: stemNow().toUtc(), version: entry.version, ), ); diff --git a/packages/stem_redis/lib/src/backend/redis_backend.dart b/packages/stem_redis/lib/src/backend/redis_backend.dart index 02379c32..c8bc925a 100644 --- a/packages/stem_redis/lib/src/backend/redis_backend.dart +++ b/packages/stem_redis/lib/src/backend/redis_backend.dart @@ -139,7 +139,7 @@ class RedisResultBackend implements ResultBackend { meta: meta, ); final key = _taskKey(taskId); - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); var createdAt = now; final existingRaw = await _send(['GET', key]); if (existingRaw is String) { diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 2060390f..8f3a4703 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -364,8 +364,7 @@ class RedisStreamsBroker implements Broker { queue: target, priority: resolvedRoute.priority ?? envelope.priority, ); - if (message.notBefore != null && - message.notBefore!.isAfter(DateTime.now())) { + if (message.notBefore != null && message.notBefore!.isAfter(stemNow())) { await _send([ 'ZADD', _delayedKey(target), @@ -428,7 +427,7 @@ class RedisStreamsBroker implements Broker { } Future _drainDelayed(String queue) async { - final nowMs = DateTime.now().millisecondsSinceEpoch; + final nowMs = stemNow().millisecondsSinceEpoch; dynamic result; try { result = await _send([ @@ -789,7 +788,7 @@ class RedisStreamsBroker implements Broker { receipt: receipt, leaseExpiresAt: lease == Duration.zero ? null - : DateTime.now().add(lease), + : stemNow().add(lease), route: route, ), ); @@ -837,9 +836,7 @@ class RedisStreamsBroker implements Broker { Delivery( envelope: envelope, receipt: receipt, - leaseExpiresAt: lease == Duration.zero - ? null - : DateTime.now().add(lease), + leaseExpiresAt: lease == Duration.zero ? null : stemNow().add(lease), route: route, ), ); @@ -855,7 +852,7 @@ class RedisStreamsBroker implements Broker { headers: (jsonDecode(map['headers'] ?? '{}') as Map) .cast(), enqueuedAt: DateTime.parse( - map['enqueuedAt'] ?? DateTime.now().toIso8601String(), + map['enqueuedAt'] ?? stemNow().toIso8601String(), ), notBefore: (map['notBefore']?.isEmpty ?? true) ? null @@ -965,7 +962,7 @@ class RedisStreamsBroker implements Broker { 'envelope': delivery.envelope.toJson(), 'reason': reason, 'meta': meta, - 'deadAt': DateTime.now().toIso8601String(), + 'deadAt': stemNow().toIso8601String(), }), ]); } @@ -1033,7 +1030,7 @@ class RedisStreamsBroker implements Broker { dryRun: true, ); } - final now = DateTime.now(); + final now = stemNow(); for (final candidate in selected) { await _send(['LREM', _deadKey(queue), '1', candidate.raw]); final replayEnvelope = candidate.entry.envelope.copyWith( @@ -1091,7 +1088,7 @@ class RedisStreamsBroker implements Broker { Future extendLease(Delivery delivery, Duration by) async { final info = _parseReceipt(delivery.receipt); await _send(['XACK', info.stream, info.group, info.id]); - final nextVisibleAt = DateTime.now().add(by); + final nextVisibleAt = stemNow().add(by); final delayedEnvelope = delivery.envelope.copyWith( notBefore: nextVisibleAt, ); diff --git a/packages/stem_redis/lib/src/scheduler/redis_schedule_store.dart b/packages/stem_redis/lib/src/scheduler/redis_schedule_store.dart index 2734a8ec..a5b6e685 100644 --- a/packages/stem_redis/lib/src/scheduler/redis_schedule_store.dart +++ b/packages/stem_redis/lib/src/scheduler/redis_schedule_store.dart @@ -218,7 +218,7 @@ class RedisScheduleStore implements ScheduleStore { @override Future upsert(ScheduleEntry entry) async { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); var nextRun = entry.nextRunAt; if (entry.enabled) { nextRun ??= _calculator.nextRun( diff --git a/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart b/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart index 2a27fee0..767a596a 100644 --- a/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart +++ b/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart @@ -171,7 +171,7 @@ class SqliteResultBackend implements ResultBackend { Map meta = const {}, Duration? ttl, }) async { - final now = DateTime.now(); + final now = stemNow(); final expiresAt = now.add(ttl ?? defaultTtl); final status = TaskStatus( id: taskId, @@ -201,7 +201,7 @@ class SqliteResultBackend implements ResultBackend { @override Future get(String taskId) async { - final now = DateTime.now(); + final now = stemNow(); final row = await _context .query() .whereEquals('id', taskId) @@ -237,7 +237,7 @@ class SqliteResultBackend implements ResultBackend { if (request.limit <= 0) { return const TaskStatusPage(items: []); } - final now = DateTime.now(); + final now = stemNow(); final matches = []; var scanOffset = 0; final target = request.offset + request.limit; @@ -296,7 +296,7 @@ class SqliteResultBackend implements ResultBackend { @override Future setWorkerHeartbeat(WorkerHeartbeat heartbeat) async { - final now = DateTime.now(); + final now = stemNow(); final expiresAt = now.add(heartbeatTtl); await _connections.runInTransaction((txn) async { final model = StemWorkerHeartbeat( @@ -322,7 +322,7 @@ class SqliteResultBackend implements ResultBackend { @override Future getWorkerHeartbeat(String workerId) async { - final now = DateTime.now(); + final now = stemNow(); final row = await _context .query() .whereEquals('workerId', workerId) @@ -334,7 +334,7 @@ class SqliteResultBackend implements ResultBackend { @override Future> listWorkerHeartbeats() async { - final now = DateTime.now(); + final now = stemNow(); final rows = await _context .query() .whereEquals('namespace', namespace) @@ -346,7 +346,7 @@ class SqliteResultBackend implements ResultBackend { @override Future initGroup(GroupDescriptor descriptor) async { - final now = DateTime.now(); + final now = stemNow(); final expiresAt = now.add(descriptor.ttl ?? groupDefaultTtl); await _connections.runInTransaction((txn) async { await txn.repository().upsert( @@ -394,7 +394,7 @@ class SqliteResultBackend implements ResultBackend { @override Future getGroup(String groupId) async { - final now = DateTime.now(); + final now = stemNow(); final groupRow = await _context .query() .whereEquals('id', groupId) @@ -433,7 +433,7 @@ class SqliteResultBackend implements ResultBackend { @override Future expire(String taskId, Duration ttl) async { - final expiresAt = DateTime.now().add(ttl); + final expiresAt = stemNow().add(ttl); await _context.repository().update( StemTaskResultUpdateDto(expiresAt: expiresAt), where: StemTaskResultPartial(id: taskId, namespace: namespace), @@ -486,7 +486,7 @@ class SqliteResultBackend implements ResultBackend { } Future _runCleanupCycle() async { - final now = DateTime.now(); + final now = stemNow(); await _connections.runInTransaction((txn) async { await txn .query() @@ -521,7 +521,7 @@ class SqliteResultBackend implements ResultBackend { } Future _groupExists(String groupId) async { - final now = DateTime.now(); + final now = stemNow(); return _context .query() .whereEquals('id', groupId) diff --git a/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart b/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart index 3f482003..35707150 100644 --- a/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart +++ b/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart @@ -228,7 +228,7 @@ class SqliteBroker implements Broker { return; } final jobId = _parseReceipt(delivery.receipt); - final now = DateTime.now(); + final now = stemNow(); await _context .query() .whereEquals('id', jobId) @@ -253,7 +253,7 @@ class SqliteBroker implements Broker { return; } final jobId = _parseReceipt(delivery.receipt); - final now = DateTime.now(); + final now = stemNow(); final row = await _context .query() @@ -286,7 +286,7 @@ class SqliteBroker implements Broker { return; } final jobId = _parseReceipt(delivery.receipt); - final now = DateTime.now(); + final now = stemNow(); await _context.repository().update( StemQueueJobUpdateDto(lockedUntil: now.add(by)), where: StemQueueJobPartial(id: jobId, namespace: namespace), @@ -304,7 +304,7 @@ class SqliteBroker implements Broker { @override Future pendingCount(String queue) async { - final now = DateTime.now(); + final now = stemNow(); return _context .query() .whereEquals('queue', queue) @@ -324,7 +324,7 @@ class SqliteBroker implements Broker { @override Future inflightCount(String queue) async { - final now = DateTime.now(); + final now = stemNow(); return _context .query() .whereEquals('queue', queue) @@ -399,7 +399,7 @@ class SqliteBroker implements Broker { for (final entry in entries) { final updatedEnvelope = delay == null ? entry.envelope - : entry.envelope.copyWith(notBefore: DateTime.now().add(delay)); + : entry.envelope.copyWith(notBefore: stemNow().add(delay)); await _insertJob( txn, envelope: updatedEnvelope, @@ -457,7 +457,7 @@ class SqliteBroker implements Broker { } Future<_QueuedJob?> _claimNextJob(String queue, String consumerId) async { - final now = DateTime.now(); + final now = stemNow(); final visibilityUntil = now.add(defaultVisibilityTimeout); return _connections.runInTransaction((txn) async { @@ -515,7 +515,7 @@ class SqliteBroker implements Broker { } Future _runSweeperCycle() async { - final now = DateTime.now(); + final now = stemNow(); await _connections.runInTransaction((txn) async { await txn .query() @@ -564,7 +564,7 @@ class SqliteBroker implements Broker { final payload = jsonEncode({ 'envelopeId': envelopeId, 'consumerId': consumerId, - 'issuedAtMicros': DateTime.now().microsecondsSinceEpoch, + 'issuedAtMicros': stemNow().microsecondsSinceEpoch, }); final encodedPayload = base64Url.encode(utf8.encode(payload)); return '$_broadcastReceiptPrefix$encodedPayload'; @@ -608,7 +608,7 @@ class SqliteBroker implements Broker { if (consumers.isEmpty) { return; } - final leaseExpiresAt = DateTime.now().add(defaultVisibilityTimeout); + final leaseExpiresAt = stemNow().add(defaultVisibilityTimeout); for (final consumer in consumers) { if (!consumer.isActive) { continue; @@ -751,7 +751,7 @@ class _Consumer { } if (jobs.isNotEmpty) { emitted = true; - final leaseExpiresAt = DateTime.now().add( + final leaseExpiresAt = stemNow().add( broker.defaultVisibilityTimeout, ); for (final job in jobs) { From 8e8dc15509fec3ebee00509e47b171c753a00c80 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 13:27:57 -0500 Subject: [PATCH 09/23] refactor(cli,dashboard): align timestamps with stem clock --- .../dashboard/lib/src/services/models.dart | 2 +- .../lib/src/services/stem_service.dart | 8 ++++---- .../lib/src/state/dashboard_state.dart | 7 ++++--- packages/dashboard/lib/src/ui/content.dart | 3 ++- packages/stem_cli/lib/src/cli/dlq.dart | 2 +- packages/stem_cli/lib/src/cli/observer.dart | 2 +- packages/stem_cli/lib/src/cli/schedule.dart | 6 +++--- packages/stem_cli/lib/src/cli/worker.dart | 20 +++++++++---------- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/dashboard/lib/src/services/models.dart b/packages/dashboard/lib/src/services/models.dart index 0f863a13..058ca9a7 100644 --- a/packages/dashboard/lib/src/services/models.dart +++ b/packages/dashboard/lib/src/services/models.dart @@ -155,7 +155,7 @@ class WorkerStatus { final Map extras; /// Age of the last heartbeat. - Duration get age => DateTime.now().toUtc().difference(timestamp); + Duration get age => stemNow().toUtc().difference(timestamp); } /// Event captured for the dashboard activity log. diff --git a/packages/dashboard/lib/src/services/stem_service.dart b/packages/dashboard/lib/src/services/stem_service.dart index fe045dcb..f1bc45de 100644 --- a/packages/dashboard/lib/src/services/stem_service.dart +++ b/packages/dashboard/lib/src/services/stem_service.dart @@ -157,7 +157,7 @@ class StemDashboardService implements DashboardDataSource { .map((target) => ControlQueueNames.worker(_namespace, target)) .toList(); - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); for (final queue in targets) { final envelope = Envelope( id: generateEnvelopeId(), @@ -189,11 +189,11 @@ class StemDashboardService implements DashboardDataSource { final iterator = StreamIterator(subscription); final replies = []; - final deadline = DateTime.now().add(timeout); + final deadline = stemNow().add(timeout); try { - while (DateTime.now().isBefore(deadline)) { - final remaining = deadline.difference(DateTime.now()); + while (stemNow().isBefore(deadline)) { + final remaining = deadline.difference(stemNow()); bool hasNext; try { hasNext = await iterator.moveNext().timeout(remaining); diff --git a/packages/dashboard/lib/src/state/dashboard_state.dart b/packages/dashboard/lib/src/state/dashboard_state.dart index f2c02c6f..0c3e5174 100644 --- a/packages/dashboard/lib/src/state/dashboard_state.dart +++ b/packages/dashboard/lib/src/state/dashboard_state.dart @@ -6,6 +6,7 @@ import 'package:routed_hotwire/routed_hotwire.dart'; import 'package:stem_dashboard/src/services/models.dart'; import 'package:stem_dashboard/src/services/stem_service.dart'; import 'package:stem_dashboard/src/ui/event_templates.dart'; +import 'package:stem/stem.dart' show stemNow; /// Manages polling, state, and event streaming for the dashboard. class DashboardState { @@ -81,7 +82,7 @@ class DashboardState { } void _updateThroughput(List queues) { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); if (_lastPollAt == null) { _lastPollAt = now; return; @@ -117,7 +118,7 @@ class DashboardState { List current, ) { final prevMap = {for (final summary in previous) summary.queue: summary}; - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); for (final summary in current) { final prev = prevMap.remove(summary.queue); if (prev == null) { @@ -192,7 +193,7 @@ class DashboardState { Map current, ) { final remaining = Map.from(previous); - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); for (final entry in current.entries) { final prev = remaining.remove(entry.key); final worker = entry.value; diff --git a/packages/dashboard/lib/src/ui/content.dart b/packages/dashboard/lib/src/ui/content.dart index 9b5b91ba..6f565d30 100644 --- a/packages/dashboard/lib/src/ui/content.dart +++ b/packages/dashboard/lib/src/ui/content.dart @@ -6,6 +6,7 @@ import 'package:intl/intl.dart'; import 'package:stem_dashboard/src/services/models.dart'; import 'package:stem_dashboard/src/ui/event_templates.dart'; import 'package:stem_dashboard/src/ui/layout.dart'; +import 'package:stem/stem.dart' show stemNow; final _numberFormat = NumberFormat.decimalPattern(); @@ -578,7 +579,7 @@ int _totalIsolates(List workers) { String _formatInt(int value) => _numberFormat.format(value); String _formatRelative(DateTime timestamp) { - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final diff = now.difference(timestamp.toUtc()); if (diff < const Duration(seconds: 30)) return 'just now'; if (diff < const Duration(minutes: 1)) { diff --git a/packages/stem_cli/lib/src/cli/dlq.dart b/packages/stem_cli/lib/src/cli/dlq.dart index 6e5813e5..7c715afb 100644 --- a/packages/stem_cli/lib/src/cli/dlq.dart +++ b/packages/stem_cli/lib/src/cli/dlq.dart @@ -334,7 +334,7 @@ class DlqReplayCommand extends Command { if (!result.dryRun) { final backend = ctx.backend; if (backend != null) { - final replayedAt = DateTime.now().toIso8601String(); + final replayedAt = stemNow().toIso8601String(); for (final entry in result.entries) { try { final status = await backend.get(entry.envelope.id); diff --git a/packages/stem_cli/lib/src/cli/observer.dart b/packages/stem_cli/lib/src/cli/observer.dart index aecf99ce..c55654c1 100644 --- a/packages/stem_cli/lib/src/cli/observer.dart +++ b/packages/stem_cli/lib/src/cli/observer.dart @@ -390,7 +390,7 @@ class ObserveSchedulesCommand extends Command { return 0; } final calculator = ScheduleCalculator(); - final now = DateTime.now(); + final now = stemNow(); var dueCount = 0; var overdueCount = 0; Duration? maxDrift; diff --git a/packages/stem_cli/lib/src/cli/schedule.dart b/packages/stem_cli/lib/src/cli/schedule.dart index 81edff10..4355e545 100644 --- a/packages/stem_cli/lib/src/cli/schedule.dart +++ b/packages/stem_cli/lib/src/cli/schedule.dart @@ -59,7 +59,7 @@ class ScheduleListCommand extends Command { return 0; } final calculator = ScheduleCalculator(); - final now = DateTime.now(); + final now = stemNow(); dependencies.out.writeln( 'ID | Task | Queue | Spec | Next Run | Last Run | Jitter | Enabled', ); @@ -374,7 +374,7 @@ class ScheduleApplyCommand extends Command { void _validateScheduleEntry(ScheduleEntry entry) { final calculator = ScheduleCalculator(); try { - final base = entry.lastRunAt ?? DateTime.now(); + final base = entry.lastRunAt ?? stemNow(); final next = calculator.nextRun(entry, base, includeJitter: false); final interval = next.difference(base); if (interval <= Duration.zero) { @@ -703,7 +703,7 @@ class ScheduleDryRunCommand extends Command { return 64; } } else { - start = DateTime.now(); + start = stemNow(); } ScheduleEntry? entry; diff --git a/packages/stem_cli/lib/src/cli/worker.dart b/packages/stem_cli/lib/src/cli/worker.dart index 280290d1..8b7f73cb 100644 --- a/packages/stem_cli/lib/src/cli/worker.dart +++ b/packages/stem_cli/lib/src/cli/worker.dart @@ -674,7 +674,7 @@ class WorkerRevokeCommand extends Command { return 70; } - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final baseVersion = generateRevokeVersion(); final entries = []; for (var i = 0; i < tasks.length; i += 1) { @@ -1449,7 +1449,7 @@ class WorkerStatusCommand extends Command { out.writeln(jsonEncode(heartbeat.toJson())); return; } - final now = DateTime.now().toUtc(); + final now = stemNow().toUtc(); final age = now.difference(heartbeat.timestamp); final isStale = expectedInterval != null && expectedInterval > Duration.zero ? age > expectedInterval @@ -1882,7 +1882,7 @@ class WorkerMultiCommand extends Command { environment: baseEnv, ); - final timestamp = DateTime.now() + final timestamp = stemNow() .toUtc() .toIso8601String() .replaceAll(':', '-') @@ -1924,7 +1924,7 @@ class WorkerMultiCommand extends Command { ); final host = _hostname; - final timestamp = DateTime.now() + final timestamp = stemNow() .toUtc() .toIso8601String() .replaceAll(':', '-') @@ -1967,7 +1967,7 @@ class WorkerMultiCommand extends Command { ); final host = _hostname; - final timestamp = DateTime.now() + final timestamp = stemNow() .toUtc() .toIso8601String() .replaceAll(':', '-') @@ -2094,9 +2094,7 @@ class WorkerHealthcheckCommand extends Command { } final since = _pidFileTimestamp(pidfile); - final uptime = since != null - ? DateTime.now().toUtc().difference(since) - : null; + final uptime = since != null ? stemNow().toUtc().difference(since) : null; final payload = { 'status': running ? 'ok' : 'error', @@ -2422,7 +2420,7 @@ String _describeUptime(String pidfile) { if (since == null) { return 'uptime unknown'; } - final duration = DateTime.now().toUtc().difference(since); + final duration = stemNow().toUtc().difference(since); return 'uptime ${formatReadableDuration(duration)}'; } @@ -2677,9 +2675,9 @@ Future _waitForExit(int pid, Duration timeout) async { if (timeout.isNegative) { return false; } - final deadline = DateTime.now().add(timeout); + final deadline = stemNow().add(timeout); while (await _isPidRunning(pid)) { - if (DateTime.now().isAfter(deadline)) { + if (stemNow().isAfter(deadline)) { return false; } await Future.delayed(const Duration(milliseconds: 200)); From 90c92572e0d102632e44fe04fad6578dfba03558 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:00:23 -0500 Subject: [PATCH 10/23] docs: address review nits across parity and event docs --- .site/docs/comparisons/stem-vs-bullmq.md | 12 +++++------ .site/docs/core-concepts/queue-events.md | 2 +- .site/docs/core-concepts/rate-limiting.md | 2 +- .site/docs/core-concepts/signals.md | 5 +++-- .../docs_snippets/lib/canvas_batch.dart | 10 +++++++++- packages/stem_adapter_tests/README.md | 20 +++++++++++++++++++ 6 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.site/docs/comparisons/stem-vs-bullmq.md b/.site/docs/comparisons/stem-vs-bullmq.md index a8a22818..c95329a2 100644 --- a/.site/docs/comparisons/stem-vs-bullmq.md +++ b/.site/docs/comparisons/stem-vs-bullmq.md @@ -45,6 +45,12 @@ It focuses on capability parity, not API-level compatibility. ## Update policy +When this matrix changes: + +1. Update the **As of** date. +2. Keep row names aligned with BullMQ terminology. +3. Update rationale links so every status remains auditable. + ## BullMQ events parity notes Stem supports the two common BullMQ event-listening styles: @@ -54,9 +60,3 @@ Stem supports the two common BullMQ event-listening styles: | `QueueEvents` listeners | `QueueEvents` + `QueueEventsProducer` (queue-scoped custom events) | | Custom queue events | `producer.emit(queue, eventName, payload, headers, meta)` | | Worker-specific event listeners | `StemSignals` convenience APIs with `workerId` filters (`onWorkerReady`, `onWorkerInit`, `onTaskFailure`, `onControlCommandCompleted`, etc.) | - -When this matrix changes: - -1. Update the **As of** date. -2. Keep row names aligned with BullMQ terminology. -3. Update rationale links so every status remains auditable. diff --git a/.site/docs/core-concepts/queue-events.md b/.site/docs/core-concepts/queue-events.md index 39da7f9a..6fa6b2cd 100644 --- a/.site/docs/core-concepts/queue-events.md +++ b/.site/docs/core-concepts/queue-events.md @@ -42,7 +42,7 @@ Multiple listeners on the same queue receive each emitted event. - `headers` and `meta` round-trip to listeners. - Event names and queue names must be non-empty. - Delivery follows the underlying broker's broadcast behavior for active - listeners (no historical replay API is built in to `QueueEvents`). + listeners (no historical replay API is built into `QueueEvents`). ## When to Use Queue Events vs Signals diff --git a/.site/docs/core-concepts/rate-limiting.md b/.site/docs/core-concepts/rate-limiting.md index 90ca3f60..60e0598b 100644 --- a/.site/docs/core-concepts/rate-limiting.md +++ b/.site/docs/core-concepts/rate-limiting.md @@ -147,7 +147,7 @@ Group rate limits share a limiter bucket across related tasks. - `groupRateKey`: optional static key (if omitted, Stem resolves from header) - `groupRateKeyHeader`: header used when `groupRateKey` is not set (default: `tenant`) -- `groupRateLimiterFailureMode`: +- `groupRateLimiterFailureMode` (default: `failOpen`): - `failOpen`: continue execution if limiter backend fails - `failClosed`: requeue/retry when limiter backend fails diff --git a/.site/docs/core-concepts/signals.md b/.site/docs/core-concepts/signals.md index 9b056a3f..3f4340cf 100644 --- a/.site/docs/core-concepts/signals.md +++ b/.site/docs/core-concepts/signals.md @@ -33,8 +33,9 @@ All signal payloads implement `StemEvent` and dispatch through - `beforeTaskPublish` fires immediately before broker IO; `afterTaskPublish` runs once persistence succeeds. -- `taskReceived` is emitted before handler execution. -- `taskPrerun` precedes handler execution; `taskPostrun` runs after completion. +- `taskReceived` is emitted when a worker claims/dequeues a task. +- `taskPrerun` fires immediately before handler invocation. +- Execution ordering is `taskReceived` -> `taskPrerun` -> handler -> `taskPostrun`. - Worker lifecycle follows `workerInit` -> `workerReady` -> optional `workerStopping` -> `workerShutdown`. - Scheduler signals emit due -> dispatched/failed. diff --git a/packages/stem/example/docs_snippets/lib/canvas_batch.dart b/packages/stem/example/docs_snippets/lib/canvas_batch.dart index 908fcbd5..3374c6b8 100644 --- a/packages/stem/example/docs_snippets/lib/canvas_batch.dart +++ b/packages/stem/example/docs_snippets/lib/canvas_batch.dart @@ -29,7 +29,15 @@ Future main() async { task('batch.double', args: {'value': 3}), ]); - final status = await app.canvas.inspectBatch(submission.batchId); + // Batches may still be running immediately after submission. + BatchStatus? status; + for (var i = 0; i < 20; i += 1) { + status = await app.canvas.inspectBatch(submission.batchId); + if (status?.isTerminal == true) { + break; + } + await Future.delayed(const Duration(milliseconds: 50)); + } print( 'Batch ${submission.batchId} state=${status?.state} ' 'completed=${status?.completed}/${status?.expected}', diff --git a/packages/stem_adapter_tests/README.md b/packages/stem_adapter_tests/README.md index ca90372b..a78a72f2 100644 --- a/packages/stem_adapter_tests/README.md +++ b/packages/stem_adapter_tests/README.md @@ -92,6 +92,12 @@ all other contract assertions active. | `verifyOwnerLookup` | `true` | `ownerOf` tests | Verifies lock owner lookup behavior. | | `verifyRenewSemantics` | `true` | Renew and expiry tests | Verifies renewal/TTL semantics for active locks. | +### RevokeStoreContractCapabilities + +| Flag | Default | Affects | Behavior when enabled | +|---|---|---|---| +| `verifyPruneExpired` | `true` | Revoke expiry pruning tests | Verifies `pruneExpired` removes only expired revocations in the target namespace. | + ## Skip Behavior Each flagged test uses explicit `skip` values (instead of implicit omission) so @@ -140,6 +146,20 @@ runResultBackendContractTests( ); ``` +### Adapter without queue-event fan-out + +```dart +runQueueEventsContractTests( + adapterName: 'single-listener-adapter', + factory: QueueEventsContractFactory(create: createBroker), + settings: const QueueEventsContractSettings( + capabilities: QueueEventsContractCapabilities( + verifyFanout: false, + ), + ), +); +``` + ## Workflow Clock Requirement Workflow store factories receive a shared `FakeWorkflowClock`. Inject that same From da08861e5cba1f62a6c19818edc8575a1a4a904b Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:00:33 -0500 Subject: [PATCH 11/23] feat(core): tighten event semantics and UTC clock behavior --- packages/stem/lib/src/core/clock.dart | 3 +- packages/stem/lib/src/core/contracts.dart | 3 + packages/stem/lib/src/core/queue_events.dart | 32 +++++++---- .../src/scheduler/in_memory_lock_store.dart | 5 +- packages/stem/lib/src/signals/payloads.dart | 55 +++++++++++++------ .../lib/src/workflow/core/workflow_clock.dart | 1 - packages/stem/lib/stem.dart | 2 +- packages/stem/lib/testing.dart | 5 ++ packages/stem/test/unit/core/clock_test.dart | 1 + .../unit/workflow/workflow_clock_test.dart | 1 + 10 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 packages/stem/lib/testing.dart diff --git a/packages/stem/lib/src/core/clock.dart b/packages/stem/lib/src/core/clock.dart index 842a303c..c8e90e92 100644 --- a/packages/stem/lib/src/core/clock.dart +++ b/packages/stem/lib/src/core/clock.dart @@ -14,8 +14,9 @@ class SystemStemClock extends StemClock { /// Creates a system clock wrapper. const SystemStemClock(); + /// Returns the current UTC instant. @override - DateTime now() => DateTime.now(); + DateTime now() => DateTime.now().toUtc(); } /// Controllable clock for deterministic testing. diff --git a/packages/stem/lib/src/core/contracts.dart b/packages/stem/lib/src/core/contracts.dart index 3fa5a147..0652888f 100644 --- a/packages/stem/lib/src/core/contracts.dart +++ b/packages/stem/lib/src/core/contracts.dart @@ -1026,6 +1026,9 @@ class TaskOptions { } static RateLimiterFailureMode? _parseFailureMode(Object? value) { + if (value is RateLimiterFailureMode) { + return value; + } final raw = value?.toString().trim().toLowerCase(); if (raw == null || raw.isEmpty) return null; for (final mode in RateLimiterFailureMode.values) { diff --git a/packages/stem/lib/src/core/queue_events.dart b/packages/stem/lib/src/core/queue_events.dart index 13684f06..9fb94669 100644 --- a/packages/stem/lib/src/core/queue_events.dart +++ b/packages/stem/lib/src/core/queue_events.dart @@ -54,6 +54,7 @@ class QueueCustomEvent implements StemEvent { Map get attributes => { 'id': id, 'queue': queue, + 'name': name, 'payload': payload, 'headers': headers, 'meta': meta, @@ -131,7 +132,7 @@ class QueueEvents { required String queue, String? consumerName, this.prefetch = 10, - }) : queue = queue.trim(), + }) : queue = _normalizeQueue(queue), consumerName = consumerName ?? 'stem-queue-events-${generateEnvelopeId().replaceAll('-', '')}'; @@ -178,14 +179,6 @@ class QueueEvents { if (_started) { return; } - if (queue.isEmpty) { - throw ArgumentError.value( - queue, - 'queue', - 'Queue name must not be empty', - ); - } - _started = true; _subscription = broker .consume( @@ -201,7 +194,7 @@ class QueueEvents { .listen( _onDelivery, onError: (Object error, StackTrace stackTrace) { - _events.addError(error, stackTrace); + _emitError(error, stackTrace); }, ); } @@ -220,11 +213,11 @@ class QueueEvents { Future _onDelivery(Delivery delivery) async { try { final event = _eventFromEnvelope(delivery.envelope); - if (event != null && event.queue == queue) { + if (!_closed && event != null && event.queue == queue) { _events.add(event); } } on Object catch (error, stackTrace) { - _events.addError(error, stackTrace); + _emitError(error, stackTrace); } finally { try { await broker.ack(delivery); @@ -233,6 +226,21 @@ class QueueEvents { } } } + + void _emitError(Object error, StackTrace stackTrace) { + if (_closed || _events.isClosed) { + return; + } + _events.addError(error, stackTrace); + } +} + +String _normalizeQueue(String queue) { + final normalized = queue.trim(); + if (normalized.isEmpty) { + throw ArgumentError.value(queue, 'queue', 'Queue name must not be empty'); + } + return normalized; } QueueCustomEvent? _eventFromEnvelope(Envelope envelope) { diff --git a/packages/stem/lib/src/scheduler/in_memory_lock_store.dart b/packages/stem/lib/src/scheduler/in_memory_lock_store.dart index 97c9dccb..a4cda106 100644 --- a/packages/stem/lib/src/scheduler/in_memory_lock_store.dart +++ b/packages/stem/lib/src/scheduler/in_memory_lock_store.dart @@ -60,11 +60,12 @@ class InMemoryLockStore implements LockStore { if (lock.owner != owner) { return false; } - if (lock.isExpired(stemNow())) { + final now = stemNow(); + if (lock.isExpired(now)) { _locks.remove(key); return false; } - lock.expiresAt = stemNow().add(ttl); + lock.expiresAt = now.add(ttl); return true; } diff --git a/packages/stem/lib/src/signals/payloads.dart b/packages/stem/lib/src/signals/payloads.dart index 1c1222da..5f8a5b76 100644 --- a/packages/stem/lib/src/signals/payloads.dart +++ b/packages/stem/lib/src/signals/payloads.dart @@ -106,7 +106,11 @@ class AfterTaskPublishPayload implements StemEvent { /// Payload emitted when a task is received by a worker. class TaskReceivedPayload implements StemEvent { /// Creates a new [TaskReceivedPayload] instance. - const TaskReceivedPayload({required this.envelope, required this.worker}); + TaskReceivedPayload({ + required this.envelope, + required this.worker, + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The task envelope that was received. final Envelope envelope; @@ -114,6 +118,8 @@ class TaskReceivedPayload implements StemEvent { /// The worker that received the task. final WorkerInfo worker; + final DateTime _occurredAt; + /// The unique identifier for the task. String get taskId => envelope.id; @@ -124,7 +130,7 @@ class TaskReceivedPayload implements StemEvent { String get eventName => 'task-received'; @override - DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { @@ -138,11 +144,12 @@ class TaskReceivedPayload implements StemEvent { /// Payload emitted before a task begins execution. class TaskPrerunPayload implements StemEvent { /// Creates a new [TaskPrerunPayload] instance. - const TaskPrerunPayload({ + TaskPrerunPayload({ required this.envelope, required this.worker, required this.context, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The task envelope to be executed. final Envelope envelope; @@ -153,6 +160,8 @@ class TaskPrerunPayload implements StemEvent { /// The execution context for the task. final TaskContext context; + final DateTime _occurredAt; + /// The unique identifier for the task. String get taskId => envelope.id; @@ -166,7 +175,7 @@ class TaskPrerunPayload implements StemEvent { String get eventName => 'task-prerun'; @override - DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { @@ -181,13 +190,14 @@ class TaskPrerunPayload implements StemEvent { /// Payload emitted after a task finishes execution. class TaskPostrunPayload implements StemEvent { /// Creates a new [TaskPostrunPayload] instance. - const TaskPostrunPayload({ + TaskPostrunPayload({ required this.envelope, required this.worker, required this.context, required this.result, required this.state, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The task envelope that was executed. final Envelope envelope; @@ -204,6 +214,8 @@ class TaskPostrunPayload implements StemEvent { /// The final state of the task. final TaskState state; + final DateTime _occurredAt; + /// The unique identifier for the task. String get taskId => envelope.id; @@ -217,7 +229,7 @@ class TaskPostrunPayload implements StemEvent { String get eventName => 'task-postrun'; @override - DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { @@ -287,11 +299,12 @@ class TaskRetryPayload implements StemEvent { /// Payload emitted when a task completes successfully. class TaskSuccessPayload implements StemEvent { /// Creates a new [TaskSuccessPayload] instance. - const TaskSuccessPayload({ + TaskSuccessPayload({ required this.envelope, required this.worker, required this.result, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The task envelope that completed successfully. final Envelope envelope; @@ -302,6 +315,8 @@ class TaskSuccessPayload implements StemEvent { /// The result returned by the successful task. final Object? result; + final DateTime _occurredAt; + /// The unique identifier for the task. String get taskId => envelope.id; @@ -315,7 +330,7 @@ class TaskSuccessPayload implements StemEvent { String get eventName => 'task-succeeded'; @override - DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { @@ -330,12 +345,13 @@ class TaskSuccessPayload implements StemEvent { /// Payload emitted when a task fails. class TaskFailurePayload implements StemEvent { /// Creates a new [TaskFailurePayload] instance. - const TaskFailurePayload({ + TaskFailurePayload({ required this.envelope, required this.worker, required this.error, required this.stackTrace, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The task envelope that failed. final Envelope envelope; @@ -349,6 +365,8 @@ class TaskFailurePayload implements StemEvent { /// The stack trace associated with the failure, if available. final StackTrace? stackTrace; + final DateTime _occurredAt; + /// The unique identifier for the task. String get taskId => envelope.id; @@ -362,7 +380,7 @@ class TaskFailurePayload implements StemEvent { String get eventName => 'task-failed'; @override - DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { @@ -379,11 +397,12 @@ class TaskFailurePayload implements StemEvent { /// Payload emitted when a task is revoked. class TaskRevokedPayload implements StemEvent { /// Creates a new [TaskRevokedPayload] instance. - const TaskRevokedPayload({ + TaskRevokedPayload({ required this.envelope, required this.worker, required this.reason, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The task envelope that was revoked. final Envelope envelope; @@ -394,11 +413,13 @@ class TaskRevokedPayload implements StemEvent { /// The reason for revoking the task. final String reason; + final DateTime _occurredAt; + @override String get eventName => 'task-revoked'; @override - DateTime get occurredAt => envelope.enqueuedAt.toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { diff --git a/packages/stem/lib/src/workflow/core/workflow_clock.dart b/packages/stem/lib/src/workflow/core/workflow_clock.dart index 8bb556bd..dc5cb795 100644 --- a/packages/stem/lib/src/workflow/core/workflow_clock.dart +++ b/packages/stem/lib/src/workflow/core/workflow_clock.dart @@ -2,7 +2,6 @@ import 'package:stem/src/core/clock.dart'; /// Abstraction over time sources used by the workflow runtime and stores. // Intentionally interface-like for injection and testing. -// ignore: one_member_abstracts abstract class WorkflowClock extends StemClock { /// Creates a workflow clock implementation. const WorkflowClock(); diff --git a/packages/stem/lib/stem.dart b/packages/stem/lib/stem.dart index 143f8729..0fc67481 100644 --- a/packages/stem/lib/stem.dart +++ b/packages/stem/lib/stem.dart @@ -87,7 +87,7 @@ export 'src/control/file_revoke_store.dart'; export 'src/control/in_memory_revoke_store.dart'; export 'src/control/revoke_store.dart'; export 'src/core/chord_metadata.dart'; -export 'src/core/clock.dart'; +export 'src/core/clock.dart' hide FakeStemClock; export 'src/core/config.dart'; export 'src/core/contracts.dart'; export 'src/core/encoder_keys.dart'; diff --git a/packages/stem/lib/testing.dart b/packages/stem/lib/testing.dart new file mode 100644 index 00000000..20a703dc --- /dev/null +++ b/packages/stem/lib/testing.dart @@ -0,0 +1,5 @@ +/// Test utilities for Stem. +library; + +export 'src/core/clock.dart' show FakeStemClock; +export 'src/testing/fake_stem.dart'; diff --git a/packages/stem/test/unit/core/clock_test.dart b/packages/stem/test/unit/core/clock_test.dart index 00027029..2ad0d2c8 100644 --- a/packages/stem/test/unit/core/clock_test.dart +++ b/packages/stem/test/unit/core/clock_test.dart @@ -1,4 +1,5 @@ import 'package:stem/stem.dart'; +import 'package:stem/testing.dart'; import 'package:test/test.dart'; void main() { diff --git a/packages/stem/test/unit/workflow/workflow_clock_test.dart b/packages/stem/test/unit/workflow/workflow_clock_test.dart index 82dfd94c..bcb140ac 100644 --- a/packages/stem/test/unit/workflow/workflow_clock_test.dart +++ b/packages/stem/test/unit/workflow/workflow_clock_test.dart @@ -1,4 +1,5 @@ import 'package:stem/stem.dart'; +import 'package:stem/testing.dart'; import 'package:stem/src/workflow/core/workflow_clock.dart'; import 'package:test/test.dart'; From af15c63911b26c45aa83dfe906026cb81f63dc7a Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:00:43 -0500 Subject: [PATCH 12/23] test(adapters): harden revoke and queue-event contract coverage --- .../lib/src/queue_events_contract_suite.dart | 4 +--- .../lib/src/revoke_store_contract_suite.dart | 11 ++++++++++- .../lib/src/control/postgres_revoke_store.dart | 2 +- .../migrations/m_20260224103000_add_revoke_store.dart | 4 ++-- .../stem_sqlite/lib/src/models/stem_revoke_entry.dart | 2 +- .../test/control/sqlite_revoke_store_test.dart | 5 +---- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart b/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart index 69ec9b4b..6f2a47e7 100644 --- a/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart +++ b/packages/stem_adapter_tests/lib/src/queue_events_contract_suite.dart @@ -244,8 +244,6 @@ void runQueueEventsContractTests({ String _queueName(String suffix) => 'contract-queue-events-$suffix-' - '${DateTime.now().microsecondsSinceEpoch}-${_counter++}'; - -int _counter = 0; + '${DateTime.now().microsecondsSinceEpoch}-${generateEnvelopeId()}'; Object _skipUnless(bool enabled, String reason) => enabled ? false : reason; diff --git a/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart b/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart index 22979765..cd0f7c03 100644 --- a/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart +++ b/packages/stem_adapter_tests/lib/src/revoke_store_contract_suite.dart @@ -163,6 +163,12 @@ void runRevokeStoreContractTests({ issuedAt: now, expiresAt: now.add(const Duration(minutes: 5)), ), + RevokeEntry( + namespace: namespaceA, + taskId: 'no-expiry', + version: 4, + issuedAt: now, + ), RevokeEntry( namespace: namespaceB, taskId: 'other-namespace', @@ -176,7 +182,10 @@ void runRevokeStoreContractTests({ expect(pruned, 1); final listedA = await current.list(namespaceA); - expect(listedA.map((entry) => entry.taskId), equals(['active'])); + expect( + listedA.map((entry) => entry.taskId), + equals(['active', 'no-expiry']), + ); final listedB = await current.list(namespaceB); expect( listedB.map((entry) => entry.taskId), diff --git a/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart b/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart index 9bdae55c..f98186be 100644 --- a/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart +++ b/packages/stem_postgres/lib/src/control/postgres_revoke_store.dart @@ -127,7 +127,7 @@ class PostgresRevokeStore implements RevokeStore { requestedBy: entry.requestedBy, issuedAt: entry.issuedAt, expiresAt: entry.expiresAt, - version: existing != null ? entry.version : entry.version, + version: entry.version, updatedAt: stemNow(), ); diff --git a/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart b/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart index e2dc8e05..e53cf049 100644 --- a/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart +++ b/packages/stem_sqlite/lib/src/database/migrations/m_20260224103000_add_revoke_store.dart @@ -12,11 +12,11 @@ class AddRevokeStore extends Migration { ..text('namespace') ..text('task_id') ..integer('version') - ..timestamp('issued_at'); + ..timestampTz('issued_at'); table.integer('terminate').defaultValue(0); table.text('reason').nullable(); table.text('requested_by').nullable(); - table.timestamp('expires_at').nullable(); + table.timestampTz('expires_at').nullable(); table ..timestampsTz() ..primary([ diff --git a/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart b/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart index 3c70903f..0e1832f3 100644 --- a/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart +++ b/packages/stem_sqlite/lib/src/models/stem_revoke_entry.dart @@ -31,7 +31,7 @@ class StemRevokeEntry extends Model with TimestampsTZ { @OrmField(columnName: 'issued_at') final DateTime issuedAt; - /// Integer flag representing terminate intent (`1` means true). + /// Integer flag representing terminate intent (`1` means true, `0` false). final int terminate; /// Optional human-readable reason. diff --git a/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart b/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart index f68a5025..d59030b5 100644 --- a/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart +++ b/packages/stem_sqlite/test/control/sqlite_revoke_store_test.dart @@ -17,9 +17,6 @@ void main() { }); tearDown(() async { - if (dbFile.existsSync()) { - await dbFile.delete(); - } await tempDir.delete(recursive: true); }); @@ -27,7 +24,7 @@ void main() { adapterName: 'SQLite', factory: RevokeStoreContractFactory( create: () async => SqliteRevokeStore.open(dbFile), - dispose: (store) => (store as SqliteRevokeStore).close(), + dispose: (store) => store.close(), ), ); From 4f4caf36cdb3177668111b07f92ec99985f78bb9 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:00:53 -0500 Subject: [PATCH 13/23] fix(worker): improve control semantics and timing robustness --- .../lib/src/services/stem_service.dart | 8 ++- packages/stem/lib/src/worker/worker.dart | 5 +- .../stem/test/unit/worker/worker_test.dart | 58 ++++++++++++------- packages/stem_cli/lib/src/cli/worker.dart | 9 ++- .../test/unit/cli/cli_worker_stats_test.dart | 5 +- 5 files changed, 56 insertions(+), 29 deletions(-) diff --git a/packages/dashboard/lib/src/services/stem_service.dart b/packages/dashboard/lib/src/services/stem_service.dart index f1bc45de..4eb872d1 100644 --- a/packages/dashboard/lib/src/services/stem_service.dart +++ b/packages/dashboard/lib/src/services/stem_service.dart @@ -192,8 +192,12 @@ class StemDashboardService implements DashboardDataSource { final deadline = stemNow().add(timeout); try { - while (stemNow().isBefore(deadline)) { - final remaining = deadline.difference(stemNow()); + while (true) { + final now = stemNow(); + final remaining = deadline.difference(now); + if (remaining <= Duration.zero) { + break; + } bool hasNext; try { hasNext = await iterator.moveNext().timeout(remaining); diff --git a/packages/stem/lib/src/worker/worker.dart b/packages/stem/lib/src/worker/worker.dart index 232ddbe2..b2b8a836 100644 --- a/packages/stem/lib/src/worker/worker.dart +++ b/packages/stem/lib/src/worker/worker.dart @@ -3734,7 +3734,10 @@ class Worker { final rawQueues = payload['queues']; if (rawQueues is List) { for (final value in rawQueues) { - final queueName = value.toString().trim(); + if (value is! String) { + continue; + } + final queueName = value.trim(); if (queueName.isNotEmpty) { queues.add(queueName); } diff --git a/packages/stem/test/unit/worker/worker_test.dart b/packages/stem/test/unit/worker/worker_test.dart index fc9b9508..05cb7c8d 100644 --- a/packages/stem/test/unit/worker/worker_test.dart +++ b/packages/stem/test/unit/worker/worker_test.dart @@ -1356,7 +1356,6 @@ void main() { timeout: const Duration(seconds: 3), ); - expect(await backend.get(firstId), isNotNull); expect((await backend.get(firstId))?.state, TaskState.succeeded); expect((await backend.get(secondId))?.state, TaskState.succeeded); expect( @@ -1445,6 +1444,7 @@ void main() { groupRateLimit: '10/m', groupRateKeyHeader: 'tenant', groupRateLimiterFailureMode: RateLimiterFailureMode.failClosed, + maxRetries: 5, ), entrypoint: (context, args) async { executed += 1; @@ -1472,9 +1472,7 @@ void main() { headers: const {'tenant': 'acme'}, ); - await Future.delayed(const Duration(milliseconds: 220)); - final status = await backend.get(taskId); - expect(status?.state, TaskState.retried); + await _waitForTaskState(backend, taskId, TaskState.retried); expect(executed, equals(0)); await worker.shutdown(); @@ -1532,9 +1530,7 @@ void main() { final stem = Stem(broker: broker, registry: registry, backend: backend); final taskId = await stem.enqueue('tasks.success'); - await Future.delayed(const Duration(milliseconds: 180)); - final pausedStatus = await backend.get(taskId); - expect(pausedStatus?.state, TaskState.queued); + await _assertTaskRemainsQueued(backend, taskId); final resumeReply = await _sendControlCommand( broker: broker, @@ -1696,15 +1692,18 @@ class _ChordCallbackTask implements TaskHandler { } Future _waitFor( - bool Function() predicate, { + FutureOr Function() predicate, { Duration timeout = const Duration(seconds: 2), Duration pollInterval = const Duration(milliseconds: 10), }) async { final deadline = DateTime.now().add(timeout); - while (!predicate()) { + while (true) { if (DateTime.now().isAfter(deadline)) { throw TimeoutException('Condition not met within $timeout'); } + if (await predicate()) { + return; + } await Future.delayed(pollInterval); } } @@ -1714,19 +1713,12 @@ Future _waitForCallbackSuccess( String taskId, { Duration timeout = const Duration(seconds: 3), }) async { - final deadline = DateTime.now().add(timeout); - while (true) { - final status = await backend.get(taskId); - if (status?.state == TaskState.succeeded) { - return; - } - if (DateTime.now().isAfter(deadline)) { - throw TimeoutException( - 'Callback $taskId did not succeed within $timeout', - ); - } - await Future.delayed(const Duration(milliseconds: 25)); - } + return _waitForTaskState( + backend, + taskId, + TaskState.succeeded, + timeout: timeout, + ); } Future _waitForTaskState( @@ -1750,6 +1742,28 @@ Future _waitForTaskState( } } +Future _assertTaskRemainsQueued( + ResultBackend backend, + String taskId, { + Duration holdFor = const Duration(milliseconds: 180), +}) async { + await _waitFor(() async { + final status = await backend.get(taskId); + return status?.state != null; + }, timeout: const Duration(seconds: 2)); + final deadline = DateTime.now().add(holdFor); + while (DateTime.now().isBefore(deadline)) { + final status = await backend.get(taskId); + if (status?.state != TaskState.queued) { + throw StateError( + 'Expected task $taskId to remain queued while paused. ' + 'Found ${status?.state}.', + ); + } + await Future.delayed(const Duration(milliseconds: 20)); + } +} + Future _sendControlCommand({ required Broker broker, required String namespace, diff --git a/packages/stem_cli/lib/src/cli/worker.dart b/packages/stem_cli/lib/src/cli/worker.dart index 8b7f73cb..37d72559 100644 --- a/packages/stem_cli/lib/src/cli/worker.dart +++ b/packages/stem_cli/lib/src/cli/worker.dart @@ -1078,9 +1078,11 @@ abstract class _WorkerQueueControlCommand extends Command { ); return 70; } - dependencies.out.writeln('Worker | Status | Updated | Paused'); dependencies.out.writeln( - '--------------+--------+---------+----------------', + 'Worker | Status | Updated | Paused queues (after)', + ); + dependencies.out.writeln( + '--------------+--------+---------+----------------------', ); final ordered = [...replies] ..sort((a, b) => a.workerId.compareTo(b.workerId)); @@ -1096,7 +1098,8 @@ abstract class _WorkerQueueControlCommand extends Command { } } - return replies.isEmpty ? 70 : 0; + final hasError = replies.any((reply) => reply.status != 'ok'); + return (replies.isEmpty || hasError) ? 70 : 0; } finally { await ctx.dispose(); } diff --git a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart index 167115f7..b314273e 100644 --- a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart @@ -481,10 +481,13 @@ Future _waitFor( Duration pollInterval = const Duration(milliseconds: 10), }) async { final deadline = DateTime.now().add(timeout); - while (!await predicate()) { + while (true) { if (DateTime.now().isAfter(deadline)) { throw TimeoutException('Condition not met within $timeout'); } + if (await predicate()) { + return; + } await Future.delayed(pollInterval); } } From e2f75b07a2c7e0d96fea033e808c2a055f34a1ba Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:01:02 -0500 Subject: [PATCH 14/23] feat(canvas): tighten batch inspection and submission idempotency --- packages/stem/lib/src/canvas/canvas.dart | 109 +++++++++++------- .../stem/test/unit/canvas/canvas_test.dart | 48 +++++--- 2 files changed, 99 insertions(+), 58 deletions(-) diff --git a/packages/stem/lib/src/canvas/canvas.dart b/packages/stem/lib/src/canvas/canvas.dart index 260851eb..385b4058 100644 --- a/packages/stem/lib/src/canvas/canvas.dart +++ b/packages/stem/lib/src/canvas/canvas.dart @@ -285,6 +285,9 @@ class Canvas { /// The task registry for resolving task metadata and handlers. final TaskRegistry registry; + final Map> _batchSubmissionLocks = + >{}; + /// Publishes a single task described by [signature]. /// /// The task is published to its configured queue and recorded as @@ -425,61 +428,81 @@ class Canvas { throw ArgumentError('Batch must include at least one task'); } final id = batchId ?? _generateId('batch'); - final existing = await backend.getGroup(id); - if (existing != null) { - if (existing.meta['stem.batch'] != true) { - throw StateError('Group "$id" already exists and is not a batch'); + return _withBatchSubmissionLock(id, () async { + final existing = await backend.getGroup(id); + if (existing != null) { + if (existing.meta['stem.batch'] != true) { + throw StateError('Group "$id" already exists and is not a batch'); + } + return BatchSubmission( + batchId: id, + taskIds: _batchTaskIdsFromGroup(existing), + ); } - return BatchSubmission( - batchId: id, - taskIds: _batchTaskIdsFromGroup(existing), - ); - } - final normalizedSignatures = >[]; - final taskIds = []; - for (final signature in signatures) { - final raw = signature(); - final grouped = raw.copyWith( - headers: {...raw.headers, 'stem-group-id': id}, - meta: {...raw.meta, 'groupId': id}, - ); - taskIds.add(grouped.id); - normalizedSignatures.add( - TaskSignature.custom( - signature.name, - () => grouped, - decode: signature.decode, + final normalizedSignatures = >[]; + final taskIds = []; + for (final signature in signatures) { + final raw = signature(); + final grouped = raw.copyWith( + headers: {...raw.headers, 'stem-group-id': id}, + meta: {...raw.meta, 'groupId': id}, + ); + taskIds.add(grouped.id); + normalizedSignatures.add( + TaskSignature.custom( + signature.name, + () => grouped, + decode: signature.decode, + ), + ); + } + + final createdAt = stemNow().toUtc().toIso8601String(); + await backend.initGroup( + GroupDescriptor( + id: id, + expected: signatures.length, + ttl: ttl, + meta: { + 'stem.batch': true, + 'stem.batch.createdAt': createdAt, + 'stem.batch.taskCount': signatures.length, + 'stem.batch.taskIds': taskIds, + }, ), ); - } - - final createdAt = stemNow().toUtc().toIso8601String(); - await backend.initGroup( - GroupDescriptor( - id: id, - expected: signatures.length, - ttl: ttl, - meta: { - 'stem.batch': true, - 'stem.batch.createdAt': createdAt, - 'stem.batch.taskCount': signatures.length, - 'stem.batch.taskIds': taskIds, - }, - ), - ); - final dispatch = await group(normalizedSignatures, groupId: id); - await dispatch.dispose(); - return BatchSubmission(batchId: id, taskIds: taskIds); + final dispatch = await group(normalizedSignatures, groupId: id); + await dispatch.dispose(); + return BatchSubmission(batchId: id, taskIds: taskIds); + }); } /// Reads durable lifecycle status for a submitted [batchId]. Future inspectBatch(String batchId) async { final status = await backend.getGroup(batchId); - if (status == null) return null; + if (status == null || status.meta['stem.batch'] != true) return null; return _buildBatchStatus(status); } + Future _withBatchSubmissionLock( + String batchId, + Future Function() operation, + ) async { + final previous = _batchSubmissionLocks[batchId] ?? Future.value(); + final completer = Completer(); + _batchSubmissionLocks[batchId] = completer.future; + await previous; + try { + return await operation(); + } finally { + completer.complete(); + if (identical(_batchSubmissionLocks[batchId], completer.future)) { + _batchSubmissionLocks.remove(batchId); + } + } + } + /// Runs tasks sequentially, passing each result to the next. /// /// Each task is published only after the previous task succeeds. The result diff --git a/packages/stem/test/unit/canvas/canvas_test.dart b/packages/stem/test/unit/canvas/canvas_test.dart index 21c4fe15..4c6ad49d 100644 --- a/packages/stem/test/unit/canvas/canvas_test.dart +++ b/packages/stem/test/unit/canvas/canvas_test.dart @@ -160,32 +160,50 @@ Future _waitForSuccess( String taskId, { Duration timeout = const Duration(seconds: 2), }) async { - final start = DateTime.now(); - while (true) { - final status = await backend.get(taskId); - if (status != null && status.state == TaskState.succeeded) { - return status; - } - if (DateTime.now().difference(start) > timeout) { - throw TimeoutException('Task $taskId did not succeed in time'); - } - await Future.delayed(const Duration(milliseconds: 50)); - } + return _waitForNonNull( + () async { + final status = await backend.get(taskId); + if (status?.state == TaskState.succeeded) { + return status; + } + return null; + }, + timeout: timeout, + errorMessage: 'Task $taskId did not succeed in time', + ); } Future _waitForBatchTerminal( Canvas canvas, String batchId, { Duration timeout = const Duration(seconds: 2), +}) async { + return _waitForNonNull( + () async { + final status = await canvas.inspectBatch(batchId); + if (status != null && status.isTerminal) { + return status; + } + return null; + }, + timeout: timeout, + errorMessage: 'Batch $batchId did not complete in time', + ); +} + +Future _waitForNonNull( + Future Function() read, { + required Duration timeout, + required String errorMessage, }) async { final start = DateTime.now(); while (true) { - final status = await canvas.inspectBatch(batchId); - if (status != null && status.isTerminal) { - return status; + final value = await read(); + if (value != null) { + return value; } if (DateTime.now().difference(start) > timeout) { - throw TimeoutException('Batch $batchId did not complete in time'); + throw TimeoutException(errorMessage); } await Future.delayed(const Duration(milliseconds: 50)); } From 477ed1a0dc7add3fe4ee343c9695d72c20bb6060 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:01:17 -0500 Subject: [PATCH 15/23] fix(redis): preserve broadcast consumer groups and cleanup disposal --- .../lib/src/brokers/redis_broker.dart | 28 +++--- .../redis_broker_integration_test.dart | 99 +------------------ 2 files changed, 19 insertions(+), 108 deletions(-) diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 8f3a4703..ce0743cb 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -327,9 +327,8 @@ class RedisStreamsBroker implements Broker { _groupsCreated.add(key); } - Future _ensureBroadcastGroup(String channel, String consumer) async { + Future _ensureBroadcastGroup(String channel, String group) async { final stream = _broadcastStreamKey(channel); - final group = _broadcastGroupKey(channel, consumer); final key = '$stream|$group'; if (_groupsCreated.contains(key)) return; try { @@ -494,8 +493,9 @@ class RedisStreamsBroker implements Broker { 'RedisStreamsBroker requires at least one queue or broadcast channel.', ); } - final group = - consumerGroup ?? (queue == null ? '__broadcast__' : _groupKey(queue)); + final queueGroup = queue == null + ? null + : (consumerGroup ?? _groupKey(queue)); final streamKeys = queue == null ? const [] : _priorityStreamKeys(queue); @@ -583,7 +583,7 @@ class RedisStreamsBroker implements Broker { result = await sendConsumerCommand([ 'XREADGROUP', 'GROUP', - group, + queueGroup!, consumer, 'BLOCK', blockTime.inMilliseconds.toString(), @@ -609,7 +609,12 @@ class RedisStreamsBroker implements Broker { if (result == null) { continue; } - final deliveries = _parseDeliveries(queue, group, consumer, result); + final deliveries = _parseDeliveries( + queue, + queueGroup!, + consumer, + result, + ); for (final delivery in deliveries) { if (controller.isClosed) { break; @@ -621,15 +626,16 @@ class RedisStreamsBroker implements Broker { unawaited(loop()); for (final stream in streamKeys) { - final key = _scheduleClaim(stream, group, consumer, controller); + final key = _scheduleClaim(stream, queueGroup!, consumer, controller); claimTimerKeys.add(key); } for (final channel in broadcastChannels) { - _listenBroadcast(channel, consumer, prefetch, controller); + final group = consumerGroup ?? _broadcastGroupKey(channel, consumer); + _listenBroadcast(channel, group, consumer, prefetch, controller); final key = _scheduleClaim( _broadcastStreamKey(channel), - _broadcastGroupKey(channel, consumer), + group, consumer, controller, ); @@ -640,6 +646,7 @@ class RedisStreamsBroker implements Broker { void _listenBroadcast( String channel, + String group, String consumer, int prefetch, StreamController controller, @@ -647,7 +654,6 @@ class RedisStreamsBroker implements Broker { unawaited( Future(() async { final streamKey = _broadcastStreamKey(channel); - final group = _broadcastGroupKey(channel, consumer); RedisConnection? broadcastConnection; Command? broadcastCommand; @@ -694,7 +700,7 @@ class RedisStreamsBroker implements Broker { } while (!controller.isClosed && !_closed) { - await _ensureBroadcastGroup(channel, consumer); + await _ensureBroadcastGroup(channel, group); dynamic result; try { result = await sendBroadcastCommand([ diff --git a/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart b/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart index 40dbd7b6..21ce6caa 100644 --- a/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart +++ b/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart @@ -35,9 +35,6 @@ void main() { ); }, dispose: (broker) async { - if (broker is _NoCloseBroker) { - return; - } await _safeCloseRedisBroker(broker as RedisStreamsBroker); }, additionalBrokerFactory: () async { @@ -54,7 +51,7 @@ void main() { claimInterval: const Duration(milliseconds: 200), blockTime: const Duration(milliseconds: 100), ); - return _NoCloseBroker(broker); + return broker; }, ), settings: const BrokerContractSettings( @@ -82,9 +79,6 @@ void main() { ); }, dispose: (broker) async { - if (broker is _NoCloseBroker) { - return; - } await _safeCloseRedisBroker(broker as RedisStreamsBroker); }, additionalBrokerFactory: () async { @@ -101,7 +95,7 @@ void main() { claimInterval: const Duration(milliseconds: 200), blockTime: const Duration(milliseconds: 100), ); - return _NoCloseBroker(broker); + return broker; }, ), ); @@ -389,92 +383,3 @@ Future _safeCloseRedisBroker(RedisStreamsBroker broker) async { // Ignore broker shutdown errors in cleanup. } } - -class _NoCloseBroker implements Broker { - _NoCloseBroker(this._inner); - - final RedisStreamsBroker _inner; - - @override - Future ack(Delivery delivery) => _inner.ack(delivery); - - @override - Future deadLetter( - Delivery delivery, { - String? reason, - Map? meta, - }) => _inner.deadLetter(delivery, reason: reason, meta: meta); - - @override - Future extendLease(Delivery delivery, Duration by) => - _inner.extendLease(delivery, by); - - @override - Future getDeadLetter(String queue, String id) => - _inner.getDeadLetter(queue, id); - - @override - Future inflightCount(String queue) => _inner.inflightCount(queue); - - @override - Future listDeadLetters( - String queue, { - int limit = 50, - int offset = 0, - }) => _inner.listDeadLetters(queue, limit: limit, offset: offset); - - @override - Future nack(Delivery delivery, {bool requeue = true}) => - _inner.nack(delivery, requeue: requeue); - - @override - Future pendingCount(String queue) => _inner.pendingCount(queue); - - @override - Future publish(Envelope envelope, {RoutingInfo? routing}) => - _inner.publish(envelope, routing: routing); - - @override - Future purge(String queue) => _inner.purge(queue); - - @override - Future purgeDeadLetters(String queue, {DateTime? since, int? limit}) => - _inner.purgeDeadLetters(queue, since: since, limit: limit); - - @override - Future replayDeadLetters( - String queue, { - int limit = 50, - DateTime? since, - Duration? delay, - bool dryRun = false, - }) => _inner.replayDeadLetters( - queue, - limit: limit, - since: since, - delay: delay, - dryRun: dryRun, - ); - - @override - bool get supportsDelayed => _inner.supportsDelayed; - - @override - bool get supportsPriority => _inner.supportsPriority; - - @override - Stream consume( - RoutingSubscription subscription, { - int prefetch = 1, - String? consumerGroup, - String? consumerName, - }) => _inner.consume( - subscription, - prefetch: prefetch, - consumerGroup: consumerGroup, - consumerName: consumerName, - ); - - @override - Future close() async {} -} From 0f44afd8ab790527e316da8e96c3a086aae0f9f3 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:04:10 -0500 Subject: [PATCH 16/23] fix(signals): freeze control command timestamps --- .../dashboard/lib/src/services/models.dart | 2 +- .../lib/src/brokers/in_memory_broker.dart | 4 +-- packages/stem/lib/src/signals/payloads.dart | 18 ++++++---- .../stem/test/unit/signals/payloads_test.dart | 33 +++++++++++++++++++ 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/packages/dashboard/lib/src/services/models.dart b/packages/dashboard/lib/src/services/models.dart index 058ca9a7..54730d0f 100644 --- a/packages/dashboard/lib/src/services/models.dart +++ b/packages/dashboard/lib/src/services/models.dart @@ -1,4 +1,4 @@ -import 'package:stem/stem.dart' show QueueHeartbeat, WorkerHeartbeat; +import 'package:stem/stem.dart' show QueueHeartbeat, WorkerHeartbeat, stemNow; /// Aggregate counts for a queue at a point in time. class QueueSummary { diff --git a/packages/stem/lib/src/brokers/in_memory_broker.dart b/packages/stem/lib/src/brokers/in_memory_broker.dart index ed710e3a..0f4fb0ee 100644 --- a/packages/stem/lib/src/brokers/in_memory_broker.dart +++ b/packages/stem/lib/src/brokers/in_memory_broker.dart @@ -340,9 +340,9 @@ class InMemoryBroker implements Broker { return !entry.deadAt.isBefore(since); }).toList()..sort((a, b) => b.deadAt.compareTo(a.deadAt)); final toRemove = - limit != null && limit >= 0 + (limit != null && limit >= 0 ? candidates.take(limit).toList() - : candidates + : candidates) ..forEach(state.deadLetters.remove); return toRemove.length; } diff --git a/packages/stem/lib/src/signals/payloads.dart b/packages/stem/lib/src/signals/payloads.dart index 5f8a5b76..a7017f6a 100644 --- a/packages/stem/lib/src/signals/payloads.dart +++ b/packages/stem/lib/src/signals/payloads.dart @@ -688,10 +688,11 @@ class ScheduleEntryFailedPayload implements StemEvent { /// Payload emitted when a control command is received by a worker. class ControlCommandReceivedPayload implements StemEvent { /// Creates a new [ControlCommandReceivedPayload] instance. - const ControlCommandReceivedPayload({ + ControlCommandReceivedPayload({ required this.worker, required this.command, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The worker that received the command. final WorkerInfo worker; @@ -699,11 +700,13 @@ class ControlCommandReceivedPayload implements StemEvent { /// The control command that was received. final ControlCommandMessage command; + final DateTime _occurredAt; + @override String get eventName => 'control-command-received'; @override - DateTime get occurredAt => stemNow().toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { @@ -719,13 +722,14 @@ class ControlCommandReceivedPayload implements StemEvent { /// Payload emitted when a control command completes execution. class ControlCommandCompletedPayload implements StemEvent { /// Creates a new [ControlCommandCompletedPayload] instance. - const ControlCommandCompletedPayload({ + ControlCommandCompletedPayload({ required this.worker, required this.command, required this.status, this.response, this.error, - }); + DateTime? occurredAt, + }) : _occurredAt = (occurredAt ?? stemNow()).toUtc(); /// The worker that executed the command. final WorkerInfo worker; @@ -742,11 +746,13 @@ class ControlCommandCompletedPayload implements StemEvent { /// Error information if the command failed, if any. final Map? error; + final DateTime _occurredAt; + @override String get eventName => 'control-command-completed'; @override - DateTime get occurredAt => stemNow().toUtc(); + DateTime get occurredAt => _occurredAt; @override Map get attributes => { diff --git a/packages/stem/test/unit/signals/payloads_test.dart b/packages/stem/test/unit/signals/payloads_test.dart index a71b74fb..9a5c013b 100644 --- a/packages/stem/test/unit/signals/payloads_test.dart +++ b/packages/stem/test/unit/signals/payloads_test.dart @@ -1,5 +1,7 @@ import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/clock.dart'; import 'package:stem/src/core/envelope.dart'; +import 'package:stem/src/control/control_messages.dart'; import 'package:stem/src/signals/payloads.dart'; import 'package:test/test.dart'; @@ -66,4 +68,35 @@ void main() { equals(DateTime.utc(2025).toIso8601String()), ); }); + + test('control command payload timestamps are frozen at creation', () { + const worker = WorkerInfo( + id: 'worker-1', + queues: ['default'], + broadcasts: [], + ); + final command = ControlCommandMessage( + requestId: 'req-1', + type: 'pause', + targets: const ['*'], + ); + final clock = FakeStemClock(DateTime.utc(2025, 1, 1, 0, 0, 0)); + + withStemClock(clock, () { + final received = ControlCommandReceivedPayload( + worker: worker, + command: command, + ); + clock.advance(const Duration(minutes: 1)); + final completed = ControlCommandCompletedPayload( + worker: worker, + command: command, + status: 'ok', + ); + clock.advance(const Duration(minutes: 1)); + + expect(received.occurredAt, DateTime.utc(2025, 1, 1, 0, 0, 0)); + expect(completed.occurredAt, DateTime.utc(2025, 1, 1, 0, 1, 0)); + }); + }); } From 3de91d1b220f5a306630754b31d61b929ffe95ce Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:06:26 -0500 Subject: [PATCH 17/23] fix(redis): remove unnecessary non-null assertions in consume loop --- .../stem_redis/lib/src/brokers/redis_broker.dart | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index ce0743cb..29eaeedd 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -570,9 +570,10 @@ class RedisStreamsBroker implements Broker { } Future loop() async { - if (queue == null) { + if (queue == null || queueGroup == null) { return; } + final group = queueGroup; while (!controller.isClosed && !_closed) { for (final stream in streamKeys) { await _ensureGroupForStream(queue, stream); @@ -583,7 +584,7 @@ class RedisStreamsBroker implements Broker { result = await sendConsumerCommand([ 'XREADGROUP', 'GROUP', - queueGroup!, + group, consumer, 'BLOCK', blockTime.inMilliseconds.toString(), @@ -611,7 +612,7 @@ class RedisStreamsBroker implements Broker { } final deliveries = _parseDeliveries( queue, - queueGroup!, + group, consumer, result, ); @@ -626,8 +627,10 @@ class RedisStreamsBroker implements Broker { unawaited(loop()); for (final stream in streamKeys) { - final key = _scheduleClaim(stream, queueGroup!, consumer, controller); - claimTimerKeys.add(key); + if (queueGroup != null) { + final key = _scheduleClaim(stream, queueGroup, consumer, controller); + claimTimerKeys.add(key); + } } for (final channel in broadcastChannels) { From 39078ca4acfe52e9813eefb2f4ae0dccf1d7ebd2 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:10:20 -0500 Subject: [PATCH 18/23] fix(redis): retain broadcast fan-out for queue subscriptions --- packages/stem_redis/lib/src/brokers/redis_broker.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 29eaeedd..c35aa7cf 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -634,7 +634,9 @@ class RedisStreamsBroker implements Broker { } for (final channel in broadcastChannels) { - final group = consumerGroup ?? _broadcastGroupKey(channel, consumer); + final group = queue == null && consumerGroup != null + ? consumerGroup + : _broadcastGroupKey(channel, consumer); _listenBroadcast(channel, group, consumer, prefetch, controller); final key = _scheduleClaim( _broadcastStreamKey(channel), From 603e80597f8fa15708e07b42de095acad2172090 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:13:10 -0500 Subject: [PATCH 19/23] fix(redis): repair broadcast NOGROUP recovery path --- packages/stem_redis/lib/src/brokers/redis_broker.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index c35aa7cf..5653a7d3 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -727,7 +727,7 @@ class RedisStreamsBroker implements Broker { } if ('$error'.contains('NOGROUP')) { _groupsCreated.remove('$streamKey|$group'); - await _ensureBroadcastGroup(channel, consumer); + await _ensureBroadcastGroup(channel, group); continue; } rethrow; From 1ebe7e9d16449881c36a8d4060f6a0e0d20d73a4 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:16:19 -0500 Subject: [PATCH 20/23] fix(redis): guard delivery emission after stream cancellation --- .../lib/src/brokers/redis_broker.dart | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 5653a7d3..d8a97eea 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -617,10 +617,9 @@ class RedisStreamsBroker implements Broker { result, ); for (final delivery in deliveries) { - if (controller.isClosed) { + if (!_tryAddDelivery(controller, delivery)) { break; } - controller.add(delivery); } } } @@ -737,10 +736,9 @@ class RedisStreamsBroker implements Broker { } final deliveries = _parseDeliveries(channel, group, consumer, result); for (final delivery in deliveries) { - if (controller.isClosed) { + if (!_tryAddDelivery(controller, delivery)) { break; } - controller.add(delivery); } } if (broadcastConnection != null && @@ -755,6 +753,24 @@ class RedisStreamsBroker implements Broker { ); } + bool _tryAddDelivery( + StreamController controller, + Delivery delivery, + ) { + if (controller.isClosed) { + return false; + } + try { + controller.add(delivery); + return true; + } on Object catch (error) { + if (error is StateError && '$error'.contains('stream is closed')) { + return false; + } + rethrow; + } + } + List _parseDeliveries( String queue, String group, From 23139abb8ab6939bf3b1ab69bb5ad6c64c0a1c68 Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:19:20 -0500 Subject: [PATCH 21/23] fix(redis): harden claim timer delivery on closed streams --- packages/stem_redis/lib/src/brokers/redis_broker.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index d8a97eea..586030a7 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -941,10 +941,9 @@ class RedisStreamsBroker implements Broker { entries: entries, ); for (final delivery in deliveries) { - if (controller.isClosed) { + if (!_tryAddDelivery(controller, delivery)) { break; } - controller.add(delivery); } } } From 99c3dc327869fae8e6b13e7553e35b207a4da34f Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:24:43 -0500 Subject: [PATCH 22/23] test(adapters): avoid auto-cancel race in broadcast fanout contract --- .../lib/src/broker_contract_suite.dart | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart b/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart index c9d27a0d..cd69c0ad 100644 --- a/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart +++ b/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart @@ -781,30 +781,39 @@ void runBrokerContractTests({ broadcastChannels: [channel], ); - final futureOne = workerOne.consume(subscription).first; - final futureTwo = workerTwo.consume(subscription).first; - - await publisher.publish( - Envelope( - name: 'contract.broadcast', - args: const {'value': 'fanout'}, - queue: queue, - ), - routing: RoutingInfo.broadcast(channel: channel), - ); - - final deliveryOne = await futureOne.timeout( - const Duration(seconds: 5), - ); - final deliveryTwo = await futureTwo.timeout( - const Duration(seconds: 5), - ); - - expect(deliveryOne.envelope.name, 'contract.broadcast'); - expect(deliveryTwo.envelope.name, 'contract.broadcast'); - - await workerOne.ack(deliveryOne); - await workerTwo.ack(deliveryTwo); + final iteratorOne = StreamIterator(workerOne.consume(subscription)); + final iteratorTwo = StreamIterator(workerTwo.consume(subscription)); + + try { + await publisher.publish( + Envelope( + name: 'contract.broadcast', + args: const {'value': 'fanout'}, + queue: queue, + ), + routing: RoutingInfo.broadcast(channel: channel), + ); + + expect( + await iteratorOne.moveNext().timeout(const Duration(seconds: 5)), + isTrue, + ); + expect( + await iteratorTwo.moveNext().timeout(const Duration(seconds: 5)), + isTrue, + ); + final deliveryOne = iteratorOne.current; + final deliveryTwo = iteratorTwo.current; + + expect(deliveryOne.envelope.name, 'contract.broadcast'); + expect(deliveryTwo.envelope.name, 'contract.broadcast'); + + await workerOne.ack(deliveryOne); + await workerTwo.ack(deliveryTwo); + } finally { + await iteratorOne.cancel(); + await iteratorTwo.cancel(); + } } finally { if (factory.dispose != null) { await factory.dispose!(workerOne); From 1e0d73ae1f5a6372ebf6d0116cd0bc0d6e33748f Mon Sep 17 00:00:00 2001 From: kingwill101 Date: Tue, 24 Feb 2026 14:29:11 -0500 Subject: [PATCH 23/23] test(adapters): pre-arm broadcast listeners before fanout publish --- .../stem_adapter_tests/lib/src/broker_contract_suite.dart | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart b/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart index cd69c0ad..a50ca4de 100644 --- a/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart +++ b/packages/stem_adapter_tests/lib/src/broker_contract_suite.dart @@ -785,6 +785,9 @@ void runBrokerContractTests({ final iteratorTwo = StreamIterator(workerTwo.consume(subscription)); try { + final nextOne = iteratorOne.moveNext(); + final nextTwo = iteratorTwo.moveNext(); + await Future.delayed(settings.queueSettleDelay); await publisher.publish( Envelope( name: 'contract.broadcast', @@ -795,11 +798,11 @@ void runBrokerContractTests({ ); expect( - await iteratorOne.moveNext().timeout(const Duration(seconds: 5)), + await nextOne.timeout(const Duration(seconds: 5)), isTrue, ); expect( - await iteratorTwo.moveNext().timeout(const Duration(seconds: 5)), + await nextTwo.timeout(const Duration(seconds: 5)), isTrue, ); final deliveryOne = iteratorOne.current;