diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..89a23f1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,54 @@ +# Abada Engine Documentation + +Welcome to the Abada Engine documentation. This documentation is organized to help you quickly find the information you need, whether you are an architect, developer, or operator. + +## 📚 Table of Contents + +### 🏗️ Architecture + +High-level design and architectural decisions. + +- [Overview](architecture/overview.md) - The big picture of Abada Engine's architecture and deployment. +- [Authentication Service](architecture/auth-service.md) - How authentication and authorization work. +- [Event Delivery](architecture/event-delivery.md) - Mechanics of event processing and delivery. +- [SPI Design](architecture/spi-design.md) - Service Provider Interface design for extensibility. +- [Event-Based Gateways](architecture/event-based-gateways-design.md) - Design of event-based gateways. + +### 🚀 Features + +Detailed guides on specific engine features. + +- [Process Variables](features/process-variables.md) - Handling data within processes. +- [Service Tasks](features/service-tasks.md) - Implementing and using service tasks. +- [Exclusive Gateway](features/exclusive-gateway.md) - Logic and usage of exclusive gateways. +- [Persistence](features/persistence.md) - How data is stored and managed. +- [Kitchen Sink Process](features/kitchen-sink-process.md) - A comprehensive example process demonstrating various features. + +### 🛠️ Operations + +Guides for deploying, monitoring, and managing the engine. + +- [Docker Deployment](operations/docker-deployment.md) - Deploying Abada Engine using Docker. +- [Docker Build Strategy](operations/docker-build-strategy.md) - How container images are built. +- [Observability](operations/observability.md) - Monitoring, metrics, tracing, and logging. +- [Loki Integration](operations/loki-integration.md) - Setting up log aggregation with Loki. +- [Sample Data Generator](operations/sample-data-generator.md) - Generating test data for development. + +### 💻 Development + +Resources for developers building on or contributing to Abada Engine. + +- [API Documentation](development/api.md) - REST API reference. +- [Frontend Guide](development/frontend-guide.md) - Guide for frontend development. +- [Orun App Specification](development/orun-app-spec.md) - Specifications for the Orun application. +- [Project Configuration](development/project-config.md) - Review of project configuration settings. +- [Roadmap](development/roadmap.md) - Future plans and upcoming features. + +### 📝 Release Notes + +History of changes and updates. + +- [Release Notes](release-notes/) - Directory containing release notes for all versions. + +--- +*Documentation structure organized for clarity and ease of navigation.* diff --git a/docs/auth-service-archictecture.md b/docs/architecture/auth-service.md similarity index 100% rename from docs/auth-service-archictecture.md rename to docs/architecture/auth-service.md diff --git a/docs/event-based-gateways-design.md b/docs/architecture/event-based-gateways-design.md similarity index 100% rename from docs/event-based-gateways-design.md rename to docs/architecture/event-based-gateways-design.md diff --git a/docs/event-delivery-mechanics.md b/docs/architecture/event-delivery.md similarity index 100% rename from docs/event-delivery-mechanics.md rename to docs/architecture/event-delivery.md diff --git a/docs/architecture-and-deployment-guide.md b/docs/architecture/overview.md similarity index 100% rename from docs/architecture-and-deployment-guide.md rename to docs/architecture/overview.md diff --git a/docs/spi-design.md b/docs/architecture/spi-design.md similarity index 100% rename from docs/spi-design.md rename to docs/architecture/spi-design.md diff --git a/docs/abada_architecture_doc.md b/docs/archive/abada_architecture_doc.md similarity index 100% rename from docs/abada_architecture_doc.md rename to docs/archive/abada_architecture_doc.md diff --git a/docs/docker-deployment-plan.md b/docs/archive/docker-deployment-plan.md similarity index 100% rename from docs/docker-deployment-plan.md rename to docs/archive/docker-deployment-plan.md diff --git a/docs/sample-data-generator-summary.md b/docs/archive/sample-data-generator-summary.md similarity index 100% rename from docs/sample-data-generator-summary.md rename to docs/archive/sample-data-generator-summary.md diff --git a/docs/service-task-design.md b/docs/archive/service-task-design.md similarity index 100% rename from docs/service-task-design.md rename to docs/archive/service-task-design.md diff --git a/docs/telemetry-implementation-plan.md b/docs/archive/telemetry-implementation-plan.md similarity index 100% rename from docs/telemetry-implementation-plan.md rename to docs/archive/telemetry-implementation-plan.md diff --git a/docs/api-documentation.md b/docs/development/api.md similarity index 100% rename from docs/api-documentation.md rename to docs/development/api.md diff --git a/docs/frontend-development-prompt.md b/docs/development/frontend-guide.md similarity index 100% rename from docs/frontend-development-prompt.md rename to docs/development/frontend-guide.md diff --git a/docs/orun-app-specification.md b/docs/development/orun-app-spec.md similarity index 100% rename from docs/orun-app-specification.md rename to docs/development/orun-app-spec.md diff --git a/docs/project-configuration-review.md b/docs/development/project-config.md similarity index 100% rename from docs/project-configuration-review.md rename to docs/development/project-config.md diff --git a/docs/roadmap-to-beta.md b/docs/development/roadmap.md similarity index 100% rename from docs/roadmap-to-beta.md rename to docs/development/roadmap.md diff --git a/docs/exclusive-gateway.md b/docs/features/exclusive-gateway.md similarity index 100% rename from docs/exclusive-gateway.md rename to docs/features/exclusive-gateway.md diff --git a/docs/kitchen-sink-process.md b/docs/features/kitchen-sink-process.md similarity index 100% rename from docs/kitchen-sink-process.md rename to docs/features/kitchen-sink-process.md diff --git a/docs/persistence.md b/docs/features/persistence.md similarity index 100% rename from docs/persistence.md rename to docs/features/persistence.md diff --git a/docs/process-variables.md b/docs/features/process-variables.md similarity index 100% rename from docs/process-variables.md rename to docs/features/process-variables.md diff --git a/docs/service-tasks-impl.md b/docs/features/service-tasks.md similarity index 100% rename from docs/service-tasks-impl.md rename to docs/features/service-tasks.md diff --git a/docs/observability-implementation.md b/docs/observability-implementation.md deleted file mode 100755 index 8b0b6dc..0000000 --- a/docs/observability-implementation.md +++ /dev/null @@ -1,237 +0,0 @@ -# Observability Setup - -## Overview -This document describes the observability setup for the Abada Engine, including metrics collection, distributed tracing, and log aggregation. The implementation uses OpenTelemetry for instrumentation, Prometheus for metrics collection, Jaeger for distributed tracing, Loki for log aggregation, and Grafana for unified visualization. - -The observability stack implements the **three pillars of observability**: -- **Metrics**: Prometheus + Micrometer -- **Traces**: Jaeger + OpenTelemetry -- **Logs**: Loki + OpenTelemetry Logback Appender - -## Metrics Implementation - -### Core Metrics -The engine captures the following key metrics: - -1. Task Metrics - - Task creation rate (`abada_tasks_created_total`) - - Task completion rate (`abada_tasks_completed_total`) - - Task failure rate (`abada_tasks_failed_total`) - - Task waiting time (`abada_task_waiting_time`) - - Task processing time (`abada_task_processing_time`) - -2. Process Metrics - - Process instance creation rate - - Process completion rate - - Process execution time - - Active process instances - -3. Event Metrics - - Event processing rate - - Event delivery time - - Failed event deliveries - -## Monitoring Dashboards - -### Overview Dashboard -Located at `monitoring/grafana/dashboards/abada-engine-overview.json` - -Provides a high-level view of the engine's performance including: -- Process execution metrics -- Task completion rates -- Event processing statistics -- System health indicators - -### Task Details Dashboard -Located at `monitoring/grafana/dashboards/abada-task-details.json` - -Offers detailed task-level metrics: -- Task waiting and processing time distributions (p50, p95) -- Task creation and completion rates by type -- Success/failure ratios -- Completion rate gauges with thresholds - -## Testing Implementation - -The observability components are properly tested with mocked dependencies: - -```java -// Example test setup -@BeforeEach -void setUp() { - engineMetrics = mock(EngineMetrics.class); - tracer = mock(Tracer.class); - span = mock(Span.class); - - // Setup timer samples - when(engineMetrics.startTaskTimer()).thenReturn(timerSample); - - // Setup span creation - when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); - when(spanBuilder.startSpan()).thenReturn(span); -} -``` - -## Integration with OpenTelemetry - -The engine uses OpenTelemetry for distributed tracing and metrics collection: - -1. Span Creation - - Task execution spans - - Process instance spans - - Event processing spans - -2. Metric Recording - - Timer-based metrics for duration measurements - - Counter-based metrics for rate calculations - - Gauge-based metrics for current state - -## Prometheus Configuration - -Metrics are exposed via a Prometheus endpoint at `/actuator/prometheus`. The metrics follow the Prometheus naming conventions and include appropriate labels for aggregation and filtering. - -## Log Aggregation with Loki - -### Overview -Loki is integrated as the centralized log aggregation system, receiving logs from the application via the OpenTelemetry Collector. This provides: -- Centralized log storage and querying -- Automatic correlation between logs and traces using trace IDs -- Efficient log storage with label-based indexing -- Seamless integration with Grafana for log visualization - -### Architecture -``` -Spring Boot App → OpenTelemetry Logback Appender → OTel Collector → Loki → Grafana -``` - -### Configuration - -#### Application Configuration -The application uses the OpenTelemetry Logback Appender to send logs via OTLP: - -**logback-spring.xml**: -```xml - - true - true - * - -``` - -**application.yaml**: -```yaml -management: - otlp: - logs: - endpoint: http://otel-collector:4318/v1/logs -``` - -#### OpenTelemetry Collector -The OTel Collector receives logs via OTLP and forwards them to Loki: - -```yaml -exporters: - otlphttp/loki: - endpoint: http://loki:3100/otlp - tls: - insecure: true - -service: - pipelines: - logs: - receivers: [otlp] - processors: [memory_limiter, batch, resource] - exporters: [otlphttp/loki, debug] -``` - -#### Loki Configuration -Loki is configured with: -- **Development**: In-memory ring for single-node operation -- **Production**: Consul-based ring for distributed coordination -- BoltDB shipper for index storage -- Filesystem storage for chunks -- 14-day retention period - -### Querying Logs - -#### Via Grafana Explore -1. Navigate to Grafana Explore view -2. Select Loki as the data source -3. Use LogQL queries: - ``` - {service_name="abada-engine"} - {service_name="abada-engine"} |= "error" - {service_name="abada-engine"} | json | level="ERROR" - ``` - -#### Via API -```bash -curl -G -s "http://localhost:3100/loki/api/v1/query_range" \ - --data-urlencode 'query={service_name="abada-engine"}' \ - --data-urlencode "start=$(date -d '5 minutes ago' +%s)000000000" \ - --data-urlencode "end=$(date +%s)000000000" -``` - -### Trace-to-Logs Correlation -Logs automatically include `traceId` and `spanId` from the MDC context, enabling: -- Jump from traces in Jaeger to related logs in Loki -- Filter logs by specific trace IDs -- Correlate distributed operations across services - -Example LogQL query for a specific trace: -``` -{service_name="abada-engine"} | json | traceId="abc123def456" -``` - -## Dashboard Setup - -To use the provided dashboards: - -1. Import the JSON files into your Grafana instance: - - `abada-engine-overview.json` - - `abada-task-details.json` - -2. Configure the Prometheus data source in Grafana - -3. The dashboards will automatically populate with data once the engine starts generating metrics - -## Best Practices - -1. Metric Naming - - Use the `abada_` prefix for all metrics - - Include units in metric names where applicable - - Use appropriate metric types (counter, gauge, histogram) - -2. Tracing - - Create spans for all significant operations - - Include relevant attributes in spans - - Maintain proper parent-child relationships - -3. Testing - - Mock all observability dependencies - - Verify metric recording in tests - - Test span lifecycle management - -## Future Enhancements - -1. Additional Dashboards - - Event processing details - - Error analysis dashboard - - Resource utilization metrics - - Log analytics dashboard with common queries - -2. Alerting - - Configure alerts for critical metrics - - Set up notification channels - - Define SLOs and SLIs - - Log-based alerting rules - -3. Extended Metrics - - Database operation metrics - - External service call metrics - - Resource pool metrics - -4. Advanced Log Features - - Log sampling for high-volume scenarios - - Structured logging enhancements - - Log-based metrics derivation \ No newline at end of file diff --git a/docs/docker-build-strategy.md b/docs/operations/docker-build-strategy.md similarity index 100% rename from docs/docker-build-strategy.md rename to docs/operations/docker-build-strategy.md diff --git a/docs/docker-deployment.md b/docs/operations/docker-deployment.md similarity index 100% rename from docs/docker-deployment.md rename to docs/operations/docker-deployment.md diff --git a/docs/loki-integration-walkthrough.md b/docs/operations/loki-integration.md similarity index 100% rename from docs/loki-integration-walkthrough.md rename to docs/operations/loki-integration.md diff --git a/docs/observability-reference-guide.md b/docs/operations/observability.md similarity index 67% rename from docs/observability-reference-guide.md rename to docs/operations/observability.md index be8b88a..aa69f77 100755 --- a/docs/observability-reference-guide.md +++ b/docs/operations/observability.md @@ -499,3 +499,244 @@ otel: - [ ] Advanced correlation with external systems - [ ] Automated anomaly detection - [ ] Performance optimization recommendations + + +# Implementation Details + +# Observability Setup + +## Overview +This document describes the observability setup for the Abada Engine, including metrics collection, distributed tracing, and log aggregation. The implementation uses OpenTelemetry for instrumentation, Prometheus for metrics collection, Jaeger for distributed tracing, Loki for log aggregation, and Grafana for unified visualization. + +The observability stack implements the **three pillars of observability**: +- **Metrics**: Prometheus + Micrometer +- **Traces**: Jaeger + OpenTelemetry +- **Logs**: Loki + OpenTelemetry Logback Appender + +## Metrics Implementation + +### Core Metrics +The engine captures the following key metrics: + +1. Task Metrics + - Task creation rate (`abada_tasks_created_total`) + - Task completion rate (`abada_tasks_completed_total`) + - Task failure rate (`abada_tasks_failed_total`) + - Task waiting time (`abada_task_waiting_time`) + - Task processing time (`abada_task_processing_time`) + +2. Process Metrics + - Process instance creation rate + - Process completion rate + - Process execution time + - Active process instances + +3. Event Metrics + - Event processing rate + - Event delivery time + - Failed event deliveries + +## Monitoring Dashboards + +### Overview Dashboard +Located at `monitoring/grafana/dashboards/abada-engine-overview.json` + +Provides a high-level view of the engine's performance including: +- Process execution metrics +- Task completion rates +- Event processing statistics +- System health indicators + +### Task Details Dashboard +Located at `monitoring/grafana/dashboards/abada-task-details.json` + +Offers detailed task-level metrics: +- Task waiting and processing time distributions (p50, p95) +- Task creation and completion rates by type +- Success/failure ratios +- Completion rate gauges with thresholds + +## Testing Implementation + +The observability components are properly tested with mocked dependencies: + +```java +// Example test setup +@BeforeEach +void setUp() { + engineMetrics = mock(EngineMetrics.class); + tracer = mock(Tracer.class); + span = mock(Span.class); + + // Setup timer samples + when(engineMetrics.startTaskTimer()).thenReturn(timerSample); + + // Setup span creation + when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); + when(spanBuilder.startSpan()).thenReturn(span); +} +``` + +## Integration with OpenTelemetry + +The engine uses OpenTelemetry for distributed tracing and metrics collection: + +1. Span Creation + - Task execution spans + - Process instance spans + - Event processing spans + +2. Metric Recording + - Timer-based metrics for duration measurements + - Counter-based metrics for rate calculations + - Gauge-based metrics for current state + +## Prometheus Configuration + +Metrics are exposed via a Prometheus endpoint at `/actuator/prometheus`. The metrics follow the Prometheus naming conventions and include appropriate labels for aggregation and filtering. + +## Log Aggregation with Loki + +### Overview +Loki is integrated as the centralized log aggregation system, receiving logs from the application via the OpenTelemetry Collector. This provides: +- Centralized log storage and querying +- Automatic correlation between logs and traces using trace IDs +- Efficient log storage with label-based indexing +- Seamless integration with Grafana for log visualization + +### Architecture +``` +Spring Boot App → OpenTelemetry Logback Appender → OTel Collector → Loki → Grafana +``` + +### Configuration + +#### Application Configuration +The application uses the OpenTelemetry Logback Appender to send logs via OTLP: + +**logback-spring.xml**: +```xml + + true + true + * + +``` + +**application.yaml**: +```yaml +management: + otlp: + logs: + endpoint: http://otel-collector:4318/v1/logs +``` + +#### OpenTelemetry Collector +The OTel Collector receives logs via OTLP and forwards them to Loki: + +```yaml +exporters: + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [otlphttp/loki, debug] +``` + +#### Loki Configuration +Loki is configured with: +- **Development**: In-memory ring for single-node operation +- **Production**: Consul-based ring for distributed coordination +- BoltDB shipper for index storage +- Filesystem storage for chunks +- 14-day retention period + +### Querying Logs + +#### Via Grafana Explore +1. Navigate to Grafana Explore view +2. Select Loki as the data source +3. Use LogQL queries: + ``` + {service_name="abada-engine"} + {service_name="abada-engine"} |= "error" + {service_name="abada-engine"} | json | level="ERROR" + ``` + +#### Via API +```bash +curl -G -s "http://localhost:3100/loki/api/v1/query_range" \ + --data-urlencode 'query={service_name="abada-engine"}' \ + --data-urlencode "start=$(date -d '5 minutes ago' +%s)000000000" \ + --data-urlencode "end=$(date +%s)000000000" +``` + +### Trace-to-Logs Correlation +Logs automatically include `traceId` and `spanId` from the MDC context, enabling: +- Jump from traces in Jaeger to related logs in Loki +- Filter logs by specific trace IDs +- Correlate distributed operations across services + +Example LogQL query for a specific trace: +``` +{service_name="abada-engine"} | json | traceId="abc123def456" +``` + +## Dashboard Setup + +To use the provided dashboards: + +1. Import the JSON files into your Grafana instance: + - `abada-engine-overview.json` + - `abada-task-details.json` + +2. Configure the Prometheus data source in Grafana + +3. The dashboards will automatically populate with data once the engine starts generating metrics + +## Best Practices + +1. Metric Naming + - Use the `abada_` prefix for all metrics + - Include units in metric names where applicable + - Use appropriate metric types (counter, gauge, histogram) + +2. Tracing + - Create spans for all significant operations + - Include relevant attributes in spans + - Maintain proper parent-child relationships + +3. Testing + - Mock all observability dependencies + - Verify metric recording in tests + - Test span lifecycle management + +## Future Enhancements + +1. Additional Dashboards + - Event processing details + - Error analysis dashboard + - Resource utilization metrics + - Log analytics dashboard with common queries + +2. Alerting + - Configure alerts for critical metrics + - Set up notification channels + - Define SLOs and SLIs + - Log-based alerting rules + +3. Extended Metrics + - Database operation metrics + - External service call metrics + - Resource pool metrics + +4. Advanced Log Features + - Log sampling for high-volume scenarios + - Structured logging enhancements + - Log-based metrics derivation \ No newline at end of file diff --git a/docs/sample-data-generator-config.example b/docs/operations/sample-data-generator-config.example similarity index 100% rename from docs/sample-data-generator-config.example rename to docs/operations/sample-data-generator-config.example diff --git a/docs/sample-data-generator.md b/docs/operations/sample-data-generator.md similarity index 51% rename from docs/sample-data-generator.md rename to docs/operations/sample-data-generator.md index aa4c4ba..ae0d6bb 100644 --- a/docs/sample-data-generator.md +++ b/docs/operations/sample-data-generator.md @@ -229,3 +229,251 @@ To customize the sample data: - Verify that user and group assignments match the BPMN definitions - Check the candidate users and groups defined in the BPMN files +# Docker Setup for Sample Data Generator + +## Quick Answer + +**Yes!** The `abada.generate-sample-data=true` property works in Docker. You just need to pass it as an environment variable. + +## How to Enable in Docker + +### Option 1: Environment Variable in docker-compose.yml + +Add to the `abada-engine` service environment section: + +```yaml +services: + abada-engine: + environment: + - ABADA_GENERATE_SAMPLE_DATA=true +``` + +### Option 2: Using .env File + +Create or edit `.env` file in the project root: + +```bash +# .env +ABADA_GENERATE_SAMPLE_DATA=true +``` + +Then reference it in docker-compose.yml: + +```yaml +services: + abada-engine: + environment: + - ABADA_GENERATE_SAMPLE_DATA=${ABADA_GENERATE_SAMPLE_DATA:-false} +``` + +### Option 3: Command Line Override + +```bash +# For dev environment +ABADA_GENERATE_SAMPLE_DATA=true docker-compose -f docker-compose.yml -f docker-compose.dev.yml up + +# For prod environment +ABADA_GENERATE_SAMPLE_DATA=true docker-compose -f docker-compose.yml -f docker-compose.prod.yml up +``` + +## Complete Example: docker-compose.dev.yml + +Here's what your `docker-compose.dev.yml` would look like with sample data enabled: + +```yaml +version: '3.8' + +services: + abada-engine: + build: + context: . + dockerfile: Dockerfile + args: + USE_LOCAL_JAR: "true" + image: abada-engine:dev + container_name: abada-engine + environment: + - SPRING_PROFILES_ACTIVE=dev + - SERVER_PORT=5601 + - DB_PASSWORD=${DB_PASSWORD:-abada123} + - MANAGEMENT_TRACING_SAMPLING_PROBABILITY=1.0 + - MANAGEMENT_OTLP_TRACING_ENDPOINT=http://otel-collector:4318/v1/traces + - MANAGEMENT_OTLP_METRICS_ENDPOINT=http://otel-collector:4318/v1/metrics + - OTEL_RESOURCE_ATTRIBUTES_DEPLOYMENT_ENVIRONMENT=dev + - OTEL_SERVICE_NAME=abada-engine-dev + - OTEL_RESOURCE_ATTRIBUTES=project=abada,service.version=0.8.3-alpha + # Enable sample data generation + - ABADA_GENERATE_SAMPLE_DATA=${ABADA_GENERATE_SAMPLE_DATA:-false} + ports: + - "5601:5601" + volumes: + - ./data:/app/data:z + - ./logs:/app/logs:z + networks: + - abada-network + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:5601/abada/api/actuator/health" ] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + otel-collector: + condition: service_started +``` + +## Environment Variable Naming + +Spring Boot automatically converts environment variables: + +- `ABADA_GENERATE_SAMPLE_DATA` → `abada.generate-sample-data` +- Underscores (`_`) become dots (`.`) +- Uppercase becomes lowercase + +## Usage Examples + +### Development with Sample Data + +```bash +# Set environment variable +export ABADA_GENERATE_SAMPLE_DATA=true + +# Start dev stack +./scripts/start-dev.sh + +# Or directly with docker-compose +docker-compose -f docker-compose.yml -f docker-compose.dev.yml up +``` + +### One-Time Sample Data Load + +```bash +# Start with sample data, then stop +ABADA_GENERATE_SAMPLE_DATA=true docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d + +# Wait for data to be generated (check logs) +docker logs -f abada-engine + +# Once you see "Sample Data Generation Complete", you can restart without the flag +docker-compose -f docker-compose.yml -f docker-compose.dev.yml restart abada-engine +``` + +### Production (Disabled by Default) + +```yaml +# docker-compose.prod.yml +services: + abada-engine: + environment: + # Explicitly disable in production + - ABADA_GENERATE_SAMPLE_DATA=false +``` + +## Verification in Docker + +After starting the container with sample data enabled: + +```bash +# Check logs to see sample data generation +docker logs abada-engine | grep "Sample Data" + +# Expected output: +# Starting Sample Data Generation +# Deployed recipe-cook.bpmn +# Deployed parallel-gateway-test.bpmn +# Scenario 1: Completed Recipe Process +# ... +# Sample Data Generation Complete + +# Verify via API +curl http://localhost:5601/abada/api/v1/processes/instances + +# Check tasks +curl -H "X-User: alice" -H "X-Groups: customers" \ + http://localhost:5601/abada/api/v1/tasks +``` + +## Important Notes + +### Data Persistence + +- **H2 (Dev)**: Data persists in `./data` volume +- **PostgreSQL (Prod)**: Data persists in database +- Sample data is generated **once** on startup +- Restarting the container won't regenerate data (unless you clear the database) + +### When to Use + +✅ **Good for:** + +- Demo environments +- Development testing +- UI development +- Integration testing + +❌ **Avoid for:** + +- Production environments +- Performance testing (use dedicated test data) +- Load testing + +### Troubleshooting + +**Sample data not appearing:** + +```bash +# Check if environment variable is set +docker exec abada-engine env | grep ABADA + +# Check application logs +docker logs abada-engine | grep -i "sample" + +# Verify the property is recognized +docker exec abada-engine cat /app/application.properties +``` + +**BPMN files not found:** + +```bash +# Verify BPMN files are in the image +docker exec abada-engine ls -la /app/BOOT-INF/classes/bpmn/ + +# Check test resources are included +docker exec abada-engine find /app -name "*.bpmn" +``` + +## Shell Script for Docker + +Create `scripts/start-dev-with-sample-data.sh`: + +```bash +#!/bin/bash +set -e + +echo "Starting Abada Engine (Dev) with Sample Data..." + +export ABADA_GENERATE_SAMPLE_DATA=true + +./scripts/start-dev.sh + +echo "" +echo "Sample data will be generated on first startup." +echo "Check logs: docker logs -f abada-engine" +``` + +Make it executable: + +```bash +chmod +x scripts/start-dev-with-sample-data.sh +``` + +## Summary + +The sample data generator works seamlessly in Docker by: + +1. Adding `ABADA_GENERATE_SAMPLE_DATA=true` to environment variables +2. Spring Boot automatically converts it to `abada.generate-sample-data=true` +3. The generator runs on container startup +4. Data persists in the configured database/volume + +For the easiest setup, add the environment variable to your `.env` file or docker-compose configuration! diff --git a/docs/0.7.0-alpha-release-notes.md b/docs/release-notes/0.7.0-alpha-release-notes.md similarity index 100% rename from docs/0.7.0-alpha-release-notes.md rename to docs/release-notes/0.7.0-alpha-release-notes.md diff --git a/docs/0.8.2-alpha-release-notes.md b/docs/release-notes/0.8.2-alpha-release-notes.md similarity index 100% rename from docs/0.8.2-alpha-release-notes.md rename to docs/release-notes/0.8.2-alpha-release-notes.md diff --git a/docs/0.8.3-alpha-release-notes.md b/docs/release-notes/0.8.3-alpha-release-notes.md similarity index 100% rename from docs/0.8.3-alpha-release-notes.md rename to docs/release-notes/0.8.3-alpha-release-notes.md diff --git a/docs/sample-data-generator-docker.md b/docs/sample-data-generator-docker.md deleted file mode 100644 index 27e1867..0000000 --- a/docs/sample-data-generator-docker.md +++ /dev/null @@ -1,248 +0,0 @@ -# Docker Setup for Sample Data Generator - -## Quick Answer - -**Yes!** The `abada.generate-sample-data=true` property works in Docker. You just need to pass it as an environment variable. - -## How to Enable in Docker - -### Option 1: Environment Variable in docker-compose.yml - -Add to the `abada-engine` service environment section: - -```yaml -services: - abada-engine: - environment: - - ABADA_GENERATE_SAMPLE_DATA=true -``` - -### Option 2: Using .env File - -Create or edit `.env` file in the project root: - -```bash -# .env -ABADA_GENERATE_SAMPLE_DATA=true -``` - -Then reference it in docker-compose.yml: - -```yaml -services: - abada-engine: - environment: - - ABADA_GENERATE_SAMPLE_DATA=${ABADA_GENERATE_SAMPLE_DATA:-false} -``` - -### Option 3: Command Line Override - -```bash -# For dev environment -ABADA_GENERATE_SAMPLE_DATA=true docker-compose -f docker-compose.yml -f docker-compose.dev.yml up - -# For prod environment -ABADA_GENERATE_SAMPLE_DATA=true docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -``` - -## Complete Example: docker-compose.dev.yml - -Here's what your `docker-compose.dev.yml` would look like with sample data enabled: - -```yaml -version: '3.8' - -services: - abada-engine: - build: - context: . - dockerfile: Dockerfile - args: - USE_LOCAL_JAR: "true" - image: abada-engine:dev - container_name: abada-engine - environment: - - SPRING_PROFILES_ACTIVE=dev - - SERVER_PORT=5601 - - DB_PASSWORD=${DB_PASSWORD:-abada123} - - MANAGEMENT_TRACING_SAMPLING_PROBABILITY=1.0 - - MANAGEMENT_OTLP_TRACING_ENDPOINT=http://otel-collector:4318/v1/traces - - MANAGEMENT_OTLP_METRICS_ENDPOINT=http://otel-collector:4318/v1/metrics - - OTEL_RESOURCE_ATTRIBUTES_DEPLOYMENT_ENVIRONMENT=dev - - OTEL_SERVICE_NAME=abada-engine-dev - - OTEL_RESOURCE_ATTRIBUTES=project=abada,service.version=0.8.3-alpha - # Enable sample data generation - - ABADA_GENERATE_SAMPLE_DATA=${ABADA_GENERATE_SAMPLE_DATA:-false} - ports: - - "5601:5601" - volumes: - - ./data:/app/data:z - - ./logs:/app/logs:z - networks: - - abada-network - restart: unless-stopped - healthcheck: - test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:5601/abada/api/actuator/health" ] - interval: 30s - timeout: 10s - retries: 3 - depends_on: - otel-collector: - condition: service_started -``` - -## Environment Variable Naming - -Spring Boot automatically converts environment variables: - -- `ABADA_GENERATE_SAMPLE_DATA` → `abada.generate-sample-data` -- Underscores (`_`) become dots (`.`) -- Uppercase becomes lowercase - -## Usage Examples - -### Development with Sample Data - -```bash -# Set environment variable -export ABADA_GENERATE_SAMPLE_DATA=true - -# Start dev stack -./scripts/start-dev.sh - -# Or directly with docker-compose -docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -``` - -### One-Time Sample Data Load - -```bash -# Start with sample data, then stop -ABADA_GENERATE_SAMPLE_DATA=true docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d - -# Wait for data to be generated (check logs) -docker logs -f abada-engine - -# Once you see "Sample Data Generation Complete", you can restart without the flag -docker-compose -f docker-compose.yml -f docker-compose.dev.yml restart abada-engine -``` - -### Production (Disabled by Default) - -```yaml -# docker-compose.prod.yml -services: - abada-engine: - environment: - # Explicitly disable in production - - ABADA_GENERATE_SAMPLE_DATA=false -``` - -## Verification in Docker - -After starting the container with sample data enabled: - -```bash -# Check logs to see sample data generation -docker logs abada-engine | grep "Sample Data" - -# Expected output: -# Starting Sample Data Generation -# Deployed recipe-cook.bpmn -# Deployed parallel-gateway-test.bpmn -# Scenario 1: Completed Recipe Process -# ... -# Sample Data Generation Complete - -# Verify via API -curl http://localhost:5601/abada/api/v1/processes/instances - -# Check tasks -curl -H "X-User: alice" -H "X-Groups: customers" \ - http://localhost:5601/abada/api/v1/tasks -``` - -## Important Notes - -### Data Persistence - -- **H2 (Dev)**: Data persists in `./data` volume -- **PostgreSQL (Prod)**: Data persists in database -- Sample data is generated **once** on startup -- Restarting the container won't regenerate data (unless you clear the database) - -### When to Use - -✅ **Good for:** - -- Demo environments -- Development testing -- UI development -- Integration testing - -❌ **Avoid for:** - -- Production environments -- Performance testing (use dedicated test data) -- Load testing - -### Troubleshooting - -**Sample data not appearing:** - -```bash -# Check if environment variable is set -docker exec abada-engine env | grep ABADA - -# Check application logs -docker logs abada-engine | grep -i "sample" - -# Verify the property is recognized -docker exec abada-engine cat /app/application.properties -``` - -**BPMN files not found:** - -```bash -# Verify BPMN files are in the image -docker exec abada-engine ls -la /app/BOOT-INF/classes/bpmn/ - -# Check test resources are included -docker exec abada-engine find /app -name "*.bpmn" -``` - -## Shell Script for Docker - -Create `scripts/start-dev-with-sample-data.sh`: - -```bash -#!/bin/bash -set -e - -echo "Starting Abada Engine (Dev) with Sample Data..." - -export ABADA_GENERATE_SAMPLE_DATA=true - -./scripts/start-dev.sh - -echo "" -echo "Sample data will be generated on first startup." -echo "Check logs: docker logs -f abada-engine" -``` - -Make it executable: - -```bash -chmod +x scripts/start-dev-with-sample-data.sh -``` - -## Summary - -The sample data generator works seamlessly in Docker by: - -1. Adding `ABADA_GENERATE_SAMPLE_DATA=true` to environment variables -2. Spring Boot automatically converts it to `abada.generate-sample-data=true` -3. The generator runs on container startup -4. Data persists in the configured database/volume - -For the easiest setup, add the environment variable to your `.env` file or docker-compose configuration! diff --git a/scripts/dev-build.sh b/scripts/dev-build.sh index 6d8e6c1..380b3b2 100755 --- a/scripts/dev-build.sh +++ b/scripts/dev-build.sh @@ -15,10 +15,15 @@ echo -e "${BLUE}=== Abada Engine Dev Build ===${NC}\n" echo -e "${YELLOW}Step 1: Building JAR locally...${NC}" ./mvnw clean package spring-boot:repackage -DskipTests -echo -e "\n${YELLOW}Step 2: Building and starting Docker containers...${NC}" +echo -e "\n${YELLOW}Step 2: Cleaning up existing environment...${NC}" +docker stop abada-engine 2>/dev/null || true +docker rm abada-engine 2>/dev/null || true +docker rmi abada-engine:dev 2>/dev/null || true + +echo -e "\n${YELLOW}Step 3: Building and starting Docker containers...${NC}" docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build abada-engine -echo -e "\n${YELLOW}Step 3: Waiting for application to start...${NC}" +echo -e "\n${YELLOW}Step 4: Waiting for application to start...${NC}" sleep 10 echo -e "\n${GREEN}✓ Build complete!${NC}" diff --git a/src/main/java/com/abada/engine/api/ExternalTaskController.java b/src/main/java/com/abada/engine/api/ExternalTaskController.java index b3963fb..c16b081 100644 --- a/src/main/java/com/abada/engine/api/ExternalTaskController.java +++ b/src/main/java/com/abada/engine/api/ExternalTaskController.java @@ -1,6 +1,7 @@ package com.abada.engine.api; import com.abada.engine.core.AbadaEngine; +import com.abada.engine.dto.ExternalTaskFailureDto; import com.abada.engine.dto.FetchAndLockRequest; import com.abada.engine.dto.LockedExternalTask; import com.abada.engine.persistence.entity.ExternalTaskEntity; @@ -17,7 +18,8 @@ /** * REST controller for external task workers. - * This API provides the necessary endpoints for external workers to fetch, lock, and complete jobs, + * This API provides the necessary endpoints for external workers to fetch, + * lock, and complete jobs, * enabling the "External Task Worker" pattern. */ @RestController @@ -35,11 +37,14 @@ public ExternalTaskController(ExternalTaskRepository externalTaskRepository, Aba /** * Fetches and locks available external tasks for a given set of topics. * This endpoint is designed to be polled by external task workers. - * The method is transactional and will attempt to find and lock one available task. + * The method is transactional and will attempt to find and lock one available + * task. * - * @param request The request body containing the worker ID, a list of topics the worker can handle, + * @param request The request body containing the worker ID, a list of topics + * the worker can handle, * and the desired lock duration in milliseconds. - * @return A ResponseEntity containing a list of locked tasks. The list will contain at most one task, + * @return A ResponseEntity containing a list of locked tasks. The list will + * contain at most one task, * or be empty if no tasks are available for the given topics. */ @PostMapping("/fetch-and-lock") @@ -49,7 +54,8 @@ public ResponseEntity> fetchAndLock(@RequestBody FetchA for (String topic : request.topics()) { Optional taskOptional = externalTaskRepository - .findFirstByTopicNameAndStatusOrTopicNameAndLockExpirationTimeLessThan(topic, ExternalTaskEntity.Status.OPEN, topic, Instant.now()); + .findFirstByTopicNameAndStatusOrTopicNameAndLockExpirationTimeLessThan(topic, + ExternalTaskEntity.Status.OPEN, topic, Instant.now()); if (taskOptional.isPresent()) { ExternalTaskEntity task = taskOptional.get(); @@ -58,9 +64,10 @@ public ResponseEntity> fetchAndLock(@RequestBody FetchA task.setLockExpirationTime(Instant.now().plusMillis(request.lockDuration())); externalTaskRepository.save(task); - Map variables = abadaEngine.getProcessInstanceById(task.getProcessInstanceId()).getVariables(); + Map variables = abadaEngine.getProcessInstanceById(task.getProcessInstanceId()) + .getVariables(); lockedTasks.add(new LockedExternalTask(task.getId(), task.getTopicName(), variables)); - + return ResponseEntity.ok(lockedTasks); } } @@ -70,12 +77,16 @@ public ResponseEntity> fetchAndLock(@RequestBody FetchA /** * Completes an external task and resumes the corresponding process instance. - * This endpoint is called by a worker after it has successfully finished its business logic. + * This endpoint is called by a worker after it has successfully finished its + * business logic. * - * @param id The unique ID of the external task to complete. This is the ID that was returned - * in the `LockedExternalTask` payload from the `fetch-and-lock` endpoint. - * It is **not** the topic name. - * @param variables A map of variables to pass back to the process instance. These variables will be + * @param id The unique ID of the external task to complete. This is the + * ID that was returned + * in the `LockedExternalTask` payload from the + * `fetch-and-lock` endpoint. + * It is **not** the topic name. + * @param variables A map of variables to pass back to the process instance. + * These variables will be * merged into the process scope before the engine advances. * @return An HTTP 200 OK response on successful completion. */ @@ -92,4 +103,43 @@ public ResponseEntity complete(@PathVariable String id, @RequestBody Map handleFailure(@PathVariable String id, @RequestBody ExternalTaskFailureDto failureDto) { + ExternalTaskEntity task = externalTaskRepository.findById(id) + .orElseThrow(() -> new RuntimeException("External task not found: " + id)); + + if (task.getWorkerId() != null && !task.getWorkerId().equals(failureDto.workerId())) { + // In a real scenario, we might want to reject this, but for now we'll allow it + // or just log a warning + // throw new RuntimeException("Worker ID mismatch"); + } + + task.setExceptionMessage(failureDto.errorMessage()); + task.setExceptionStacktrace(failureDto.errorDetails()); + task.setRetries(failureDto.retries()); + + if (failureDto.retries() != null && failureDto.retries() == 0) { + task.setStatus(ExternalTaskEntity.Status.FAILED); + task.setLockExpirationTime(null); + task.setWorkerId(null); + } else { + task.setStatus(ExternalTaskEntity.Status.OPEN); + task.setLockExpirationTime(null); // Release lock so it can be picked up again immediately or after timeout + task.setWorkerId(null); + // TODO: Implement retry timeout (backoff) + } + + externalTaskRepository.save(task); + + return ResponseEntity.ok().build(); + } } diff --git a/src/main/java/com/abada/engine/dto/ExternalTaskFailureDto.java b/src/main/java/com/abada/engine/dto/ExternalTaskFailureDto.java new file mode 100644 index 0000000..c507189 --- /dev/null +++ b/src/main/java/com/abada/engine/dto/ExternalTaskFailureDto.java @@ -0,0 +1,9 @@ +package com.abada.engine.dto; + +public record ExternalTaskFailureDto( + String workerId, + String errorMessage, + String errorDetails, + Integer retries, + Long retryTimeout) { +} diff --git a/src/test/java/com/abada/engine/api/ExternalTaskTest.java b/src/test/java/com/abada/engine/api/ExternalTaskTest.java index ba8a3d8..da52432 100644 --- a/src/test/java/com/abada/engine/api/ExternalTaskTest.java +++ b/src/test/java/com/abada/engine/api/ExternalTaskTest.java @@ -3,8 +3,11 @@ import com.abada.engine.AbadaEngineApplication; import com.abada.engine.core.AbadaEngine; import com.abada.engine.core.ProcessInstance; +import com.abada.engine.dto.ExternalTaskFailureDto; +import com.abada.engine.dto.FailedJobDTO; import com.abada.engine.dto.FetchAndLockRequest; import com.abada.engine.dto.LockedExternalTask; +import com.abada.engine.persistence.entity.ExternalTaskEntity; import com.abada.engine.persistence.repository.ExternalTaskRepository; import com.abada.engine.util.BpmnTestUtils; import org.junit.jupiter.api.BeforeEach; @@ -57,7 +60,8 @@ void shouldCreateAndCompleteExternalTaskViaApi() { // 1. Start the process ProcessInstance pi = abadaEngine.startProcess("ExternalTaskTestProcess"); - // 2. Assert that the process is waiting at the service task and a job was created + // 2. Assert that the process is waiting at the service task and a job was + // created assertEquals("ServiceTask_External", pi.getActiveTokens().get(0)); assertEquals(1, externalTaskRepository.count()); @@ -68,8 +72,8 @@ void shouldCreateAndCompleteExternalTaskViaApi() { "/v1/external-tasks/fetch-and-lock", HttpMethod.POST, requestEntity, - new ParameterizedTypeReference<>() {} - ); + new ParameterizedTypeReference<>() { + }); assertEquals(HttpStatus.OK, lockResponse.getStatusCode()); List lockedTasks = lockResponse.getBody(); @@ -81,7 +85,8 @@ void shouldCreateAndCompleteExternalTaskViaApi() { // 4. Simulate the worker completing the task Map outputVariables = Map.of("externalTaskResult", "SUCCESS"); HttpEntity> completeRequestEntity = new HttpEntity<>(outputVariables, headers); - restTemplate.postForEntity("/v1/external-tasks/{id}/complete", completeRequestEntity, Void.class, lockedTask.id()); + restTemplate.postForEntity("/v1/external-tasks/{id}/complete", completeRequestEntity, Void.class, + lockedTask.id()); // 5. Assert that the process has resumed and moved to the final user task ProcessInstance resumedPi = abadaEngine.getProcessInstanceById(pi.getId()); @@ -91,4 +96,53 @@ void shouldCreateAndCompleteExternalTaskViaApi() { // 6. Assert that the external task job has been deleted assertEquals(0, externalTaskRepository.count()); } + + @Test + @DisplayName("External worker should report failure and job should appear in failed jobs list") + void shouldHandleExternalTaskFailure() { + // 1. Start the process + ProcessInstance pi = abadaEngine.startProcess("ExternalTaskTestProcess"); + + // 2. Fetch and lock + FetchAndLockRequest fetchRequest = new FetchAndLockRequest("worker-1", List.of("test-topic"), 10000L); + HttpEntity requestEntity = new HttpEntity<>(fetchRequest, headers); + ResponseEntity> lockResponse = restTemplate.exchange( + "/v1/external-tasks/fetch-and-lock", + HttpMethod.POST, + requestEntity, + new ParameterizedTypeReference<>() { + }); + LockedExternalTask lockedTask = lockResponse.getBody().get(0); + + // 3. Report failure + ExternalTaskFailureDto failureDto = new ExternalTaskFailureDto( + "worker-1", + "Something went wrong", + "Stack trace details...", + 0, + 1000L); + HttpEntity failureRequestEntity = new HttpEntity<>(failureDto, headers); + restTemplate.postForEntity("/v1/external-tasks/{id}/failure", failureRequestEntity, Void.class, + lockedTask.id()); + + // 4. Verify task is FAILED in DB + ExternalTaskEntity task = externalTaskRepository.findById(lockedTask.id()).orElseThrow(); + assertEquals(ExternalTaskEntity.Status.FAILED, task.getStatus()); + assertEquals("Something went wrong", task.getExceptionMessage()); + assertEquals(0, task.getRetries()); + + // 5. Verify it appears in failed jobs list + ResponseEntity> failedJobsResponse = restTemplate.exchange( + "/v1/jobs", + HttpMethod.GET, + new HttpEntity<>(headers), + new ParameterizedTypeReference<>() { + }); + + List failedJobs = failedJobsResponse.getBody(); + assertNotNull(failedJobs); + assertEquals(1, failedJobs.size()); + assertEquals(lockedTask.id(), failedJobs.get(0).jobId()); + assertEquals("Something went wrong", failedJobs.get(0).exceptionMessage()); + } }