diff --git a/.github/workflows/dts-e2e-tests.yaml b/.github/workflows/dts-e2e-tests.yaml new file mode 100644 index 0000000..cfcab6d --- /dev/null +++ b/.github/workflows/dts-e2e-tests.yaml @@ -0,0 +1,56 @@ +name: ๐Ÿงช DTS Emulator E2E Tests + +# This workflow runs E2E tests against the Durable Task Scheduler (DTS) emulator. +# It mirrors the Python testing setup at durabletask-python for Azure-managed tests. + +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + dts-e2e-tests: + strategy: + fail-fast: false + matrix: + node-version: ["18.x", "20.x", "22.x"] + env: + EMULATOR_VERSION: "latest" + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 + + - name: ๐Ÿณ Pull Docker image + run: docker pull mcr.microsoft.com/dts/dts-emulator:$EMULATOR_VERSION + + - name: ๐Ÿš€ Run Docker container + run: | + docker run --name dtsemulator -d -p 8080:8080 mcr.microsoft.com/dts/dts-emulator:$EMULATOR_VERSION + + - name: โณ Wait for container to be ready + run: sleep 10 # Adjust if your service needs more time to start + + - name: ๐Ÿ”ง Set environment variables + run: | + echo "TASKHUB=default" >> $GITHUB_ENV + echo "ENDPOINT=localhost:8080" >> $GITHUB_ENV + + - name: โš™๏ธ NodeJS - Install + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + registry-url: "https://registry.npmjs.org" + + - name: โš™๏ธ Install dependencies + run: npm install + + - name: โœ… Run E2E tests against DTS emulator + run: npm run test:e2e:azuremanaged:internal diff --git a/.github/workflows/pr-validation.yaml b/.github/workflows/pr-validation.yaml index 7cd88d2..ec92ad1 100644 --- a/.github/workflows/pr-validation.yaml +++ b/.github/workflows/pr-validation.yaml @@ -8,29 +8,22 @@ on: branches: - main +permissions: + contents: read + jobs: - build: + lint-and-unit-tests: runs-on: ubuntu-latest env: - NODE_VER: 16.14.0 - - services: - # docker run --name durabletask-sidecar -p 4001:4001 --env 'DURABLETASK_SIDECAR_LOGLEVEL=Debug' --rm cgillum/durabletask-sidecar:latest start --backend Emulator - durabletask-sidecar: - image: cgillum/durabletask-sidecar:latest - ports: - - 4001:4001 - env: - DURABLETASK_SIDECAR_LOGLEVEL: Debug - DURABLETASK_STORAGE_PROVIDER: Emulator + NODE_VER: 18.x steps: - name: ๐Ÿ“ฅ Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: โš™๏ธ NodeJS - Install - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VER }} registry-url: "https://registry.npmjs.org" @@ -38,8 +31,43 @@ jobs: - name: โš™๏ธ Install dependencies run: npm install + - name: ๐Ÿ” Run linting + run: npm run lint + - name: โœ… Run unit tests - run: npm test test/unit + run: npm run test:unit + + e2e-tests: + strategy: + fail-fast: false + matrix: + node-version: ["18.x", "20.x", "22.x"] + needs: lint-and-unit-tests + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 + + - name: โš™๏ธ NodeJS - Install + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + registry-url: "https://registry.npmjs.org" + + - name: โš™๏ธ Install dependencies + run: npm install + + # Install Go SDK for durabletask-go sidecar + - name: ๐Ÿ”ง Install Go SDK + uses: actions/setup-go@v5 + with: + go-version: "stable" - - name: โœ… Run e2e tests - run: ./scripts/test-e2e.sh + # Install and run the durabletask-go sidecar for running e2e tests + - name: โœ… Run E2E tests with durabletask-go sidecar + run: | + go install github.com/microsoft/durabletask-go@main + durabletask-go --port 4001 & + sleep 5 # Wait for sidecar to be ready + npm run test:e2e:internal diff --git a/package.json b/package.json index 82049c0..fbbbf10 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "test:e2e:internal": "jest test/e2e --runInBand --detectOpenHandles", "test:e2e": "./scripts/test-e2e.sh", "test:e2e:one": "jest test/e2e --runInBand --detectOpenHandles --testNamePattern", + "test:e2e:azuremanaged:internal": "jest test/e2e-azuremanaged --runInBand --detectOpenHandles", + "test:e2e:azuremanaged": "./scripts/test-e2e-azuremanaged.sh", "start": "ts-node --swc ./src/index.ts", "example": "ts-node --swc", "lint": "eslint . --ext .js,.jsx,.ts,.tsx", diff --git a/scripts/test-e2e-azuremanaged.sh b/scripts/test-e2e-azuremanaged.sh new file mode 100755 index 0000000..971ec8f --- /dev/null +++ b/scripts/test-e2e-azuremanaged.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Script to run E2E tests against the DTS (Durable Task Scheduler) emulator. +# +# This script mirrors the Python testing setup at durabletask-python for Azure-managed tests. +# It expects the DTS emulator to be running at the specified endpoint. +# +# Environment variables: +# - ENDPOINT: The endpoint for the DTS emulator (default: localhost:8080) +# - TASKHUB: The task hub name (default: default) + +ENDPOINT="${ENDPOINT:-localhost:8080}" +TASKHUB="${TASKHUB:-default}" + +# Start the DTS emulator if it is not running yet +if [ ! "$(docker ps -q -f name=dts-emulator)" ]; then + if [ "$(docker ps -aq -f status=exited -f name=dts-emulator)" ]; then + # cleanup + docker rm dts-emulator + fi + + # run your container + echo "Starting DTS emulator" + docker run \ + --name dts-emulator -d --rm \ + -p 8080:8080 \ + mcr.microsoft.com/dts/dts-emulator:latest + + # Wait for container to be ready + echo "Waiting for DTS emulator to be ready..." + sleep 10 +fi + +echo "Running E2E tests against DTS emulator" +echo "Endpoint: $ENDPOINT" +echo "TaskHub: $TASKHUB" + +ENDPOINT="$ENDPOINT" TASKHUB="$TASKHUB" npm run test:e2e:azuremanaged:internal + +# It should fail if the npm run fails +if [ $? -ne 0 ]; then + echo "E2E tests failed" + exit 1 +fi + +echo "Stopping DTS emulator" +docker stop dts-emulator diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 3ba92bf..30345f1 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,4 +1,13 @@ #!/bin/bash +# Script to run E2E tests against the durabletask-sidecar. +# +# This script uses the cgillum/durabletask-sidecar Docker container for local testing. +# In CI/CD, we use durabletask-go sidecar instead (similar to Python SDK testing approach). +# +# NOTE: To run tests similar to the Python SDK setup: +# go install github.com/microsoft/durabletask-go@main +# durabletask-go --port 4001 + # Start the sidecar if it is not running yet if [ ! "$(docker ps -q -f name=durabletask-sidecar)" ]; then if [ "$(docker ps -aq -f status=exited -f name=durabletask-sidecar)" ]; then diff --git a/test/e2e-azuremanaged/orchestration.spec.ts b/test/e2e-azuremanaged/orchestration.spec.ts new file mode 100644 index 0000000..8eedbdc --- /dev/null +++ b/test/e2e-azuremanaged/orchestration.spec.ts @@ -0,0 +1,272 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * E2E tests for Durable Task Scheduler (DTS) emulator. + * + * NOTE: These tests assume the DTS emulator is running. Example command: + * docker run -i -p 8080:8080 -d mcr.microsoft.com/dts/dts-emulator:latest + * + * Environment variables: + * - ENDPOINT: The endpoint for the DTS emulator (default: localhost:8080) + * - TASKHUB: The task hub name (default: default) + */ + +import { TaskHubGrpcClient } from "../../src/client/client"; +import { OrchestrationStatus } from "../../src/proto/orchestrator_service_pb"; +import { getName, whenAll } from "../../src/task"; +import { ActivityContext } from "../../src/task/context/activity-context"; +import { OrchestrationContext } from "../../src/task/context/orchestration-context"; +import { Task } from "../../src/task/task"; +import { TOrchestrator } from "../../src/types/orchestrator.type"; +import { TaskHubGrpcWorker } from "../../src/worker/task-hub-grpc-worker"; + +// Read environment variables +const endpoint = process.env.ENDPOINT || "localhost:8080"; +const _taskHub = process.env.TASKHUB || "default"; + +describe("Durable Task Scheduler (DTS) E2E Tests", () => { + let taskHubClient: TaskHubGrpcClient; + let taskHubWorker: TaskHubGrpcWorker; + + beforeEach(async () => { + // Start a worker, which will connect to the DTS emulator in a background thread + taskHubWorker = new TaskHubGrpcWorker(endpoint); + taskHubClient = new TaskHubGrpcClient(endpoint); + }); + + afterEach(async () => { + await taskHubWorker.stop(); + await taskHubClient.stop(); + }); + + it("should be able to run an empty orchestration", async () => { + let invoked = false; + + const emptyOrchestrator: TOrchestrator = async (_: OrchestrationContext) => { + invoked = true; + }; + + taskHubWorker.addOrchestrator(emptyOrchestrator); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(emptyOrchestrator); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + + expect(invoked).toBe(true); + expect(state).toBeDefined(); + expect(state?.name).toEqual(getName(emptyOrchestrator)); + expect(state?.instanceId).toEqual(id); + expect(state?.failureDetails).toBeUndefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + }); + + it("should be able to run an activity sequence", async () => { + const plusOne = async (_: ActivityContext, input: number) => { + return input + 1; + }; + + const sequence: TOrchestrator = async function* (ctx: OrchestrationContext, startVal: number): any { + const numbers = [startVal]; + let current = startVal; + + for (let i = 0; i < 10; i++) { + current = yield ctx.callActivity(plusOne, current); + numbers.push(current); + } + + return numbers; + }; + + taskHubWorker.addOrchestrator(sequence); + taskHubWorker.addActivity(plusOne); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(sequence, 1); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + + expect(state).toBeDefined(); + expect(state?.name).toEqual(getName(sequence)); + expect(state?.instanceId).toEqual(id); + expect(state?.failureDetails).toBeUndefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.serializedInput).toEqual(JSON.stringify(1)); + expect(state?.serializedOutput).toEqual(JSON.stringify([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])); + }, 31000); + + it("should be able to run fan-out/fan-in", async () => { + let activityCounter = 0; + + const increment = (_: ActivityContext) => { + activityCounter++; + }; + + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, count: number): any { + // Fan out to multiple activities + const tasks: Task[] = []; + + for (let i = 0; i < count; i++) { + tasks.push(ctx.callActivity(increment)); + } + + // Wait for all the activities to complete + yield whenAll(tasks); + }; + + taskHubWorker.addActivity(increment); + taskHubWorker.addOrchestrator(orchestrator); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(orchestrator, 10); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.failureDetails).toBeUndefined(); + expect(activityCounter).toEqual(10); + }, 31000); + + it("should be able to use the sub-orchestration", async () => { + let activityCounter = 0; + + const increment = (_: ActivityContext) => { + activityCounter++; + }; + + const orchestratorChild: TOrchestrator = async function* (ctx: OrchestrationContext): any { + yield ctx.callActivity(increment); + }; + + const orchestratorParent: TOrchestrator = async function* (ctx: OrchestrationContext): any { + // Call sub-orchestration + yield ctx.callSubOrchestrator(orchestratorChild); + }; + + taskHubWorker.addActivity(increment); + taskHubWorker.addOrchestrator(orchestratorChild); + taskHubWorker.addOrchestrator(orchestratorParent); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(orchestratorParent, 10); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.failureDetails).toBeUndefined(); + expect(activityCounter).toEqual(1); + }, 31000); + + it("should allow waiting for multiple external events", async () => { + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, _: any): any { + const a = yield ctx.waitForExternalEvent("A"); + const b = yield ctx.waitForExternalEvent("B"); + const c = yield ctx.waitForExternalEvent("C"); + return [a, b, c]; + }; + + taskHubWorker.addOrchestrator(orchestrator); + await taskHubWorker.start(); + + // Send events to the client immediately + const id = await taskHubClient.scheduleNewOrchestration(orchestrator); + taskHubClient.raiseOrchestrationEvent(id, "A", "a"); + taskHubClient.raiseOrchestrationEvent(id, "B", "b"); + taskHubClient.raiseOrchestrationEvent(id, "C", "c"); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify(["a", "b", "c"])); + }); + + it("should be able to run a single timer", async () => { + const delay = 3; + const singleTimer: TOrchestrator = async function* (ctx: OrchestrationContext): any { + yield ctx.createTimer(delay); + }; + + taskHubWorker.addOrchestrator(singleTimer); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(singleTimer); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + + let expectedCompletionSecond = state?.createdAt?.getTime() ?? 0; + if (state && state.createdAt !== undefined) { + expectedCompletionSecond += delay * 1000; + } + expect(expectedCompletionSecond).toBeDefined(); + const actualCompletionSecond = state?.lastUpdatedAt?.getTime() ?? 0; + expect(actualCompletionSecond).toBeDefined(); + + expect(state).toBeDefined(); + expect(state?.name).toEqual(getName(singleTimer)); + expect(state?.instanceId).toEqual(id); + expect(state?.failureDetails).toBeUndefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.createdAt).toBeDefined(); + expect(state?.lastUpdatedAt).toBeDefined(); + expect(expectedCompletionSecond).toBeLessThanOrEqual(actualCompletionSecond); + }, 31000); + + it("should be able to terminate an orchestration", async () => { + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, _: any): any { + const res = yield ctx.waitForExternalEvent("my_event"); + return res; + }; + + taskHubWorker.addOrchestrator(orchestrator); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(orchestrator); + let state = await taskHubClient.waitForOrchestrationStart(id, undefined, 30); + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING); + + await taskHubClient.terminateOrchestration(id, "some reason for termination"); + state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_TERMINATED); + expect(state?.serializedOutput).toEqual(JSON.stringify("some reason for termination")); + }, 31000); + + it("should allow to continue as new", async () => { + const orchestrator: TOrchestrator = async (ctx: OrchestrationContext, input: number) => { + if (input < 10) { + ctx.continueAsNew(input + 1, true); + } else { + return input; + } + }; + + taskHubWorker.addOrchestrator(orchestrator); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(orchestrator, 1); + + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify(10)); + }, 31000); + + it("should be able to run a single orchestration without activity", async () => { + const orchestrator: TOrchestrator = async (ctx: OrchestrationContext, startVal: number) => { + return startVal + 1; + }; + + taskHubWorker.addOrchestrator(orchestrator); + await taskHubWorker.start(); + + const id = await taskHubClient.scheduleNewOrchestration(orchestrator, 15); + const state = await taskHubClient.waitForOrchestrationCompletion(id, undefined, 30); + + expect(state).toBeDefined(); + expect(state?.name).toEqual(getName(orchestrator)); + expect(state?.instanceId).toEqual(id); + expect(state?.failureDetails).toBeUndefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED); + expect(state?.serializedInput).toEqual(JSON.stringify(15)); + expect(state?.serializedOutput).toEqual(JSON.stringify(16)); + }, 31000); +}); diff --git a/test/e2e/orchestration.spec.ts b/test/e2e/orchestration.spec.ts index 03daf47..1f3e937 100644 --- a/test/e2e/orchestration.spec.ts +++ b/test/e2e/orchestration.spec.ts @@ -1,6 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +/** + * E2E tests for Durable Task using durabletask-go sidecar. + * + * NOTE: These tests assume a sidecar process is running. Example command: + * go install github.com/microsoft/durabletask-go@main + * durabletask-go --port 4001 + * + * Alternatively, you can use the Docker sidecar: + * docker run --name durabletask-sidecar -p 4001:4001 \ + * --env 'DURABLETASK_SIDECAR_LOGLEVEL=Debug' \ + * cgillum/durabletask-sidecar:latest start --backend Emulator + */ + import { TaskHubGrpcClient } from "../../src/client/client"; import { PurgeInstanceCriteria } from "../../src/orchestration/orchestration-purge-criteria"; import { OrchestrationStatus } from "../../src/proto/orchestrator_service_pb";