diff --git a/.github/workflows/codeQL.yml b/.github/workflows/codeQL.yml index 4b5a168..7edfa7c 100644 --- a/.github/workflows/codeQL.yml +++ b/.github/workflows/codeQL.yml @@ -46,8 +46,6 @@ jobs: # queries: security-extended,security-and-quality - uses: actions/checkout@v3 - with: - submodules: true # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index b371516..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "submodules/durabletask-protobuf"] - path = submodules/durabletask-protobuf - url = https://github.com/microsoft/durabletask-protobuf diff --git a/.prettierignore b/.prettierignore index dabb131..bbb10ce 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,6 +6,4 @@ dist # don't lint nyc coverage output coverage # don't lint proto files and output -proto -# don't lint submodules -submodules \ No newline at end of file +proto \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ffc881..1ba9710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## Upcoming + +### New + +- Add automatic proto file download and commit hash tracking during build ([#XX](https://github.com/microsoft/durabletask-js/pull/XX)) + ## v0.1.0-alpha.2 ### New diff --git a/README.md b/README.md index 06b2c9f..2786936 100644 --- a/README.md +++ b/README.md @@ -145,30 +145,22 @@ See the [examples](./examples) directory for a list of sample orchestrations and ### Generating protobufs -Protobuf definitions are stored in the [./submodules/durabletask-proto](./submodules/durabletask-proto) directory, which is a submodule. To update the submodule, run the following command from the project root: +Protobuf definitions are downloaded from the [durabletask-protobuf](https://github.com/microsoft/durabletask-protobuf) repository. To download the latest proto files, run: ```sh -git submodule update --init +npm run download-proto ``` -Once the submodule is available, the corresponding source code can be regenerated using the following command from the project root: +This will download the proto files to `internal/durabletask-protobuf/protos/`. + +Once the proto files are available, the corresponding TypeScript source code can be regenerated using the following command from the project root: ```sh -npm install grpc_tools_node_protoc_ts --save-dev - -# generate js codes via grpc-tools -grpc_tools_node_protoc \ ---js_out=import_style=commonjs,binary:src/proto \ ---grpc_out=grpc_js:src/proto \ ---plugin=protoc-gen-grpc=`which grpc_tools_node_protoc_plugin` \ --I ./submodules/durabletask-protobuf/protos orchestrator_service.proto - -protoc \ ---plugin=protoc-gen-ts=./node_modules/.bin/protoc-gen-ts \ ---ts_out=grpc_js:src/proto \ --I ./submodules/durabletask-protobuf/protos orchestrator_service.proto +./tools/generate-grpc-javascript.sh ./src/proto ``` +Note: You need `grpc-tools` installed globally (`npm install -g grpc-tools`). + ### Running unit tests Unit tests can be run using the following command from the project root. Unit tests _don't_ require a sidecar process to be running. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e7f5f4c..04e44f5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,6 @@ pool: steps: - checkout: self - submodules: true - task: NodeTool@0 inputs: diff --git a/eng/templates/build.yml b/eng/templates/build.yml index def53e4..88ca92a 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -11,7 +11,6 @@ jobs: sbomBuildComponentPath: "$(Build.SourcesDirectory)/node_modules" steps: - checkout: self - submodules: true - task: NodeTool@0 inputs: versionSpec: 20.x diff --git a/internal/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH b/internal/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH new file mode 100644 index 0000000..7971873 --- /dev/null +++ b/internal/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH @@ -0,0 +1 @@ +026329c53fe6363985655857b9ca848ec7238bd2 diff --git a/internal/durabletask-protobuf/README.md b/internal/durabletask-protobuf/README.md new file mode 100644 index 0000000..ac6e1bf --- /dev/null +++ b/internal/durabletask-protobuf/README.md @@ -0,0 +1,25 @@ +# Durable Task Protobuf Files + +This directory contains the protocol buffer definitions used by the Durable Task Framework JavaScript SDK. The files in this directory are automatically downloaded and updated during the build process from the [microsoft/durabletask-protobuf](https://github.com/microsoft/durabletask-protobuf) repository. + +## Directory Structure + +- `protos/` - Contains the proto files +- `PROTO_SOURCE_COMMIT_HASH` - Contains the commit hash of the proto file version + +## Auto-Update Process + +The proto files are automatically downloaded and updated when running `npm run build`. This is handled by the `scripts/download-proto.sh` script. The script: + +1. Downloads the latest version of `orchestrator_service.proto` +2. Saves the current commit hash for tracking purposes + +## Manual Update + +To manually update the proto files, you can run: + +```bash +npm run download-proto +# or +./scripts/download-proto.sh [branch-name] +``` diff --git a/internal/durabletask-protobuf/protos/orchestrator_service.proto b/internal/durabletask-protobuf/protos/orchestrator_service.proto new file mode 100644 index 0000000..8ef46a4 --- /dev/null +++ b/internal/durabletask-protobuf/protos/orchestrator_service.proto @@ -0,0 +1,881 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +syntax = "proto3"; + +option csharp_namespace = "Microsoft.DurableTask.Protobuf"; +option java_package = "com.microsoft.durabletask.implementation.protobuf"; +option go_package = "/internal/protos"; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +message OrchestrationInstance { + string instanceId = 1; + google.protobuf.StringValue executionId = 2; +} + +message ActivityRequest { + string name = 1; + google.protobuf.StringValue version = 2; + google.protobuf.StringValue input = 3; + OrchestrationInstance orchestrationInstance = 4; + int32 taskId = 5; + TraceContext parentTraceContext = 6; +} + +message ActivityResponse { + string instanceId = 1; + int32 taskId = 2; + google.protobuf.StringValue result = 3; + TaskFailureDetails failureDetails = 4; + string completionToken = 5; +} + +message TaskFailureDetails { + string errorType = 1; + string errorMessage = 2; + google.protobuf.StringValue stackTrace = 3; + TaskFailureDetails innerFailure = 4; + bool isNonRetriable = 5; + map properties = 6; +} + +enum OrchestrationStatus { + ORCHESTRATION_STATUS_RUNNING = 0; + ORCHESTRATION_STATUS_COMPLETED = 1; + ORCHESTRATION_STATUS_CONTINUED_AS_NEW = 2; + ORCHESTRATION_STATUS_FAILED = 3; + ORCHESTRATION_STATUS_CANCELED = 4; + ORCHESTRATION_STATUS_TERMINATED = 5; + ORCHESTRATION_STATUS_PENDING = 6; + ORCHESTRATION_STATUS_SUSPENDED = 7; +} + +message ParentInstanceInfo { + int32 taskScheduledId = 1; + google.protobuf.StringValue name = 2; + google.protobuf.StringValue version = 3; + OrchestrationInstance orchestrationInstance = 4; +} + +message TraceContext { + string traceParent = 1; + string spanID = 2 [deprecated=true]; + google.protobuf.StringValue traceState = 3; +} + +message ExecutionStartedEvent { + string name = 1; + google.protobuf.StringValue version = 2; + google.protobuf.StringValue input = 3; + OrchestrationInstance orchestrationInstance = 4; + ParentInstanceInfo parentInstance = 5; + google.protobuf.Timestamp scheduledStartTimestamp = 6; + TraceContext parentTraceContext = 7; + google.protobuf.StringValue orchestrationSpanID = 8; + map tags = 9; +} + +message ExecutionCompletedEvent { + OrchestrationStatus orchestrationStatus = 1; + google.protobuf.StringValue result = 2; + TaskFailureDetails failureDetails = 3; +} + +message ExecutionTerminatedEvent { + google.protobuf.StringValue input = 1; + bool recurse = 2; +} + +message TaskScheduledEvent { + string name = 1; + google.protobuf.StringValue version = 2; + google.protobuf.StringValue input = 3; + TraceContext parentTraceContext = 4; + map tags = 5; +} + +message TaskCompletedEvent { + int32 taskScheduledId = 1; + google.protobuf.StringValue result = 2; +} + +message TaskFailedEvent { + int32 taskScheduledId = 1; + TaskFailureDetails failureDetails = 2; +} + +message SubOrchestrationInstanceCreatedEvent { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue version = 3; + google.protobuf.StringValue input = 4; + TraceContext parentTraceContext = 5; + map tags = 6; +} + +message SubOrchestrationInstanceCompletedEvent { + int32 taskScheduledId = 1; + google.protobuf.StringValue result = 2; +} + +message SubOrchestrationInstanceFailedEvent { + int32 taskScheduledId = 1; + TaskFailureDetails failureDetails = 2; +} + +message TimerCreatedEvent { + google.protobuf.Timestamp fireAt = 1; +} + +message TimerFiredEvent { + google.protobuf.Timestamp fireAt = 1; + int32 timerId = 2; +} + +message OrchestratorStartedEvent { + // No payload data +} + +message OrchestratorCompletedEvent { + // No payload data +} + +message EventSentEvent { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue input = 3; +} + +message EventRaisedEvent { + string name = 1; + google.protobuf.StringValue input = 2; +} + +message GenericEvent { + google.protobuf.StringValue data = 1; +} + +message HistoryStateEvent { + OrchestrationState orchestrationState = 1; +} + +message ContinueAsNewEvent { + google.protobuf.StringValue input = 1; +} + +message ExecutionSuspendedEvent { + google.protobuf.StringValue input = 1; +} + +message ExecutionResumedEvent { + google.protobuf.StringValue input = 1; +} + +message EntityOperationSignaledEvent { + string requestId = 1; + string operation = 2; + google.protobuf.Timestamp scheduledTime = 3; + google.protobuf.StringValue input = 4; + google.protobuf.StringValue targetInstanceId = 5; // used only within histories, null in messages +} + +message EntityOperationCalledEvent { + string requestId = 1; + string operation = 2; + google.protobuf.Timestamp scheduledTime = 3; + google.protobuf.StringValue input = 4; + google.protobuf.StringValue parentInstanceId = 5; // used only within messages, null in histories + google.protobuf.StringValue parentExecutionId = 6; // used only within messages, null in histories + google.protobuf.StringValue targetInstanceId = 7; // used only within histories, null in messages +} + +message EntityLockRequestedEvent { + string criticalSectionId = 1; + repeated string lockSet = 2; + int32 position = 3; + google.protobuf.StringValue parentInstanceId = 4; // used only within messages, null in histories +} + +message EntityOperationCompletedEvent { + string requestId = 1; + google.protobuf.StringValue output = 2; +} + +message EntityOperationFailedEvent { + string requestId = 1; + TaskFailureDetails failureDetails = 2; +} + +message EntityUnlockSentEvent { + string criticalSectionId = 1; + google.protobuf.StringValue parentInstanceId = 2; // used only within messages, null in histories + google.protobuf.StringValue targetInstanceId = 3; // used only within histories, null in messages +} + +message EntityLockGrantedEvent { + string criticalSectionId = 1; +} + +message ExecutionRewoundEvent { + google.protobuf.StringValue reason = 1; + google.protobuf.StringValue parentExecutionId = 2; // used only for rewinding suborchestrations, null otherwise + google.protobuf.StringValue instanceId = 3; // used only for rewinding suborchestrations, null otherwise + TraceContext parentTraceContext = 4; // used only for rewinding suborchestrations, null otherwise + google.protobuf.StringValue name = 5; // used by DTS backend only + google.protobuf.StringValue version = 6; // used by DTS backend only + google.protobuf.StringValue input = 7; // used by DTS backend only + ParentInstanceInfo parentInstance = 8; // used by DTS backend only + map tags = 9; // used by DTS backend only +} + +message HistoryEvent { + int32 eventId = 1; + google.protobuf.Timestamp timestamp = 2; + oneof eventType { + ExecutionStartedEvent executionStarted = 3; + ExecutionCompletedEvent executionCompleted = 4; + ExecutionTerminatedEvent executionTerminated = 5; + TaskScheduledEvent taskScheduled = 6; + TaskCompletedEvent taskCompleted = 7; + TaskFailedEvent taskFailed = 8; + SubOrchestrationInstanceCreatedEvent subOrchestrationInstanceCreated = 9; + SubOrchestrationInstanceCompletedEvent subOrchestrationInstanceCompleted = 10; + SubOrchestrationInstanceFailedEvent subOrchestrationInstanceFailed = 11; + TimerCreatedEvent timerCreated = 12; + TimerFiredEvent timerFired = 13; + OrchestratorStartedEvent orchestratorStarted = 14; + OrchestratorCompletedEvent orchestratorCompleted = 15; + EventSentEvent eventSent = 16; + EventRaisedEvent eventRaised = 17; + GenericEvent genericEvent = 18; + HistoryStateEvent historyState = 19; + ContinueAsNewEvent continueAsNew = 20; + ExecutionSuspendedEvent executionSuspended = 21; + ExecutionResumedEvent executionResumed = 22; + EntityOperationSignaledEvent entityOperationSignaled = 23; + EntityOperationCalledEvent entityOperationCalled = 24; + EntityOperationCompletedEvent entityOperationCompleted = 25; + EntityOperationFailedEvent entityOperationFailed = 26; + EntityLockRequestedEvent entityLockRequested = 27; + EntityLockGrantedEvent entityLockGranted = 28; + EntityUnlockSentEvent entityUnlockSent = 29; + ExecutionRewoundEvent executionRewound = 30; + } +} + +message ScheduleTaskAction { + string name = 1; + google.protobuf.StringValue version = 2; + google.protobuf.StringValue input = 3; + map tags = 4; + TraceContext parentTraceContext = 5; +} + +message CreateSubOrchestrationAction { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue version = 3; + google.protobuf.StringValue input = 4; + TraceContext parentTraceContext = 5; + map tags = 6; +} + +message CreateTimerAction { + google.protobuf.Timestamp fireAt = 1; +} + +message SendEventAction { + OrchestrationInstance instance = 1; + string name = 2; + google.protobuf.StringValue data = 3; +} + +message CompleteOrchestrationAction { + OrchestrationStatus orchestrationStatus = 1; + google.protobuf.StringValue result = 2; + google.protobuf.StringValue details = 3; + google.protobuf.StringValue newVersion = 4; + repeated HistoryEvent carryoverEvents = 5; + TaskFailureDetails failureDetails = 6; + map tags = 7; +} + +message TerminateOrchestrationAction { + string instanceId = 1; + google.protobuf.StringValue reason = 2; + bool recurse = 3; +} + +message SendEntityMessageAction { + oneof EntityMessageType { + EntityOperationSignaledEvent entityOperationSignaled = 1; + EntityOperationCalledEvent entityOperationCalled = 2; + EntityLockRequestedEvent entityLockRequested = 3; + EntityUnlockSentEvent entityUnlockSent = 4; + } +} + +message OrchestratorAction { + int32 id = 1; + oneof orchestratorActionType { + ScheduleTaskAction scheduleTask = 2; + CreateSubOrchestrationAction createSubOrchestration = 3; + CreateTimerAction createTimer = 4; + SendEventAction sendEvent = 5; + CompleteOrchestrationAction completeOrchestration = 6; + TerminateOrchestrationAction terminateOrchestration = 7; + SendEntityMessageAction sendEntityMessage = 8; + } +} + +message OrchestrationTraceContext { + google.protobuf.StringValue spanID = 1; + google.protobuf.Timestamp spanStartTime = 2; +} + +message OrchestratorRequest { + string instanceId = 1; + google.protobuf.StringValue executionId = 2; + repeated HistoryEvent pastEvents = 3; + repeated HistoryEvent newEvents = 4; + OrchestratorEntityParameters entityParameters = 5; + bool requiresHistoryStreaming = 6; + map properties = 7; + + OrchestrationTraceContext orchestrationTraceContext = 8; +} + +message OrchestratorResponse { + string instanceId = 1; + repeated OrchestratorAction actions = 2; + google.protobuf.StringValue customStatus = 3; + string completionToken = 4; + + // The number of work item events that were processed by the orchestrator. + // This field is optional. If not set, the service should assume that the orchestrator processed all events. + google.protobuf.Int32Value numEventsProcessed = 5; + OrchestrationTraceContext orchestrationTraceContext = 6; + + // Whether or not a history is required to complete the original OrchestratorRequest and none was provided. + bool requiresHistory = 7; + + // True if this is a partial (chunked) completion. The backend must keep the work item open until the final chunk (isPartial=false). + bool isPartial = 8; + + // Zero-based position of the current chunk within a chunked completion sequence. + // This field is omitted for non-chunked completions. + google.protobuf.Int32Value chunkIndex = 9; +} + +message CreateInstanceRequest { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue version = 3; + google.protobuf.StringValue input = 4; + google.protobuf.Timestamp scheduledStartTimestamp = 5; + OrchestrationIdReusePolicy orchestrationIdReusePolicy = 6; + google.protobuf.StringValue executionId = 7; + map tags = 8; + TraceContext parentTraceContext = 9; + google.protobuf.Timestamp requestTime = 10; +} + +message OrchestrationIdReusePolicy { + repeated OrchestrationStatus replaceableStatus = 1; + reserved 2; +} + +message CreateInstanceResponse { + string instanceId = 1; +} + +message GetInstanceRequest { + string instanceId = 1; + bool getInputsAndOutputs = 2; +} + +message GetInstanceResponse { + bool exists = 1; + OrchestrationState orchestrationState = 2; +} + +message RewindInstanceRequest { + string instanceId = 1; + google.protobuf.StringValue reason = 2; +} + +message RewindInstanceResponse { + // Empty for now. Using explicit type incase we want to add content later. +} + +message OrchestrationState { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue version = 3; + OrchestrationStatus orchestrationStatus = 4; + google.protobuf.Timestamp scheduledStartTimestamp = 5; + google.protobuf.Timestamp createdTimestamp = 6; + google.protobuf.Timestamp lastUpdatedTimestamp = 7; + google.protobuf.StringValue input = 8; + google.protobuf.StringValue output = 9; + google.protobuf.StringValue customStatus = 10; + TaskFailureDetails failureDetails = 11; + google.protobuf.StringValue executionId = 12; + google.protobuf.Timestamp completedTimestamp = 13; + google.protobuf.StringValue parentInstanceId = 14; + map tags = 15; +} + +message RaiseEventRequest { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue input = 3; +} + +message RaiseEventResponse { + // No payload +} + +message TerminateRequest { + string instanceId = 1; + google.protobuf.StringValue output = 2; + bool recursive = 3; +} + +message TerminateResponse { + // No payload +} + +message SuspendRequest { + string instanceId = 1; + google.protobuf.StringValue reason = 2; +} + +message SuspendResponse { + // No payload +} + +message ResumeRequest { + string instanceId = 1; + google.protobuf.StringValue reason = 2; +} + +message ResumeResponse { + // No payload +} + +message QueryInstancesRequest { + InstanceQuery query = 1; +} + +message InstanceQuery{ + repeated OrchestrationStatus runtimeStatus = 1; + google.protobuf.Timestamp createdTimeFrom = 2; + google.protobuf.Timestamp createdTimeTo = 3; + repeated google.protobuf.StringValue taskHubNames = 4; + int32 maxInstanceCount = 5; + google.protobuf.StringValue continuationToken = 6; + google.protobuf.StringValue instanceIdPrefix = 7; + bool fetchInputsAndOutputs = 8; +} + +message QueryInstancesResponse { + repeated OrchestrationState orchestrationState = 1; + google.protobuf.StringValue continuationToken = 2; +} + +message ListInstanceIdsRequest { + repeated OrchestrationStatus runtimeStatus = 1; + google.protobuf.Timestamp completedTimeFrom = 2; + google.protobuf.Timestamp completedTimeTo = 3; + int32 pageSize = 4; + google.protobuf.StringValue lastInstanceKey = 5; +} + +message ListInstanceIdsResponse { + repeated string instanceIds = 1; + google.protobuf.StringValue lastInstanceKey = 2; +} + +message PurgeInstancesRequest { + oneof request { + string instanceId = 1; + PurgeInstanceFilter purgeInstanceFilter = 2; + InstanceBatch instanceBatch = 4; + } + bool recursive = 3; + // used in the case when an instanceId is specified to determine if the purge request is for an orchestration (as opposed to an entity) + bool isOrchestration = 5; +} + +message PurgeInstanceFilter { + google.protobuf.Timestamp createdTimeFrom = 1; + google.protobuf.Timestamp createdTimeTo = 2; + repeated OrchestrationStatus runtimeStatus = 3; +} + +message PurgeInstancesResponse { + int32 deletedInstanceCount = 1; + google.protobuf.BoolValue isComplete = 2; +} + +message RestartInstanceRequest { + string instanceId = 1; + bool restartWithNewInstanceId = 2; +} + +message RestartInstanceResponse { + string instanceId = 1; +} + +message CreateTaskHubRequest { + bool recreateIfExists = 1; +} + +message CreateTaskHubResponse { + //no playload +} + +message DeleteTaskHubRequest { + //no playload +} + +message DeleteTaskHubResponse { + //no playload +} + +message SignalEntityRequest { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue input = 3; + string requestId = 4; + google.protobuf.Timestamp scheduledTime = 5; + TraceContext parentTraceContext = 6; + google.protobuf.Timestamp requestTime = 7; +} + +message SignalEntityResponse { + // no payload +} + +message GetEntityRequest { + string instanceId = 1; + bool includeState = 2; +} + +message GetEntityResponse { + bool exists = 1; + EntityMetadata entity = 2; +} + +message EntityQuery { + google.protobuf.StringValue instanceIdStartsWith = 1; + google.protobuf.Timestamp lastModifiedFrom = 2; + google.protobuf.Timestamp lastModifiedTo = 3; + bool includeState = 4; + bool includeTransient = 5; + google.protobuf.Int32Value pageSize = 6; + google.protobuf.StringValue continuationToken = 7; +} + +message QueryEntitiesRequest { + EntityQuery query = 1; +} + +message QueryEntitiesResponse { + repeated EntityMetadata entities = 1; + google.protobuf.StringValue continuationToken = 2; +} + +message EntityMetadata { + string instanceId = 1; + google.protobuf.Timestamp lastModifiedTime = 2; + int32 backlogQueueSize = 3; + google.protobuf.StringValue lockedBy = 4; + google.protobuf.StringValue serializedState = 5; +} + +message CleanEntityStorageRequest { + google.protobuf.StringValue continuationToken = 1; + bool removeEmptyEntities = 2; + bool releaseOrphanedLocks = 3; +} + +message CleanEntityStorageResponse { + google.protobuf.StringValue continuationToken = 1; + int32 emptyEntitiesRemoved = 2; + int32 orphanedLocksReleased = 3; +} + +message OrchestratorEntityParameters { + google.protobuf.Duration entityMessageReorderWindow = 1; +} + +message EntityBatchRequest { + string instanceId = 1; + google.protobuf.StringValue entityState = 2; + repeated OperationRequest operations = 3; + map properties = 4; +} + +message EntityBatchResult { + repeated OperationResult results = 1; + repeated OperationAction actions = 2; + google.protobuf.StringValue entityState = 3; + TaskFailureDetails failureDetails = 4; + string completionToken = 5; + repeated OperationInfo operationInfos = 6; // used only with DTS + // Whether or not an entity state is required to complete the original EntityBatchRequest and none was provided. + bool requiresState = 7; +} + +message EntityRequest { + string instanceId = 1; + string executionId = 2; + google.protobuf.StringValue entityState = 3; // null if entity does not exist + repeated HistoryEvent operationRequests = 4; +} + +message OperationRequest { + string operation = 1; + string requestId = 2; + google.protobuf.StringValue input = 3; + TraceContext traceContext = 4; +} + +message OperationResult { + oneof resultType { + OperationResultSuccess success = 1; + OperationResultFailure failure = 2; + } +} + +message OperationInfo { + string requestId = 1; + OrchestrationInstance responseDestination = 2; // null for signals +} + +message OperationResultSuccess { + google.protobuf.StringValue result = 1; + google.protobuf.Timestamp startTimeUtc = 2; + google.protobuf.Timestamp endTimeUtc = 3; +} + +message OperationResultFailure { + TaskFailureDetails failureDetails = 1; + google.protobuf.Timestamp startTimeUtc = 2; + google.protobuf.Timestamp endTimeUtc = 3; +} + +message OperationAction { + int32 id = 1; + oneof operationActionType { + SendSignalAction sendSignal = 2; + StartNewOrchestrationAction startNewOrchestration = 3; + } +} + +message SendSignalAction { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue input = 3; + google.protobuf.Timestamp scheduledTime = 4; + google.protobuf.Timestamp requestTime = 5; + TraceContext parentTraceContext = 6; +} + +message StartNewOrchestrationAction { + string instanceId = 1; + string name = 2; + google.protobuf.StringValue version = 3; + google.protobuf.StringValue input = 4; + google.protobuf.Timestamp scheduledTime = 5; + google.protobuf.Timestamp requestTime = 6; + TraceContext parentTraceContext = 7; +} + +message AbandonActivityTaskRequest { + string completionToken = 1; +} + +message AbandonActivityTaskResponse { + // Empty. +} + +message AbandonOrchestrationTaskRequest { + string completionToken = 1; +} + +message AbandonOrchestrationTaskResponse { + // Empty. +} + +message AbandonEntityTaskRequest { + string completionToken = 1; +} + +message AbandonEntityTaskResponse { + // Empty. +} + +message SkipGracefulOrchestrationTerminationsRequest { + InstanceBatch instanceBatch = 1; + google.protobuf.StringValue reason = 2; +} + +message SkipGracefulOrchestrationTerminationsResponse { + // Those instances which could not be terminated because they had locked entities at the time of this termination call, + // are already in a terminal state (completed, failed, terminated, etc.), are not orchestrations, or do not exist (i.e. have been purged) + repeated string unterminatedInstanceIds = 1; +} + +service TaskHubSidecarService { + // Sends a hello request to the sidecar service. + rpc Hello(google.protobuf.Empty) returns (google.protobuf.Empty); + + // Starts a new orchestration instance. + rpc StartInstance(CreateInstanceRequest) returns (CreateInstanceResponse); + + // Gets the status of an existing orchestration instance. + rpc GetInstance(GetInstanceRequest) returns (GetInstanceResponse); + + // Rewinds an orchestration instance to last known good state and replays from there. + rpc RewindInstance(RewindInstanceRequest) returns (RewindInstanceResponse); + + // Restarts an orchestration instance. + rpc RestartInstance(RestartInstanceRequest) returns (RestartInstanceResponse); + + // Waits for an orchestration instance to reach a running or completion state. + rpc WaitForInstanceStart(GetInstanceRequest) returns (GetInstanceResponse); + + // Waits for an orchestration instance to reach a completion state (completed, failed, terminated, etc.). + rpc WaitForInstanceCompletion(GetInstanceRequest) returns (GetInstanceResponse); + + // Raises an event to a running orchestration instance. + rpc RaiseEvent(RaiseEventRequest) returns (RaiseEventResponse); + + // Terminates a running orchestration instance. + rpc TerminateInstance(TerminateRequest) returns (TerminateResponse); + + // Suspends a running orchestration instance. + rpc SuspendInstance(SuspendRequest) returns (SuspendResponse); + + // Resumes a suspended orchestration instance. + rpc ResumeInstance(ResumeRequest) returns (ResumeResponse); + + // rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse); + + rpc QueryInstances(QueryInstancesRequest) returns (QueryInstancesResponse); + + rpc ListInstanceIds(ListInstanceIdsRequest) returns (ListInstanceIdsResponse); + + rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); + + rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); + rpc CompleteActivityTask(ActivityResponse) returns (CompleteTaskResponse); + rpc CompleteOrchestratorTask(OrchestratorResponse) returns (CompleteTaskResponse); + rpc CompleteEntityTask(EntityBatchResult) returns (CompleteTaskResponse); + + // Gets the history of an orchestration instance as a stream of events. + rpc StreamInstanceHistory(StreamInstanceHistoryRequest) returns (stream HistoryChunk); + + // Deletes and Creates the necessary resources for the orchestration service and the instance store + rpc CreateTaskHub(CreateTaskHubRequest) returns (CreateTaskHubResponse); + + // Deletes the resources for the orchestration service and optionally the instance store + rpc DeleteTaskHub(DeleteTaskHubRequest) returns (DeleteTaskHubResponse); + + // sends a signal to an entity + rpc SignalEntity(SignalEntityRequest) returns (SignalEntityResponse); + + // get information about a specific entity + rpc GetEntity(GetEntityRequest) returns (GetEntityResponse); + + // query entities + rpc QueryEntities(QueryEntitiesRequest) returns (QueryEntitiesResponse); + + // clean entity storage + rpc CleanEntityStorage(CleanEntityStorageRequest) returns (CleanEntityStorageResponse); + + // Abandons a single work item + rpc AbandonTaskActivityWorkItem(AbandonActivityTaskRequest) returns (AbandonActivityTaskResponse); + + // Abandon an orchestration work item + rpc AbandonTaskOrchestratorWorkItem(AbandonOrchestrationTaskRequest) returns (AbandonOrchestrationTaskResponse); + + // Abandon an entity work item + rpc AbandonTaskEntityWorkItem(AbandonEntityTaskRequest) returns (AbandonEntityTaskResponse); + + // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". + // Note that a maximum of 500 orchestrations can be terminated at a time using this method. + rpc SkipGracefulOrchestrationTerminations(SkipGracefulOrchestrationTerminationsRequest) returns (SkipGracefulOrchestrationTerminationsResponse); +} + +message GetWorkItemsRequest { + int32 maxConcurrentOrchestrationWorkItems = 1; + int32 maxConcurrentActivityWorkItems = 2; + int32 maxConcurrentEntityWorkItems = 3; + + repeated WorkerCapability capabilities = 10; +} + +enum WorkerCapability { + WORKER_CAPABILITY_UNSPECIFIED = 0; + + // Indicates that the worker is capable of streaming instance history as a more optimized + // alternative to receiving the full history embedded in the orchestrator work-item. + // When set, the service may return work items without any history events as an optimization. + // It is strongly recommended that all SDKs support this capability. + WORKER_CAPABILITY_HISTORY_STREAMING = 1; + + // Indicates that the worker supports scheduled tasks. + // The service may send schedule-triggered orchestration work items, + // and the worker must handle them, including the scheduledTime field. + WORKER_CAPABILITY_SCHEDULED_TASKS = 2; + + // Signals that the worker can handle large payloads stored externally (e.g., Blob Storage). + // Work items may contain URI references instead of inline data, and the worker must fetch them. + // This avoids message size limits and reduces network overhead. + WORKER_CAPABILITY_LARGE_PAYLOADS = 3; +} + +message WorkItem { + oneof request { + OrchestratorRequest orchestratorRequest = 1; + ActivityRequest activityRequest = 2; + EntityBatchRequest entityRequest = 3; // (older) used by orchestration services implementations + HealthPing healthPing = 4; + EntityRequest entityRequestV2 = 5; // (newer) used by backend service implementations + } + string completionToken = 10; +} + +message CompleteTaskResponse { + // No payload +} + +message HealthPing { + // No payload +} + +message StreamInstanceHistoryRequest { + string instanceId = 1; + google.protobuf.StringValue executionId = 2; + + // When set to true, the service may return a more optimized response suitable for workers. + bool forWorkItemProcessing = 3; +} + +message HistoryChunk { + repeated HistoryEvent events = 1; +} + +message InstanceBatch { + // A maximum of 500 instance IDs can be provided in this list. + repeated string instanceIds = 1; +} diff --git a/package.json b/package.json index 1f4c22e..82049c0 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,9 @@ "dev": "nodemon --watch './src/**/*.ts' --exec 'ts-node --swc' ./src/index.ts", "pretty": "prettier --list-different \"**/*.{ts,tsx,js,jsx,json,md}\"", "pretty-fix": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"", + "download-proto": "./scripts/download-proto.sh", "prebuild": "./scripts/prebuild.sh", - "build": "npm install && npm run lint && npm run pretty && ./scripts/build.sh" + "build": "npm install && npm run download-proto && npm run lint && npm run pretty && ./scripts/build.sh" }, "repository": { "type": "git", diff --git a/scripts/build.sh b/scripts/build.sh index a61e585..c77bd53 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -42,4 +42,4 @@ cp README.md build/ # Copy Proto Files cp -R ./src/proto ./build -cp -R ./submodules/durabletask-protobuf/protos/orchestrator_service.proto ./build/proto +cp -R ./internal/durabletask-protobuf/protos/orchestrator_service.proto ./build/proto diff --git a/scripts/download-proto.sh b/scripts/download-proto.sh new file mode 100755 index 0000000..ced378f --- /dev/null +++ b/scripts/download-proto.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Copyright 2022 The Dapr Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script downloads the proto files from the durabletask-protobuf repository. +# Usage: ./scripts/download-proto.sh [branch-name] +# If no branch is specified, 'main' is used. + +set -e + +BRANCH=${1:-main} +PROTO_DIR="internal/durabletask-protobuf/protos" +COMMIT_HASH_FILE="internal/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH" + +echo "=====================================================" +echo "========== Download Proto Files =====================" +echo "=====================================================" +echo "Branch: $BRANCH" + +# Create directory if it doesn't exist +mkdir -p "$PROTO_DIR" + +# Download the proto file +echo "Downloading orchestrator_service.proto..." +curl -sfL "https://raw.githubusercontent.com/microsoft/durabletask-protobuf/${BRANCH}/protos/orchestrator_service.proto" -o "$PROTO_DIR/orchestrator_service.proto" + +# Get and save the commit hash +echo "Fetching commit hash..." +API_RESPONSE=$(curl -sfL "https://api.github.com/repos/microsoft/durabletask-protobuf/commits?path=protos/orchestrator_service.proto&sha=${BRANCH}&per_page=1") + +# Try to parse with jq if available, otherwise fall back to grep/sed +if command -v jq &> /dev/null; then + COMMIT_HASH=$(echo "$API_RESPONSE" | jq -r '.[0].sha') +else + COMMIT_HASH=$(echo "$API_RESPONSE" | grep '"sha":' | head -1 | sed 's/.*"sha": "\([^"]*\)".*/\1/') +fi + +if [ -n "$COMMIT_HASH" ] && [ "$COMMIT_HASH" != "null" ]; then + echo "$COMMIT_HASH" > "$COMMIT_HASH_FILE" + echo "Commit hash saved: $COMMIT_HASH" +else + echo "Error: Could not fetch commit hash from GitHub API" + exit 1 +fi + +echo "Proto files downloaded successfully!" diff --git a/submodules/durabletask-protobuf b/submodules/durabletask-protobuf deleted file mode 160000 index 139a8e3..0000000 --- a/submodules/durabletask-protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 139a8e31fa37694163e02985ddc73f25e08e13c6 diff --git a/tools/generate-grpc-javascript.sh b/tools/generate-grpc-javascript.sh index 99a6a28..cb9eb64 100644 --- a/tools/generate-grpc-javascript.sh +++ b/tools/generate-grpc-javascript.sh @@ -1,5 +1,5 @@ PATH_ROOT=$(readlink -f $(dirname $0))/.. -PATH_PROTO_SRC=$(readlink -f $(dirname $0))/../submodules/durabletask-protobuf/protos +PATH_PROTO_SRC=$(readlink -f $(dirname $0))/../internal/durabletask-protobuf/protos PATH_PROTO_OUT=$(readlink -f $1) echo "=============================================="