From 5069df001131175f564656da548be2ac99215429 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Thu, 11 Dec 2025 23:15:22 +0100 Subject: [PATCH 01/49] FIX TYPO in transcription_request_completed example message --- server/config/messageBroker/LavinMQ.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/config/messageBroker/LavinMQ.js b/server/config/messageBroker/LavinMQ.js index 9e585a0..47a4d69 100644 --- a/server/config/messageBroker/LavinMQ.js +++ b/server/config/messageBroker/LavinMQ.js @@ -80,7 +80,7 @@ export const connectToMessageBroker = async () => { /* const exampleReceivedMessage = { analysisEntryId: transcriptionJobDetails._id, - transcriptionData: array of objects, each representing a segment of the transcription + transcriptionSegments: array of objects, each representing a segment of the transcription fullTranscript: 'lorem ipsum dolor sit amet, consectetur adipiscing elit' }; */ From 20a1b4dfc5ce3260139c52dac3f0661941493a91 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Thu, 11 Dec 2025 23:20:54 +0100 Subject: [PATCH 02/49] FIX TYPO analysisPresignedUploadUrl --> analysisEntryPresignedUploadUrl --- server/controllers/analysisController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/controllers/analysisController.js b/server/controllers/analysisController.js index 74702ce..3f21a91 100644 --- a/server/controllers/analysisController.js +++ b/server/controllers/analysisController.js @@ -135,6 +135,6 @@ export const participateInAnalysis = async (req, res) => { message: 'Analysis info retrieved successfully', analysisData: analysisData, analysisEntryId: analysisEntry.id, - analysisPresignedUploadUrl: analysisEntryPresignedUploadUrl, + analysisEntryPresignedUploadUrl: analysisEntryPresignedUploadUrl, }); }; From 637f80f3d11083b49b2791e9f80dcd36679e2c8d Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 12 Dec 2025 23:21:57 +0100 Subject: [PATCH 03/49] ADD transcriptionJob model to prisma schema --- prisma/schema.prisma | 76 +++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b380f13..cb18ab6 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -44,18 +44,18 @@ model Company { } model ParticipantProfile { - id Int @id @default(autoincrement()) - name String? - last_name String? - age String? // String to allow for prefer not to say option - gender Genders? - invited_by String? - country String? - devices String[] @default([]) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt - User User @relation(fields: [user_id], references: [id]) - user_id String @unique + id Int @id @default(autoincrement()) + name String? + last_name String? + age String? // String to allow for prefer not to say option + gender Genders? + invited_by String? + country String? + devices String[] @default([]) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + User User @relation(fields: [user_id], references: [id]) + user_id String @unique AnalysisEntry AnalysisEntry[] } @@ -85,11 +85,11 @@ model Session { } model Subscription { - id String @id @unique @default(uuid()) - company_id String @unique - Company Company @relation(fields: [company_id], references: [id]) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt + id String @id @unique @default(uuid()) + company_id String @unique + Company Company @relation(fields: [company_id], references: [id]) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt expires_at DateTime? } @@ -105,30 +105,46 @@ model Analysis { tasks Json url String - status AnalysisStatus @default(draft) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt + status AnalysisStatus @default(draft) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt max_number_of_participants Int - AnalysisEntry AnalysisEntry[] + AnalysisEntry AnalysisEntry[] @@index([owner_company_id]) // Not a unique field, but requires an index } model AnalysisEntry { - id String @id @default(uuid()) - Analysis Analysis @relation(fields: [analysis_id], references: [id]) - analysis_id String - ParticipantProfile ParticipantProfile? @relation(fields: [user_id], references: [user_id]) - user_id String? - status AnalysisEntryCompletionStatus @default(in_progress) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt + id String @id @default(uuid()) + Analysis Analysis @relation(fields: [analysis_id], references: [id]) + analysis_id String + ParticipantProfile ParticipantProfile? @relation(fields: [user_id], references: [user_id]) + user_id String? + status AnalysisEntryCompletionStatus @default(in_progress) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt transcription_segments Json? - full_transcript String? + full_transcript String? + transcriptionJob TranscriptionJob? @@index([analysis_id, status]) } +model TranscriptionJob { + id String @id @unique @default(uuid()) + AnalysisEntry AnalysisEntry @relation(fields: [analysis_entry_id], references: [id]) + analysis_entry_id String @unique + status TranscriptionJobStatus @default(pending) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt +} + +enum TranscriptionJobStatus { + pending + processing + completed +} + enum Genders { male female From 56d9658626d4d0367b92ea08742a1f5b1cd4b56f Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 12 Dec 2025 23:24:03 +0100 Subject: [PATCH 04/49] ADD npm package AWS transcribe --- package-lock.json | 461 +++++++++++++++++++++++++++++++++++++++++++--- package.json | 1 + 2 files changed, 435 insertions(+), 27 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8016666..239b0f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "Proprietary", "dependencies": { "@aws-sdk/client-s3": "^3.936.0", + "@aws-sdk/client-transcribe": "^3.948.0", "@aws-sdk/s3-request-presigner": "^3.936.0", "@cloudamqp/amqp-client": "^3.4.0", "@getbrevo/brevo": "^3.0.1", @@ -365,6 +366,412 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/client-transcribe": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe/-/client-transcribe-3.948.0.tgz", + "integrity": "sha512-EOPYaW/lL2UHZbsG6PxPeHu/Pcw8MTsUznrRW6z7svVHCgsQkGUoWJs9gxTr601r+TMPgt8rdv2bv+WgXeN/SQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/credential-provider-node": "3.948.0", + "@aws-sdk/middleware-host-header": "3.936.0", + "@aws-sdk/middleware-logger": "3.936.0", + "@aws-sdk/middleware-recursion-detection": "3.948.0", + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/region-config-resolver": "3.936.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@aws-sdk/util-user-agent-browser": "3.936.0", + "@aws-sdk/util-user-agent-node": "3.947.0", + "@smithy/config-resolver": "^4.4.3", + "@smithy/core": "^3.18.7", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/hash-node": "^4.2.5", + "@smithy/invalid-dependency": "^4.2.5", + "@smithy/middleware-content-length": "^4.2.5", + "@smithy/middleware-endpoint": "^4.3.14", + "@smithy/middleware-retry": "^4.4.14", + "@smithy/middleware-serde": "^4.2.6", + "@smithy/middleware-stack": "^4.2.5", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/url-parser": "^4.2.5", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.13", + "@smithy/util-defaults-mode-node": "^4.2.16", + "@smithy/util-endpoints": "^3.2.5", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-retry": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/client-sso": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.948.0.tgz", + "integrity": "sha512-iWjchXy8bIAVBUsKnbfKYXRwhLgRg3EqCQ5FTr3JbR+QR75rZm4ZOYXlvHGztVTmtAZ+PQVA1Y4zO7v7N87C0A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/middleware-host-header": "3.936.0", + "@aws-sdk/middleware-logger": "3.936.0", + "@aws-sdk/middleware-recursion-detection": "3.948.0", + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/region-config-resolver": "3.936.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@aws-sdk/util-user-agent-browser": "3.936.0", + "@aws-sdk/util-user-agent-node": "3.947.0", + "@smithy/config-resolver": "^4.4.3", + "@smithy/core": "^3.18.7", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/hash-node": "^4.2.5", + "@smithy/invalid-dependency": "^4.2.5", + "@smithy/middleware-content-length": "^4.2.5", + "@smithy/middleware-endpoint": "^4.3.14", + "@smithy/middleware-retry": "^4.4.14", + "@smithy/middleware-serde": "^4.2.6", + "@smithy/middleware-stack": "^4.2.5", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/url-parser": "^4.2.5", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.13", + "@smithy/util-defaults-mode-node": "^4.2.16", + "@smithy/util-endpoints": "^3.2.5", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-retry": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/core": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.947.0.tgz", + "integrity": "sha512-Khq4zHhuAkvCFuFbgcy3GrZTzfSX7ZIjIcW1zRDxXRLZKRtuhnZdonqTUfaWi5K42/4OmxkYNpsO7X7trQOeHw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.936.0", + "@aws-sdk/xml-builder": "3.930.0", + "@smithy/core": "^3.18.7", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/signature-v4": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.947.0.tgz", + "integrity": "sha512-VR2V6dRELmzwAsCpK4GqxUi6UW5WNhAXS9F9AzWi5jvijwJo3nH92YNJUP4quMpgFZxJHEWyXLWgPjh9u0zYOA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.947.0.tgz", + "integrity": "sha512-inF09lh9SlHj63Vmr5d+LmwPXZc2IbK8lAruhOr3KLsZAIHEgHgGPXWDC2ukTEMzg0pkexQ6FOhXXad6klK4RA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/util-stream": "^4.5.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.948.0.tgz", + "integrity": "sha512-Cl//Qh88e8HBL7yYkJNpF5eq76IO6rq8GsatKcfVBm7RFVxCqYEPSSBtkHdbtNwQdRQqAMXc6E/lEB/CZUDxnA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/credential-provider-env": "3.947.0", + "@aws-sdk/credential-provider-http": "3.947.0", + "@aws-sdk/credential-provider-login": "3.948.0", + "@aws-sdk/credential-provider-process": "3.947.0", + "@aws-sdk/credential-provider-sso": "3.948.0", + "@aws-sdk/credential-provider-web-identity": "3.948.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/credential-provider-imds": "^4.2.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-login": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.948.0.tgz", + "integrity": "sha512-gcKO2b6eeTuZGp3Vvgr/9OxajMrD3W+FZ2FCyJox363ZgMoYJsyNid1vuZrEuAGkx0jvveLXfwiVS0UXyPkgtw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.948.0.tgz", + "integrity": "sha512-ep5vRLnrRdcsP17Ef31sNN4g8Nqk/4JBydcUJuFRbGuyQtrZZrVT81UeH2xhz6d0BK6ejafDB9+ZpBjXuWT5/Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.947.0", + "@aws-sdk/credential-provider-http": "3.947.0", + "@aws-sdk/credential-provider-ini": "3.948.0", + "@aws-sdk/credential-provider-process": "3.947.0", + "@aws-sdk/credential-provider-sso": "3.948.0", + "@aws-sdk/credential-provider-web-identity": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/credential-provider-imds": "^4.2.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.947.0.tgz", + "integrity": "sha512-WpanFbHe08SP1hAJNeDdBDVz9SGgMu/gc0XJ9u3uNpW99nKZjDpvPRAdW7WLA4K6essMjxWkguIGNOpij6Do2Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.948.0.tgz", + "integrity": "sha512-gqLhX1L+zb/ZDnnYbILQqJ46j735StfWV5PbDjxRzBKS7GzsiYoaf6MyHseEopmWrez5zl5l6aWzig7UpzSeQQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-sso": "3.948.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/token-providers": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.948.0.tgz", + "integrity": "sha512-MvYQlXVoJyfF3/SmnNzOVEtANRAiJIObEUYYyjTqKZTmcRIVVky0tPuG26XnB8LmTYgtESwJIZJj/Eyyc9WURQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.948.0.tgz", + "integrity": "sha512-Qa8Zj+EAqA0VlAVvxpRnpBpIWJI9KUwaioY1vkeNVwXPlNaz9y9zCKVM9iU9OZ5HXpoUg6TnhATAHXHAE8+QsQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.936.0", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/protocol-http": "^5.3.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.947.0.tgz", + "integrity": "sha512-7rpKV8YNgCP2R4F9RjWZFcD2R+SO/0R4VHIbY9iZJdH2MzzJ8ZG7h8dZ2m8QkQd1fjx4wrFJGGPJUTYXPV3baA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@smithy/core": "^3.18.7", + "@smithy/protocol-http": "^5.3.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/nested-clients": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.948.0.tgz", + "integrity": "sha512-zcbJfBsB6h254o3NuoEkf0+UY1GpE9ioiQdENWv7odo69s8iaGBEQ4BDpsIMqcuiiUXw1uKIVNxCB1gUGYz8lw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/middleware-host-header": "3.936.0", + "@aws-sdk/middleware-logger": "3.936.0", + "@aws-sdk/middleware-recursion-detection": "3.948.0", + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/region-config-resolver": "3.936.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@aws-sdk/util-user-agent-browser": "3.936.0", + "@aws-sdk/util-user-agent-node": "3.947.0", + "@smithy/config-resolver": "^4.4.3", + "@smithy/core": "^3.18.7", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/hash-node": "^4.2.5", + "@smithy/invalid-dependency": "^4.2.5", + "@smithy/middleware-content-length": "^4.2.5", + "@smithy/middleware-endpoint": "^4.3.14", + "@smithy/middleware-retry": "^4.4.14", + "@smithy/middleware-serde": "^4.2.6", + "@smithy/middleware-stack": "^4.2.5", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/url-parser": "^4.2.5", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.13", + "@smithy/util-defaults-mode-node": "^4.2.16", + "@smithy/util-endpoints": "^3.2.5", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-retry": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/token-providers": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.948.0.tgz", + "integrity": "sha512-V487/kM4Teq5dcr1t5K6eoUKuqlGr9FRWL3MIMukMERJXHZvio6kox60FZ/YtciRHRI75u14YUqm2Dzddcu3+A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.947.0.tgz", + "integrity": "sha512-+vhHoDrdbb+zerV4noQk1DHaUMNzWFWPpPYjVTwW2186k5BEJIecAMChYkghRrBVJ3KPWP1+JnZwOd72F3d4rQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, "node_modules/@aws-sdk/core": { "version": "3.940.0", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.940.0.tgz", @@ -958,9 +1365,9 @@ } }, "node_modules/@aws/lambda-invoke-store": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.1.tgz", - "integrity": "sha512-sIyFcoPZkTtNu9xFeEoynMef3bPJIAbOfUh+ueYcfhVl6xm2VRtMcMclSxmZCMnHHd4hlYKJeq/aggmBEWynww==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.2.tgz", + "integrity": "sha512-C0NBLsIqzDIae8HFw9YIrIBsbc0xTiOtt7fAukGPnqQ/+zZNaq+4jhuccltK0QuWHBnNm/a6kLIRA6GFiM10eg==", "license": "Apache-2.0", "engines": { "node": ">=18.0.0" @@ -2905,9 +3312,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.18.6", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.18.6.tgz", - "integrity": "sha512-8Q/ugWqfDUEU1Exw71+DoOzlONJ2Cn9QA8VeeDzLLjzO/qruh9UKFzbszy4jXcIYgGofxYiT0t1TT6+CT/GupQ==", + "version": "3.18.7", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.18.7.tgz", + "integrity": "sha512-axG9MvKhMWOhFbvf5y2DuyTxQueO0dkedY9QC3mAfndLosRI/9LJv8WaL0mw7ubNhsO4IuXX9/9dYGPFvHrqlw==", "license": "Apache-2.0", "dependencies": { "@smithy/middleware-serde": "^4.2.6", @@ -3125,12 +3532,12 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.3.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.3.13.tgz", - "integrity": "sha512-X4za1qCdyx1hEVVXuAWlZuK6wzLDv1uw1OY9VtaYy1lULl661+frY7FeuHdYdl7qAARUxH2yvNExU2/SmRFfcg==", + "version": "4.3.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.3.14.tgz", + "integrity": "sha512-v0q4uTKgBM8dsqGjqsabZQyH85nFaTnFcgpWU1uydKFsdyyMzfvOkNum9G7VK+dOP01vUnoZxIeRiJ6uD0kjIg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.18.6", + "@smithy/core": "^3.18.7", "@smithy/middleware-serde": "^4.2.6", "@smithy/node-config-provider": "^4.3.5", "@smithy/shared-ini-file-loader": "^4.4.0", @@ -3144,15 +3551,15 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.4.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.13.tgz", - "integrity": "sha512-RzIDF9OrSviXX7MQeKOm8r/372KTyY8Jmp6HNKOOYlrguHADuM3ED/f4aCyNhZZFLG55lv5beBin7nL0Nzy1Dw==", + "version": "4.4.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.14.tgz", + "integrity": "sha512-Z2DG8Ej7FyWG1UA+7HceINtSLzswUgs2np3sZX0YBBxCt+CXG4QUxv88ZDS3+2/1ldW7LqtSY1UO/6VQ1pND8Q==", "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.5", "@smithy/protocol-http": "^5.3.5", "@smithy/service-error-classification": "^4.2.5", - "@smithy/smithy-client": "^4.9.9", + "@smithy/smithy-client": "^4.9.10", "@smithy/types": "^4.9.0", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", @@ -3319,13 +3726,13 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.9.9", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.9.9.tgz", - "integrity": "sha512-SUnZJMMo5yCmgjopJbiNeo1vlr8KvdnEfIHV9rlD77QuOGdRotIVBcOrBuMr+sI9zrnhtDtLP054bZVbpZpiQA==", + "version": "4.9.10", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.9.10.tgz", + "integrity": "sha512-Jaoz4Jw1QYHc1EFww/E6gVtNjhoDU+gwRKqXP6C3LKYqqH2UQhP8tMP3+t/ePrhaze7fhLE8vS2q6vVxBANFTQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.18.6", - "@smithy/middleware-endpoint": "^4.3.13", + "@smithy/core": "^3.18.7", + "@smithy/middleware-endpoint": "^4.3.14", "@smithy/middleware-stack": "^4.2.5", "@smithy/protocol-http": "^5.3.5", "@smithy/types": "^4.9.0", @@ -3426,13 +3833,13 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.12", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.12.tgz", - "integrity": "sha512-TKc6FnOxFULKxLgTNHYjcFqdOYzXVPFFVm5JhI30F3RdhT7nYOtOsjgaOwfDRmA/3U66O9KaBQ3UHoXwayRhAg==", + "version": "4.3.13", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.13.tgz", + "integrity": "sha512-hlVLdAGrVfyNei+pKIgqDTxfu/ZI2NSyqj4IDxKd5bIsIqwR/dSlkxlPaYxFiIaDVrBy0he8orsFy+Cz119XvA==", "license": "Apache-2.0", "dependencies": { "@smithy/property-provider": "^4.2.5", - "@smithy/smithy-client": "^4.9.9", + "@smithy/smithy-client": "^4.9.10", "@smithy/types": "^4.9.0", "tslib": "^2.6.2" }, @@ -3441,16 +3848,16 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.15", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.15.tgz", - "integrity": "sha512-94NqfQVo+vGc5gsQ9SROZqOvBkGNMQu6pjXbnn8aQvBUhc31kx49gxlkBEqgmaZQHUUfdRUin5gK/HlHKmbAwg==", + "version": "4.2.16", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.16.tgz", + "integrity": "sha512-F1t22IUiJLHrxW9W1CQ6B9PN+skZ9cqSuzB18Eh06HrJPbjsyZ7ZHecAKw80DQtyGTRcVfeukKaCRYebFwclbg==", "license": "Apache-2.0", "dependencies": { "@smithy/config-resolver": "^4.4.3", "@smithy/credential-provider-imds": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/property-provider": "^4.2.5", - "@smithy/smithy-client": "^4.9.9", + "@smithy/smithy-client": "^4.9.10", "@smithy/types": "^4.9.0", "tslib": "^2.6.2" }, diff --git a/package.json b/package.json index 151a5ca..a14a165 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.936.0", + "@aws-sdk/client-transcribe": "^3.948.0", "@aws-sdk/s3-request-presigner": "^3.936.0", "@cloudamqp/amqp-client": "^3.4.0", "@getbrevo/brevo": "^3.0.1", From 7ec6f1fdb275e2d97d19ae100a996415f58da4ea Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 00:42:44 +0100 Subject: [PATCH 05/49] RENAME updateAnalysisEntryInDb to markAnalysisEntryAsSubmitted for clarity --- server/controllers/analysisEntryController.js | 4 ++-- server/models/analysisEntryModel.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index 196e00e..12b68ed 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -16,9 +16,9 @@ export const createAnalysisEntry = async (req, res) => { }; export const updateAnalysisEntry = async (req, res) => { - const { analysisEntryId, analysisEntryStatus, analysisId } = req.body; + const { analysisEntryId } = req.body; - await updateAnalysisEntryInDb(analysisEntryId, analysisEntryStatus); + await markAnalysisEntryAsSubmitted(analysisEntryId); try { const message = { diff --git a/server/models/analysisEntryModel.js b/server/models/analysisEntryModel.js index 4bc7844..b353bcd 100644 --- a/server/models/analysisEntryModel.js +++ b/server/models/analysisEntryModel.js @@ -43,7 +43,7 @@ export const createAnalysisEntryInDb = async (analysisId) => { return createAnalysisEntryQuery; }; -export const updateAnalysisEntryInDb = async (analysisEntryId, status) => { +export const markAnalysisEntryAsSubmitted = async (analysisEntryId) => { const whereClause = { id: analysisEntryId, }; @@ -51,7 +51,7 @@ export const updateAnalysisEntryInDb = async (analysisEntryId, status) => { const analysisEntryUpdateQuery = await prisma.analysisEntry.update({ where: whereClause, data: { - status: status, + status: 'submitted', }, select: { status: true, From 6906c00753ce5d909d881636c18116fd9acf16b3 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 00:43:04 +0100 Subject: [PATCH 06/49] ADD transcription normalizer util --- .../transcription/transcriptionNormalizer.js | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 server/utils/transcription/transcriptionNormalizer.js diff --git a/server/utils/transcription/transcriptionNormalizer.js b/server/utils/transcription/transcriptionNormalizer.js new file mode 100644 index 0000000..73862e5 --- /dev/null +++ b/server/utils/transcription/transcriptionNormalizer.js @@ -0,0 +1,181 @@ +/** + * Converts string numbers to actual numbers + * @param {string|number} value - Value to convert + * @returns {number} Converted number or 0 if invalid + */ +const convertToNumber = (value) => { + if (value === undefined || value === null) { + return 0; + } + + const num = parseFloat(value); + return Number.isNaN(num) ? 0 : num; +}; + +/** + * Groups transcription items into meaningful segments with reduced cluttering + * @param {Array} items - Array of transcription items from AWS Transcribe + * @returns {Array} Array of segments with start_time, end_time, and transcript + */ +const createSegmentsFromItems = (items) => { + if (!items || items.length === 0) { + return []; + } + + const segments = []; + let currentSegment = null; + const SEGMENT_GAP_THRESHOLD = 2.0; // seconds - gap for natural pauses to start new segment + const SEGMENT_MAX_DURATION = 25.0; // seconds - maximum duration to avoid overly long segments + + for (let i = 0; i < items.length; i += 1) { + const item = items[i]; + + // Skip items without alternatives + if (!item.alternatives || item.alternatives.length === 0) { + // Skip this iteration + } else { + const alternative = item.alternatives[0]; + const content = alternative.content || ''; + + // Handle items without timing information (like some punctuation) + if (!item.start_time || !item.end_time) { + if (currentSegment && item.type === 'punctuation') { + currentSegment.transcript += content; + } + } else { + const startTime = convertToNumber(item.start_time); + const endTime = convertToNumber(item.end_time); + + // Start a new segment or continue current segment + if (!currentSegment) { + // Start first segment + currentSegment = { + start_time: startTime, + end_time: endTime, + transcript: content, + }; + } else { + // Check if we should start a new segment based on time gap or max duration + const timeGap = startTime - currentSegment.end_time; + const segmentDuration = startTime - currentSegment.start_time; + + // Start new segment if there's a significant gap OR if we've reached max duration + if (timeGap > SEGMENT_GAP_THRESHOLD || segmentDuration >= SEGMENT_MAX_DURATION) { + // Significant gap or max duration reached - finalize current segment and start new one + segments.push(currentSegment); + currentSegment = { + start_time: startTime, + end_time: endTime, + transcript: content, + }; + } else { + // Continue current segment + // Improved logic for adding spaces around punctuation + const currentText = currentSegment.transcript; + const newContent = content.trim(); + + // Don't add space if current text is empty + if (currentText.length === 0) { + currentSegment.transcript += newContent; + } else { + // Get the last character of current text and first character of new content + const lastChar = currentText[currentText.length - 1]; + const firstChar = newContent[0]; + + // Determine if we need to add space + let shouldAddSpace = false; + + // Add space if: + // 1. Current text doesn't end with punctuation and new content doesn't start with punctuation + // 2. Current text ends with punctuation (except quotes/brackets) and new content starts with a letter/number + // 3. Current text ends with letter/number and new content starts with punctuation (.,!?;:) + if (!/[.,!?;:)\]}'"]$/.test(lastChar) && !/^[.,!?;:([{'"]/.test(firstChar)) { + // Neither ends nor starts with punctuation - add space + shouldAddSpace = true; + } else if (/[.,!?;:)]'?]*$/.test(lastChar) && /^[A-Za-zÁÉÍÓÚÑÜáéíóúñü0-9]/.test(firstChar)) { + // Ends with punctuation and starts with letter/number - add space + shouldAddSpace = true; + } else if (/[A-Za-zÁÉÍÓÚÑÜáéíóúñü0-9]$/.test(lastChar) && /^[.,!?;:]/.test(firstChar)) { + // Ends with letter/number and starts with punctuation - don't add space + shouldAddSpace = false; + } else if (/[)\]}'"]$/.test(lastChar) && /^[A-Za-zÁÉÍÓÚÑÜáéíóúñü0-9]/.test(firstChar)) { + // Ends with closing bracket/quote and starts with letter/number - add space + shouldAddSpace = true; + } + + // Special handling for common Spanish patterns + // Add space after periods followed by capital letters (sentence boundaries) + if (/[.] $/.test(lastChar) && /^[A-ZÁÉÍÓÚÑÜ]/.test(firstChar)) { + shouldAddSpace = true; + } + + // Add space after commas, semicolons, and colons + if (/[,;:]$/.test(lastChar)) { + shouldAddSpace = true; + } + + if (shouldAddSpace) { + currentSegment.transcript += ' '; + } + + currentSegment.transcript += newContent; + } + currentSegment.end_time = endTime; + } + } + } + } + } + + // Add the last segment if it exists + if (currentSegment) { + segments.push(currentSegment); + } + + return segments; +}; + +/** + * Normalizes AWS Transcribe output to the required format + * @param {object} transcript - AWS Transcribe output as parsed object + * @returns {Promise} Normalized transcription data + */ +export const normalizeTranscript = async (transcript) => { + try { + // Handle missing or malformed data + if (!transcript) { + return { + status: 'FAILED', + results: { + segments: [], + }, + }; + } + + // Extract status + const status = transcript.status || 'UNKNOWN'; + + // Extract results section + const results = transcript.results || {}; + + // Extract items array and create segments + const items = results.items || []; + const segments = createSegmentsFromItems(items); + + // Return normalized structure matching the required format + return { + status, + results: { + segments, + }, + }; + } catch (error) { + console.error('Error normalizing transcript:', error); + return { + status: 'FAILED', + results: { + segments: [], + }, + }; + } +}; From 34a299923d5420ca767bcd972d3248930ec203be Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 00:45:05 +0100 Subject: [PATCH 07/49] ADD functionality to get S3 objects from AWS S3 --- server/integrations/aws/s3.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/integrations/aws/s3.js b/server/integrations/aws/s3.js index 18f5e5c..8413349 100644 --- a/server/integrations/aws/s3.js +++ b/server/integrations/aws/s3.js @@ -33,3 +33,17 @@ export const generateS3PutPresignedUrl = async (key) => { return analysisEntryPutPresignedUrl; }; + +export const getS3Object = async (key) => { + const command = new GetObjectCommand({ + Bucket: process.env.AWS_BUCKET, + Key: key, + }); + + const s3Object = await s3client.send(command); + + // Read the response body as a stream and convert to string so it is workable + const responseBody = await s3Object.Body.transformToString(); + + return responseBody; +}; From c9cd86c25bbc75584aaf614f071673d647571068 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 00:48:58 +0100 Subject: [PATCH 08/49] ADD cronjob to get completedTranscriptionJobs --- server/cron/getCompletedTranscriptionJobsScheduler.js | 10 ++++++++++ server/cron/jobsContainer.js | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 server/cron/getCompletedTranscriptionJobsScheduler.js diff --git a/server/cron/getCompletedTranscriptionJobsScheduler.js b/server/cron/getCompletedTranscriptionJobsScheduler.js new file mode 100644 index 0000000..fef45bc --- /dev/null +++ b/server/cron/getCompletedTranscriptionJobsScheduler.js @@ -0,0 +1,10 @@ +import { CronJob } from 'cron'; +import { handleCompletedVideoTranscriptionJobs } from '../services/analysisService.js'; + +export const getCompletedTranscriptionJobsScheduler = new CronJob('15 * * * *', async () => { + try { + await handleCompletedVideoTranscriptionJobs(); + } catch (error) { + console.error('Error checking transcription job status:', error); + } +}); diff --git a/server/cron/jobsContainer.js b/server/cron/jobsContainer.js index 054b7ef..ba15ceb 100644 --- a/server/cron/jobsContainer.js +++ b/server/cron/jobsContainer.js @@ -1,11 +1,13 @@ import { logError } from '../config/loggerFunctions.js'; import { deletePasswordResetTokensScheduler } from './deletePasswordResetTokensScheduler.js'; +import { getCompletedTranscriptionJobsScheduler } from './getCompletedTranscriptionJobsScheduler.js'; import { markAnalysisEntriesAsCancelledScheduler } from './markAsCancelledAnalysisEntriesScheduler.js'; export const startCronJobs = () => { try { deletePasswordResetTokensScheduler.start(); markAnalysisEntriesAsCancelledScheduler.start(); + getCompletedTranscriptionJobsScheduler.start(); console.log('Cron jobs started'); } catch (error) { From 8d410caff13fe148a94ac058eff46f7182460142 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:00:57 +0100 Subject: [PATCH 09/49] UPDATE prisma schema to include language code --- prisma/schema.prisma | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index cb18ab6..8ef0502 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -131,12 +131,13 @@ model AnalysisEntry { } model TranscriptionJob { - id String @id @unique @default(uuid()) + id Int @id @unique @default(autoincrement()) AnalysisEntry AnalysisEntry @relation(fields: [analysis_entry_id], references: [id]) analysis_entry_id String @unique status TranscriptionJobStatus @default(pending) created_at DateTime @default(now()) updated_at DateTime @default(now()) @updatedAt + language_code String } enum TranscriptionJobStatus { From 48cdbc289858debda33500992d79374e52eba65f Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:01:25 +0100 Subject: [PATCH 10/49] ADD analysisService layer --- server/services/analysisService.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 server/services/analysisService.js diff --git a/server/services/analysisService.js b/server/services/analysisService.js new file mode 100644 index 0000000..a65a731 --- /dev/null +++ b/server/services/analysisService.js @@ -0,0 +1,23 @@ +import { logError } from '../config/loggerFunctions.js'; +import { + insertTranscriptionRequestInDb, + updateSingleTranscriptionRequestInDb, + getSingleTranscriptionJobDetailsFromDb, + storeNormalizedTranscriptionInDb, + markTranscriptionAsPublishedToQueue, +} from '../models/transcriptionModel.js'; +import { + requestAnalysisEntryTranscriptionToAWSTranscribe, deleteCompletedTranscriptionJobFromAWS, + fetchSingleTranscriptionJob, +} from '../integrations/aws/Transcribe.js'; +import { normalizeTranscript } from '../utils/transcription/transcriptionNormalizer.js'; + +export const processTranscriptionRequest = async (transcriptionRequest) => { + try { + await insertTranscriptionRequestInDb(transcriptionRequest); + + await requestAnalysisEntryTranscriptionToAWSTranscribe(transcriptionRequest); + } catch (error) { + logError('Error processing transcription request', error); + } +}; From 18e69825f0b8d951395f0512764f6103251a88a2 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:01:32 +0100 Subject: [PATCH 11/49] ADD transcriptionModel - WIP --- server/models/transcriptionModel.js | 112 ++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 server/models/transcriptionModel.js diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js new file mode 100644 index 0000000..dd96efa --- /dev/null +++ b/server/models/transcriptionModel.js @@ -0,0 +1,112 @@ +import { PrismaClient } from '../config/generated/prisma/client/index.js'; + +const prisma = new PrismaClient(); + +export const insertTranscriptionRequestInDb = async (transcriptionRequest) => { + await prisma.transcriptionRequest.create({ + data: { + analysis_entry_id: transcriptionRequest.analysisEntryId, + status: 'pending', + type: transcriptionRequest.mediaType, + language_code: transcriptionRequest.languageCode, + }, + }); +}; + +export const updateSingleTranscriptionRequestInDb = async (transcriptionRequestInsertId) => { + const db = await connectToMongoDB(); + + const transcriptionRequests = db.collection('transcriptionRequests'); + + const updateResult = await transcriptionRequests.updateOne( + { _id: transcriptionRequestInsertId }, + { + $set: { + status: 'IN_PROGRESS', + updatedAt: new Date(), + }, + }, + ); + + return updateResult; +}; + +export const getCompletedTranscriptions = async () => { + const db = await connectToMongoDB(); + + const transcriptionsCollection = db.collection('transcriptionRequests'); + + const completedTranscriptions = await transcriptionsCollection.find({ + publishedToQueue: false, + status: 'COMPLETED', + }).toArray(); + + return completedTranscriptions; +}; + +export const getSingleTranscriptionJobDetailsFromDb = async (transcriptionJobDetails) => { + const db = await connectToMongoDB(); + + const transcriptionsCollection = db.collection('transcriptionRequests'); + + const transcriptionJobDetailsResult = await transcriptionsCollection.findOne( + { _id: transcriptionJobDetails }, + ); + + return transcriptionJobDetailsResult; +}; + +export const storeNormalizedTranscriptionInDb = async (transcriptionJobInsertId, normalizedTranscriptionJob, transcriptionJobResult) => { + const db = await connectToMongoDB(); + + const transcriptionsCollection = db.collection('transcriptionRequests'); + + const updateResult = await transcriptionsCollection.updateOne( + { _id: transcriptionJobInsertId }, + { + $set: { + status: 'COMPLETED', + transcriptionData: { + fullTranscript: transcriptionJobResult.results.transcripts[0].transcript, + segments: normalizedTranscriptionJob.results.segments, + }, + updatedAt: new Date(), + publishedToQueue: false, + }, + }, + ); + + return updateResult; +}; + +export const markTranscriptionAsPublishedToQueue = async (transcriptionJobInsertId) => { + const db = await connectToMongoDB(); + + const transcriptionsCollection = db.collection('transcriptionRequests'); + + const updateResult = await transcriptionsCollection.updateOne( + { _id: transcriptionJobInsertId }, + { + $set: { + // publishedToQueue: false, // * DEBUG + publishedToQueue: true, + updatedAt: new Date(), + }, + }, + ); + + return updateResult; +}; + +export const getTranscriptionJobsInCompletedStatusNotPublishedToQueue = async () => { + const db = await connectToMongoDB(); + + const transcriptionsCollection = db.collection('transcriptionRequests'); + + const transcriptionJobs = await transcriptionsCollection.find({ + status: 'COMPLETED', + publishedToQueue: false, + }).toArray(); + + return transcriptionJobs; +}; From c5002b861657f3b403ccf0a233991fd54c3a4bb1 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:01:45 +0100 Subject: [PATCH 12/49] ADD AWS Transcribe logic --- server/integrations/aws/Transcribe.js | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 server/integrations/aws/Transcribe.js diff --git a/server/integrations/aws/Transcribe.js b/server/integrations/aws/Transcribe.js new file mode 100644 index 0000000..3f5aa0f --- /dev/null +++ b/server/integrations/aws/Transcribe.js @@ -0,0 +1,55 @@ +import { + TranscribeClient, + StartTranscriptionJobCommand, + ListTranscriptionJobsCommand, + DeleteTranscriptionJobCommand, +} from '@aws-sdk/client-transcribe'; +import { getS3Object } from './s3.js'; + +const transcribeClient = new TranscribeClient({ region: process.env.AWS_REGION }); + +export const requestAnalysisEntryTranscriptionToAWSTranscribe = async (transcriptionRequest) => { + const command = new StartTranscriptionJobCommand({ + TranscriptionJobName: transcriptionRequest.analysisEntryId, + LanguageCode: transcriptionRequest.languageCode, + Media: { + MediaFileUri: `s3://${process.env.AWS_BUCKET}/analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/recording.mp4`, + }, + OutputBucketName: process.env.AWS_BUCKET, + OutputKey: `analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/transcription.json`, + }); + + await transcribeClient.send(command); +}; + +export const listCompletedTranscriptionJobsFromAWS = async () => { + const command = new ListTranscriptionJobsCommand({ + Status: 'COMPLETED', + MaxResults: 100, + + }); + + const completedTranscriptionJobs = await transcribeClient.send(command); + const completedTranscriptionJobsSummary = completedTranscriptionJobs.TranscriptionJobSummaries; // returns an array + + return completedTranscriptionJobsSummary; +}; + +export const fetchSingleTranscriptionJob = async (analysisId, analysisEntryId) => { + const key = process.env.DEPLOY_ENVIRONMENT === 'localhost' ? 'analysis/464d4419-3822-41e6-8d8e-27b1783632df/eabd3179-ece0-4163-8c7e-0e2c728e11e6/transcription.json' : `analysis/${analysisId}/${analysisEntryId}/transcription.json`; + + const transcriptionJobResult = await getS3Object(key); + + return transcriptionJobResult; +}; + +export const deleteCompletedTranscriptionJobFromAWS = async (transcriptionJobName) => { + const command = new DeleteTranscriptionJobCommand({ + TranscriptionJobName: transcriptionJobName, + }); + + const deletedTranscriptionJobs = await transcribeClient.send(command); + // returns an array + + return deletedTranscriptionJobs; +}; From 699cda8fed3767a7ca16f6ec8fda73489b0ee234 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:02:21 +0100 Subject: [PATCH 13/49] REFACTOR analysisEntry controller to favor internal functions vs queues --- server/controllers/analysisEntryController.js | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index 12b68ed..e20d9c2 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -1,7 +1,6 @@ -import { logError } from '../config/loggerFunctions.js'; -import { publishToTranscriptionRequestedQueue } from '../config/messageBroker/LavinMQ.js'; import { generateS3GetPresignedUrl, generateS3PutPresignedUrl } from '../integrations/aws/s3.js'; -import { createAnalysisEntryInDb, getAnalysisEntryDetailsById, updateAnalysisEntryInDb } from '../models/analysisEntryModel.js'; +import { createAnalysisEntryInDb, getAnalysisEntryDetailsById, markAnalysisEntryAsSubmitted } from '../models/analysisEntryModel.js'; +import { processTranscriptionRequest } from '../services/analysisService.js'; export const createAnalysisEntry = async (req, res) => { const { analysisId } = req.body; @@ -16,25 +15,18 @@ export const createAnalysisEntry = async (req, res) => { }; export const updateAnalysisEntry = async (req, res) => { - const { analysisEntryId } = req.body; + const { analysisEntryId, analysisId } = req.body; await markAnalysisEntryAsSubmitted(analysisEntryId); - try { - const message = { - analysisEntryId: analysisEntryId, - analysisId: analysisId, - timestamp: new Date().toISOString(), - mediaType: 'video', - languageCode: 'es-ES', - }; - - const stringifiedMessage = JSON.stringify(message); + const transcriptionRequest = { + analysisEntryId: analysisEntryId, + analysisId: analysisId, + mediaType: 'video', + languageCode: 'es-ES', + }; - await publishToTranscriptionRequestedQueue(stringifiedMessage); - } catch (error) { - logError(`Error sending transcription request analysisEntry ${analysisEntryId}`, error); - } + processTranscriptionRequest(transcriptionRequest); // Fire-and-forget return res.status(200).json({ success: true, From fc8df483d706a74eec5e8a3565c7877ea75739b1 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:04:36 +0100 Subject: [PATCH 14/49] UPDATE swagger & REMOVE "success" attribute from PATCH analysisEntry --- server/controllers/analysisEntryController.js | 1 - swagger.yaml | 5 ----- 2 files changed, 6 deletions(-) diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index e20d9c2..f74a4e7 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -29,7 +29,6 @@ export const updateAnalysisEntry = async (req, res) => { processTranscriptionRequest(transcriptionRequest); // Fire-and-forget return res.status(200).json({ - success: true, message: 'Analysis entry updated successfully', }); }; diff --git a/swagger.yaml b/swagger.yaml index c76d21a..0be1519 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1489,13 +1489,10 @@ paths: type: object required: - analysisId - - status - analysisEntryId properties: analysisEntryId: $ref: "#/components/schemas/AnalysisEntryId" - analysisEntryStatus: - $ref: "#/components/schemas/AnalysisEntryCompletionStatus" analysisId: $ref: "#/components/schemas/AnalysisId" responses: @@ -1506,8 +1503,6 @@ paths: schema: type: object properties: - success: - $ref: "#/components/schemas/SuccessTrueStatus" message: type: string example: Analysis entry updated successfully From a8101ef506dff53a2b1b031c2bc96b96e9349ced Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 13 Dec 2025 01:16:00 +0100 Subject: [PATCH 15/49] ADD logic to update the transcriptJob after successful AWS transcribe response --- server/models/transcriptionModel.js | 23 +++++++++-------------- server/services/analysisService.js | 2 ++ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index dd96efa..db8b71a 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -13,22 +13,17 @@ export const insertTranscriptionRequestInDb = async (transcriptionRequest) => { }); }; -export const updateSingleTranscriptionRequestInDb = async (transcriptionRequestInsertId) => { - const db = await connectToMongoDB(); - - const transcriptionRequests = db.collection('transcriptionRequests'); +export const updateSingleTranscriptionRequestInDb = async (analysisEntryId) => { + const whereClause = { + analysis_entry_id: analysisEntryId, + }; - const updateResult = await transcriptionRequests.updateOne( - { _id: transcriptionRequestInsertId }, - { - $set: { - status: 'IN_PROGRESS', - updatedAt: new Date(), - }, + await prisma.transcriptionJob.update({ + where: whereClause, + data: { + status: 'IN_PROGRESS', }, - ); - - return updateResult; + }); }; export const getCompletedTranscriptions = async () => { diff --git a/server/services/analysisService.js b/server/services/analysisService.js index a65a731..b55bd3b 100644 --- a/server/services/analysisService.js +++ b/server/services/analysisService.js @@ -17,6 +17,8 @@ export const processTranscriptionRequest = async (transcriptionRequest) => { await insertTranscriptionRequestInDb(transcriptionRequest); await requestAnalysisEntryTranscriptionToAWSTranscribe(transcriptionRequest); + + await updateSingleTranscriptionRequestInDb(transcriptionRequest.analysisEntryId); } catch (error) { logError('Error processing transcription request', error); } From 42c08add305368f59352daf9758b1565422d6a4f Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 20 Dec 2025 18:52:59 +0100 Subject: [PATCH 16/49] REMOVE queue-related transcription model functions They were used in the microservice - no longer used in the monolith functions: - mark as published to queue - get completed & not published to queue --- server/models/transcriptionModel.js | 31 ----------------------------- 1 file changed, 31 deletions(-) diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index db8b71a..5282f6b 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -74,34 +74,3 @@ export const storeNormalizedTranscriptionInDb = async (transcriptionJobInsertId, return updateResult; }; -export const markTranscriptionAsPublishedToQueue = async (transcriptionJobInsertId) => { - const db = await connectToMongoDB(); - - const transcriptionsCollection = db.collection('transcriptionRequests'); - - const updateResult = await transcriptionsCollection.updateOne( - { _id: transcriptionJobInsertId }, - { - $set: { - // publishedToQueue: false, // * DEBUG - publishedToQueue: true, - updatedAt: new Date(), - }, - }, - ); - - return updateResult; -}; - -export const getTranscriptionJobsInCompletedStatusNotPublishedToQueue = async () => { - const db = await connectToMongoDB(); - - const transcriptionsCollection = db.collection('transcriptionRequests'); - - const transcriptionJobs = await transcriptionsCollection.find({ - status: 'COMPLETED', - publishedToQueue: false, - }).toArray(); - - return transcriptionJobs; -}; From 6e9b64603ac77526e2a472d2df24cb573dea1b07 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 20 Dec 2025 18:58:06 +0100 Subject: [PATCH 17/49] REFACTOR transcription model from microservice to monolith --- server/models/transcriptionModel.js | 65 +++++++++++++---------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index 5282f6b..41946f8 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -26,51 +26,42 @@ export const updateSingleTranscriptionRequestInDb = async (analysisEntryId) => { }); }; -export const getCompletedTranscriptions = async () => { - const db = await connectToMongoDB(); - - const transcriptionsCollection = db.collection('transcriptionRequests'); - - const completedTranscriptions = await transcriptionsCollection.find({ - publishedToQueue: false, - status: 'COMPLETED', - }).toArray(); - - return completedTranscriptions; -}; - -export const getSingleTranscriptionJobDetailsFromDb = async (transcriptionJobDetails) => { - const db = await connectToMongoDB(); - - const transcriptionsCollection = db.collection('transcriptionRequests'); +export const getSingleTranscriptionJobDetailsFromDb = async (transcriptionJobName) => { + const whereClause = { + id: transcriptionJobName, + }; - const transcriptionJobDetailsResult = await transcriptionsCollection.findOne( - { _id: transcriptionJobDetails }, - ); + const transcriptionJobDetailsResult = await prisma.analysisEntry.findUnique({ + where: whereClause, + select: { + analysis_id: true, + transcriptionJob: { + select: { + status: true, + }, + }, + }, + }); return transcriptionJobDetailsResult; }; -export const storeNormalizedTranscriptionInDb = async (transcriptionJobInsertId, normalizedTranscriptionJob, transcriptionJobResult) => { - const db = await connectToMongoDB(); - - const transcriptionsCollection = db.collection('transcriptionRequests'); +export const storeNormalizedTranscriptionInDb = async (transcriptionJobName, normalizedTranscriptionJob, transcriptionJobResult) => { + const whereClause = { + analysis_entry_id: transcriptionJobName, + }; - const updateResult = await transcriptionsCollection.updateOne( - { _id: transcriptionJobInsertId }, - { - $set: { - status: 'COMPLETED', - transcriptionData: { - fullTranscript: transcriptionJobResult.results.transcripts[0].transcript, - segments: normalizedTranscriptionJob.results.segments, + await prisma.analysisEntry.update({ + where: whereClause, + data: { + full_transcript: transcriptionJobResult.results.transcripts[0].transcript, + transcription_segments: normalizedTranscriptionJob.results.segments, + transcriptionJob: { + update: { + status: 'COMPLETED', }, - updatedAt: new Date(), - publishedToQueue: false, }, }, - ); - - return updateResult; + }); }; From 31725c57902aa4f00803a47a59ec85fab2b1bc38 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 20 Dec 2025 18:58:49 +0100 Subject: [PATCH 18/49] ADD no-plusplus rule exception to eslint --- .eslintrc.cjs | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 8967536..c81ea04 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -44,5 +44,6 @@ module.exports = { 'import/no-relative-packages': 'off', 'no-case-declarations': 'off', 'max-len': 'off', + 'no-plusplus': 'off', }, }; From b618c7565182cca54423a016ea5a0b8d2a18b6c2 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sat, 20 Dec 2025 21:02:25 +0100 Subject: [PATCH 19/49] ADD no-await-in-loop rule exception in eslint --- .eslintrc.cjs | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index c81ea04..7782acb 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -45,5 +45,6 @@ module.exports = { 'no-case-declarations': 'off', 'max-len': 'off', 'no-plusplus': 'off', + 'no-await-in-loop': 'off', }, }; From fb3ce07e9df9f0b8facf64a63a71674ec203ceb2 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 21 Dec 2025 20:16:41 +0100 Subject: [PATCH 20/49] REMOVE lavinMQ implementation --- .env.example | 2 - .kilocode/rules/memory-bank/tech.md | 3 - compose.yaml | 1 - package-lock.json | 10 -- package.json | 1 - server/app.js | 2 - server/config/messageBroker/LavinMQ.js | 131 ------------------------- 7 files changed, 150 deletions(-) delete mode 100644 server/config/messageBroker/LavinMQ.js diff --git a/.env.example b/.env.example index 9ec90ca..01f5059 100644 --- a/.env.example +++ b/.env.example @@ -23,5 +23,3 @@ AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= AWS_REGION= AWS_BUCKET= - -LAVINMQ_HOST= \ No newline at end of file diff --git a/.kilocode/rules/memory-bank/tech.md b/.kilocode/rules/memory-bank/tech.md index 62bca33..60bdc49 100644 --- a/.kilocode/rules/memory-bank/tech.md +++ b/.kilocode/rules/memory-bank/tech.md @@ -107,9 +107,6 @@ TELEGRAM_BOT_API_KEY=your-bot-token # Frontend FRONT_WEB_APP_ORIGIN_URL=http://localhost:5173 - -# Message Queue -LAVINMQ_HOST=localhost ``` ### Local Development Commands diff --git a/compose.yaml b/compose.yaml index b310e0a..a2c8073 100644 --- a/compose.yaml +++ b/compose.yaml @@ -22,7 +22,6 @@ services: AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_BUCKET: dev-analysis-entry-storage AWS_REGION: ${AWS_REGION} - LAVINMQ_HOST: ${LAVINMQ_HOST} ports: - 3000:3000 diff --git a/package-lock.json b/package-lock.json index 239b0f8..ab8f1da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@aws-sdk/client-s3": "^3.936.0", "@aws-sdk/client-transcribe": "^3.948.0", "@aws-sdk/s3-request-presigner": "^3.936.0", - "@cloudamqp/amqp-client": "^3.4.0", "@getbrevo/brevo": "^3.0.1", "@prisma/client": "^6.19.0", "@quixo3/prisma-session-store": "^3.1.13", @@ -1870,15 +1869,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@cloudamqp/amqp-client": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/@cloudamqp/amqp-client/-/amqp-client-3.4.1.tgz", - "integrity": "sha512-A53N3dpS4zEaVL1GMIDg/A4hTy/t/nqVDNaY6m+48VKi2N1cpGkmTmc6RHMAD06UTV6Z0O8CeGqs7076w6VFMQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@emnapi/core": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", diff --git a/package.json b/package.json index a14a165..b89d886 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "@aws-sdk/client-s3": "^3.936.0", "@aws-sdk/client-transcribe": "^3.948.0", "@aws-sdk/s3-request-presigner": "^3.936.0", - "@cloudamqp/amqp-client": "^3.4.0", "@getbrevo/brevo": "^3.0.1", "@prisma/client": "^6.19.0", "@quixo3/prisma-session-store": "^3.1.13", diff --git a/server/app.js b/server/app.js index 9be7c7d..b91eed7 100644 --- a/server/app.js +++ b/server/app.js @@ -12,7 +12,6 @@ import { slowLimiter } from './middlewares/express-slow-down.js'; import { startCronJobs } from './cron/jobsContainer.js'; import { globalErrorHandler } from './middlewares/globalErrorHandler.js'; import { webhookRouter } from './webhooks/webhooksRouter.js'; -import { connectToMessageBroker } from './config/messageBroker/LavinMQ.js'; const app = express(); const server = createServer(app); @@ -50,7 +49,6 @@ server.listen(process.env.PORT, () => { console.log(`Server running at http://localhost:${process.env.PORT}/`); }); -connectToMessageBroker(); startCronJobs(); const gracefulShutdown = () => { diff --git a/server/config/messageBroker/LavinMQ.js b/server/config/messageBroker/LavinMQ.js deleted file mode 100644 index 47a4d69..0000000 --- a/server/config/messageBroker/LavinMQ.js +++ /dev/null @@ -1,131 +0,0 @@ -import { AMQPClient } from '@cloudamqp/amqp-client'; -import { logError, logInfo } from '../loggerFunctions.js'; -import { insertAnalysisEntryTranscriptionInDb } from '../../models/analysisEntryModel.js'; - -let connection; -let channel; -let transcriptionRequestedQueue; -let insightsRequestedQueue; - -// Main AMQP setup function -export const connectToMessageBroker = async () => { - try { - // 1. Establish connection - const amqp = new AMQPClient(process.env.LAVINMQ_HOST); - connection = await amqp.connect(); // Establish connection to the message broker - one connection for all channels - - // 2. Open producer and consumer channels - channel = await connection.channel(); // One channel for producing & consuming messages - - // 3. Declare exchanges & queues & bindings - - // 3.1 Declare exchanges - - const analysisExchange = await channel.exchangeDeclare('analysis_exchange', 'topic', { - durable: true, - passive: false, - autoDelete: false, - internal: false, - }); - - // 3.2 Declare queues - transcriptionRequestedQueue = await channel.queue('transcription_requested_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - const transcriptionCompletedQueue = await channel.queue('transcription_completed_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - insightsRequestedQueue = await channel.queue('insights_requested_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - const insightsCompletedQueue = await channel.queue('insights_completed_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - // 3.3 Bind queues to exchange with routing keys - - await transcriptionRequestedQueue.bind('analysis_exchange', 'analysis.analysisEntry.transcription.requested', { - }); - - await insightsRequestedQueue.bind('analysis_exchange', 'analysis.analysisEntry.insights.requested', { - }); - - await insightsCompletedQueue.bind('analysis_exchange', 'analysis.analysisEntry.insights.completed', { - }); - - // Start consumers after successful connection - - await transcriptionCompletedQueue.subscribe({ noAck: false }, async (msg) => { - try { - const contentStr = msg.bodyToString(msg); - const completedTranscriptionRequest = JSON.parse(contentStr); - - await insertAnalysisEntryTranscriptionInDb(completedTranscriptionRequest); - - /* - const exampleReceivedMessage = { - analysisEntryId: transcriptionJobDetails._id, - transcriptionSegments: array of objects, each representing a segment of the transcription - fullTranscript: 'lorem ipsum dolor sit amet, consectetur adipiscing elit' - }; - */ - - await msg.ack(); - } catch (error) { - logError('Error processing transcription completed message', error); - await msg.nack(true); // Requeue on failure - } - }); - - await insightsCompletedQueue.subscribe({ noAck: false }, async (msg) => { - try { - await msg.ack(); - } catch (error) { - await msg.nack(true); // Requeue on failure - } - }); - - logInfo('monolith successfully connected to LavinMQ message broker'); - - return { - connection: connection, channel: channel, analysisExchange, transcriptionQueue: transcriptionRequestedQueue, - }; - } catch (e) { - logError('error connecting to message broker', e); - e.connection?.close(); - return setTimeout(connectToMessageBroker, 1000); // will try to reconnect in 1s - } -}; - -// function for publishing to transcription queue - -export const publishToTranscriptionRequestedQueue = async (message) => { - try { - return await transcriptionRequestedQueue.publish(message); - } catch (err) { - return logError('error publishing transcription request', err); - } -}; - -export const publishToInsightsRequestedQueue = async (message) => { - try { - return await insightsRequestedQueue.publish(message); - } catch (err) { - return logError('error publishing transcription request', err); - } -}; From 6a7666d2bba446df9ee224514cb083132f777c32 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 21 Dec 2025 20:17:18 +0100 Subject: [PATCH 21/49] ADD error logging to completedTranscriptionJobScheduler CRON job --- server/cron/getCompletedTranscriptionJobsScheduler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/cron/getCompletedTranscriptionJobsScheduler.js b/server/cron/getCompletedTranscriptionJobsScheduler.js index fef45bc..c176939 100644 --- a/server/cron/getCompletedTranscriptionJobsScheduler.js +++ b/server/cron/getCompletedTranscriptionJobsScheduler.js @@ -1,10 +1,11 @@ import { CronJob } from 'cron'; import { handleCompletedVideoTranscriptionJobs } from '../services/analysisService.js'; +import { logError } from '../config/loggerFunctions.js'; export const getCompletedTranscriptionJobsScheduler = new CronJob('15 * * * *', async () => { try { await handleCompletedVideoTranscriptionJobs(); } catch (error) { - console.error('Error checking transcription job status:', error); + logError('Error checking transcription job status', error); } }); From 5bb51e6ed7e1871d0a808eda8de656e669deb3cc Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 21 Dec 2025 20:18:06 +0100 Subject: [PATCH 22/49] FIX typo in transcription job creation function --- server/models/transcriptionModel.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index 41946f8..af7aa1b 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -3,11 +3,10 @@ import { PrismaClient } from '../config/generated/prisma/client/index.js'; const prisma = new PrismaClient(); export const insertTranscriptionRequestInDb = async (transcriptionRequest) => { - await prisma.transcriptionRequest.create({ + await prisma.transcriptionJob.create({ data: { analysis_entry_id: transcriptionRequest.analysisEntryId, status: 'pending', - type: transcriptionRequest.mediaType, language_code: transcriptionRequest.languageCode, }, }); @@ -64,4 +63,3 @@ export const storeNormalizedTranscriptionInDb = async (transcriptionJobName, nor }, }); }; - From 3d5aa703cf9b83e6c3b87de046b46d3b046cf494 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 21 Dec 2025 20:19:28 +0100 Subject: [PATCH 23/49] REMOVE mediaType attribute & remove requirement for analysisId --- server/controllers/analysisEntryController.js | 7 +++---- server/models/analysisEntryModel.js | 2 +- swagger.yaml | 3 --- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index f74a4e7..9fc8c40 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -15,14 +15,13 @@ export const createAnalysisEntry = async (req, res) => { }; export const updateAnalysisEntry = async (req, res) => { - const { analysisEntryId, analysisId } = req.body; + const { analysisEntryId } = req.body; - await markAnalysisEntryAsSubmitted(analysisEntryId); + const updatedAnalysisEntry = await markAnalysisEntryAsSubmitted(analysisEntryId); const transcriptionRequest = { analysisEntryId: analysisEntryId, - analysisId: analysisId, - mediaType: 'video', + analysisId: updatedAnalysisEntry.analysis_id, languageCode: 'es-ES', }; diff --git a/server/models/analysisEntryModel.js b/server/models/analysisEntryModel.js index b353bcd..9ccb29b 100644 --- a/server/models/analysisEntryModel.js +++ b/server/models/analysisEntryModel.js @@ -54,7 +54,7 @@ export const markAnalysisEntryAsSubmitted = async (analysisEntryId) => { status: 'submitted', }, select: { - status: true, + analysis_id: true, }, }); diff --git a/swagger.yaml b/swagger.yaml index 0be1519..bc56fb1 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1488,13 +1488,10 @@ paths: schema: type: object required: - - analysisId - analysisEntryId properties: analysisEntryId: $ref: "#/components/schemas/AnalysisEntryId" - analysisId: - $ref: "#/components/schemas/AnalysisId" responses: 200: description: Success From 610fa72423518bada161c6bc788772694b93cbb1 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 11:38:00 +0100 Subject: [PATCH 24/49] ADD logging to transcription request job --- server/integrations/aws/Transcribe.js | 2 +- server/models/transcriptionModel.js | 2 +- server/services/analysisService.js | 15 ++++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/server/integrations/aws/Transcribe.js b/server/integrations/aws/Transcribe.js index 3f5aa0f..9ab433b 100644 --- a/server/integrations/aws/Transcribe.js +++ b/server/integrations/aws/Transcribe.js @@ -36,7 +36,7 @@ export const listCompletedTranscriptionJobsFromAWS = async () => { }; export const fetchSingleTranscriptionJob = async (analysisId, analysisEntryId) => { - const key = process.env.DEPLOY_ENVIRONMENT === 'localhost' ? 'analysis/464d4419-3822-41e6-8d8e-27b1783632df/eabd3179-ece0-4163-8c7e-0e2c728e11e6/transcription.json' : `analysis/${analysisId}/${analysisEntryId}/transcription.json`; + const key = `analysis/${analysisId}/${analysisEntryId}/transcription.json`; const transcriptionJobResult = await getS3Object(key); diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index af7aa1b..c6d9f44 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -6,7 +6,7 @@ export const insertTranscriptionRequestInDb = async (transcriptionRequest) => { await prisma.transcriptionJob.create({ data: { analysis_entry_id: transcriptionRequest.analysisEntryId, - status: 'pending', + status: 'PENDING', language_code: transcriptionRequest.languageCode, }, }); diff --git a/server/services/analysisService.js b/server/services/analysisService.js index b55bd3b..baff3ab 100644 --- a/server/services/analysisService.js +++ b/server/services/analysisService.js @@ -1,4 +1,4 @@ -import { logError } from '../config/loggerFunctions.js'; +import { logError, logInfo } from '../config/loggerFunctions.js'; import { insertTranscriptionRequestInDb, updateSingleTranscriptionRequestInDb, @@ -15,10 +15,19 @@ import { normalizeTranscript } from '../utils/transcription/transcriptionNormali export const processTranscriptionRequest = async (transcriptionRequest) => { try { await insertTranscriptionRequestInDb(transcriptionRequest); + logInfo('Transcription request stored in DB', transcriptionRequest); - await requestAnalysisEntryTranscriptionToAWSTranscribe(transcriptionRequest); + try { // Handle errors gracefully - errors will be picked up by a cron job if failed + await requestAnalysisEntryTranscriptionToAWSTranscribe(transcriptionRequest); + logInfo('Transcription request sent to AWS Transcribe', transcriptionRequest); - await updateSingleTranscriptionRequestInDb(transcriptionRequest.analysisEntryId); + await updateSingleTranscriptionRequestInDb(transcriptionRequest.analysisEntryId); + logInfo('Transcription request updated in DB', transcriptionRequest); + } catch (error) { + logError('Error requesting transcription to AWS Transcribe', error); + } + } catch (error) { + logError('Error storing transcription request in DB', error); } catch (error) { logError('Error processing transcription request', error); } From 8b75d68467576f7ec3a41c8765d474be278c3cc2 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 12:01:18 +0100 Subject: [PATCH 25/49] ADD logic to handle completed transcription jobs --- server/services/analysisService.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/server/services/analysisService.js b/server/services/analysisService.js index baff3ab..9672132 100644 --- a/server/services/analysisService.js +++ b/server/services/analysisService.js @@ -28,7 +28,26 @@ export const processTranscriptionRequest = async (transcriptionRequest) => { } } catch (error) { logError('Error storing transcription request in DB', error); + } +}; + +const processSingleCompletedTranscriptionJob = async (transcriptionJob) => { +}; + +export const handleCompletedVideoTranscriptionJobs = async () => { + try { + // Get the completed Jobs from AWS Transcribe + logInfo('Retrieving completed transcription jobs'); + const completedTranscriptionJobsSummary = await listCompletedTranscriptionJobsFromAWS(); + + if (completedTranscriptionJobsSummary.length === 0) { + logInfo('no completed transcription jobs available to process'); + } + + for (const transcriptionJob of completedTranscriptionJobsSummary) { + await processSingleCompletedTranscriptionJob(transcriptionJob); + } } catch (error) { - logError('Error processing transcription request', error); + logError('Error processing transcription jobs', error); } }; From 765db4ba4fecdbfffb83c17b4c5e488f61b2738e Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 13:28:18 +0100 Subject: [PATCH 26/49] ADD no-restricted-syntax rule to eslint exceptions --- .eslintrc.cjs | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 7782acb..b0a4d2d 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -46,5 +46,6 @@ module.exports = { 'max-len': 'off', 'no-plusplus': 'off', 'no-await-in-loop': 'off', + 'no-restricted-syntax': 'off', }, }; From aedb52e072ab276bb740555d8f0565fc7aea8675 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 14:08:03 +0100 Subject: [PATCH 27/49] COMPLETE logic for requesting & processing transcription jobs --- server/app.js | 2 +- .../getCompletedTranscriptionJobsScheduler.js | 3 +- server/integrations/aws/Transcribe.js | 5 ++- server/models/transcriptionModel.js | 2 +- server/services/analysisService.js | 39 ++++++++++++++++++- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/server/app.js b/server/app.js index b91eed7..c0f0065 100644 --- a/server/app.js +++ b/server/app.js @@ -46,7 +46,7 @@ app.use(globalErrorHandler); //* Start the server server.listen(process.env.PORT, () => { // eslint-disable-next-line no-console - console.log(`Server running at http://localhost:${process.env.PORT}/`); + console.log('Server running'); }); startCronJobs(); diff --git a/server/cron/getCompletedTranscriptionJobsScheduler.js b/server/cron/getCompletedTranscriptionJobsScheduler.js index c176939..36a2f44 100644 --- a/server/cron/getCompletedTranscriptionJobsScheduler.js +++ b/server/cron/getCompletedTranscriptionJobsScheduler.js @@ -1,9 +1,10 @@ import { CronJob } from 'cron'; import { handleCompletedVideoTranscriptionJobs } from '../services/analysisService.js'; -import { logError } from '../config/loggerFunctions.js'; +import { logError, logInfo } from '../config/loggerFunctions.js'; export const getCompletedTranscriptionJobsScheduler = new CronJob('15 * * * *', async () => { try { + logInfo('Checking transcription job status'); await handleCompletedVideoTranscriptionJobs(); } catch (error) { logError('Error checking transcription job status', error); diff --git a/server/integrations/aws/Transcribe.js b/server/integrations/aws/Transcribe.js index 9ab433b..0404b06 100644 --- a/server/integrations/aws/Transcribe.js +++ b/server/integrations/aws/Transcribe.js @@ -5,6 +5,7 @@ import { DeleteTranscriptionJobCommand, } from '@aws-sdk/client-transcribe'; import { getS3Object } from './s3.js'; +import { logInfo } from '../../config/loggerFunctions.js'; const transcribeClient = new TranscribeClient({ region: process.env.AWS_REGION }); @@ -25,8 +26,7 @@ export const requestAnalysisEntryTranscriptionToAWSTranscribe = async (transcrip export const listCompletedTranscriptionJobsFromAWS = async () => { const command = new ListTranscriptionJobsCommand({ Status: 'COMPLETED', - MaxResults: 100, - + MaxResults: 10, // Ensure memory is not hogged - if more ara available, they will be processed in the next iteration }); const completedTranscriptionJobs = await transcribeClient.send(command); @@ -44,6 +44,7 @@ export const fetchSingleTranscriptionJob = async (analysisId, analysisEntryId) = }; export const deleteCompletedTranscriptionJobFromAWS = async (transcriptionJobName) => { + logInfo(`deleting ${transcriptionJobName} from AWS Transcribe`); const command = new DeleteTranscriptionJobCommand({ TranscriptionJobName: transcriptionJobName, }); diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index c6d9f44..48ac846 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -47,7 +47,7 @@ export const getSingleTranscriptionJobDetailsFromDb = async (transcriptionJobNam export const storeNormalizedTranscriptionInDb = async (transcriptionJobName, normalizedTranscriptionJob, transcriptionJobResult) => { const whereClause = { - analysis_entry_id: transcriptionJobName, + id: transcriptionJobName, }; await prisma.analysisEntry.update({ diff --git a/server/services/analysisService.js b/server/services/analysisService.js index 9672132..176cfa3 100644 --- a/server/services/analysisService.js +++ b/server/services/analysisService.js @@ -4,11 +4,11 @@ import { updateSingleTranscriptionRequestInDb, getSingleTranscriptionJobDetailsFromDb, storeNormalizedTranscriptionInDb, - markTranscriptionAsPublishedToQueue, } from '../models/transcriptionModel.js'; import { requestAnalysisEntryTranscriptionToAWSTranscribe, deleteCompletedTranscriptionJobFromAWS, fetchSingleTranscriptionJob, + listCompletedTranscriptionJobsFromAWS, } from '../integrations/aws/Transcribe.js'; import { normalizeTranscript } from '../utils/transcription/transcriptionNormalizer.js'; @@ -32,6 +32,42 @@ export const processTranscriptionRequest = async (transcriptionRequest) => { }; const processSingleCompletedTranscriptionJob = async (transcriptionJob) => { + logInfo(`Processing completed transcription job: ${transcriptionJob.TranscriptionJobName}`); + + try { + // 1. Get transcription job details from database + const transcriptionJobDetails = await getSingleTranscriptionJobDetailsFromDb(transcriptionJob.TranscriptionJobName); + + // Delete from AWS Transcribe if already processed - Shouldnt happen if AWS Transcribe job deletion is working properly + + if (transcriptionJobDetails.transcriptionJob.status === 'COMPLETED') { // Handle duplicate entries to avoid normalization reprocessing + logInfo(`Deleting already processed job: ${transcriptionJob.TranscriptionJobName}`); + await deleteCompletedTranscriptionJobFromAWS(transcriptionJob.TranscriptionJobName); + } + + // 2. Construct S3 key and fetch transcription file from AWS + const transcriptionJobResultString = await fetchSingleTranscriptionJob(transcriptionJobDetails.analysis_id, transcriptionJob.TranscriptionJobName); + + // 3. Parse the transcription job result (JSON string to object) + const transcriptionJobResult = JSON.parse(transcriptionJobResultString); + + // 4. Normalize transcription job result + const normalizedTranscriptionJob = await normalizeTranscript(transcriptionJobResult); + + // 5. Store normalized transcript in DB and update status to COMPLETED + await storeNormalizedTranscriptionInDb(transcriptionJob.TranscriptionJobName, normalizedTranscriptionJob, transcriptionJobResult); + + try { + await deleteCompletedTranscriptionJobFromAWS(transcriptionJob.TranscriptionJobName); + } catch (error) { + logError(`Error deleting transcription job ${transcriptionJob.TranscriptionJobName} from AWS Transcribe`, error); + // AWS Transcribe deletion failing is not an issue since it will be caught by a CRON-based retry mechanism + } + + logInfo(`Successfully processed transcription job: ${transcriptionJob.TranscriptionJobName}`); + } catch (error) { + logError(`Error processing transcription job ${transcriptionJob.TranscriptionJobName}`, error); + } }; export const handleCompletedVideoTranscriptionJobs = async () => { @@ -42,6 +78,7 @@ export const handleCompletedVideoTranscriptionJobs = async () => { if (completedTranscriptionJobsSummary.length === 0) { logInfo('no completed transcription jobs available to process'); + return; } for (const transcriptionJob of completedTranscriptionJobsSummary) { From 8bf9d8488c86dd617aad7a2bead50027971a9ea1 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 14:12:11 +0100 Subject: [PATCH 28/49] UPDATE db to include transcriptionJob table --- .../migration.sql | 23 +++++++++++++++++++ prisma/schema.prisma | 8 +++---- 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 prisma/migrations/20251222131114_add_transcription_job_table/migration.sql diff --git a/prisma/migrations/20251222131114_add_transcription_job_table/migration.sql b/prisma/migrations/20251222131114_add_transcription_job_table/migration.sql new file mode 100644 index 0000000..b9a5f8a --- /dev/null +++ b/prisma/migrations/20251222131114_add_transcription_job_table/migration.sql @@ -0,0 +1,23 @@ +-- CreateEnum +CREATE TYPE "TranscriptionJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED'); + +-- CreateTable +CREATE TABLE "TranscriptionJob" ( + "id" SERIAL NOT NULL, + "analysis_entry_id" TEXT NOT NULL, + "status" "TranscriptionJobStatus" NOT NULL DEFAULT 'PENDING', + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "language_code" TEXT NOT NULL, + + CONSTRAINT "TranscriptionJob_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "TranscriptionJob_id_key" ON "TranscriptionJob"("id"); + +-- CreateIndex +CREATE UNIQUE INDEX "TranscriptionJob_analysis_entry_id_key" ON "TranscriptionJob"("analysis_entry_id"); + +-- AddForeignKey +ALTER TABLE "TranscriptionJob" ADD CONSTRAINT "TranscriptionJob_analysis_entry_id_fkey" FOREIGN KEY ("analysis_entry_id") REFERENCES "AnalysisEntry"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8ef0502..3e347bd 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -134,16 +134,16 @@ model TranscriptionJob { id Int @id @unique @default(autoincrement()) AnalysisEntry AnalysisEntry @relation(fields: [analysis_entry_id], references: [id]) analysis_entry_id String @unique - status TranscriptionJobStatus @default(pending) + status TranscriptionJobStatus @default(PENDING) created_at DateTime @default(now()) updated_at DateTime @default(now()) @updatedAt language_code String } enum TranscriptionJobStatus { - pending - processing - completed + PENDING + IN_PROGRESS + COMPLETED } enum Genders { From fdd9ef2468ebacadd28ab2bf00f037687d93c09a Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 15:12:47 +0100 Subject: [PATCH 29/49] FIX typo in Transcribe.js --- server/integrations/aws/Transcribe.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/integrations/aws/Transcribe.js b/server/integrations/aws/Transcribe.js index 0404b06..fe225ea 100644 --- a/server/integrations/aws/Transcribe.js +++ b/server/integrations/aws/Transcribe.js @@ -26,7 +26,7 @@ export const requestAnalysisEntryTranscriptionToAWSTranscribe = async (transcrip export const listCompletedTranscriptionJobsFromAWS = async () => { const command = new ListTranscriptionJobsCommand({ Status: 'COMPLETED', - MaxResults: 10, // Ensure memory is not hogged - if more ara available, they will be processed in the next iteration + MaxResults: 10, // Ensure memory is not hogged - if more are available, they will be processed in the next iteration }); const completedTranscriptionJobs = await transcribeClient.send(command); From e1030ca70b8be2267a91c47bbbce5e776d12e974 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 15:13:08 +0100 Subject: [PATCH 30/49] ADD proper error logging to transcription function --- server/utils/transcription/transcriptionNormalizer.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/utils/transcription/transcriptionNormalizer.js b/server/utils/transcription/transcriptionNormalizer.js index 73862e5..500fd11 100644 --- a/server/utils/transcription/transcriptionNormalizer.js +++ b/server/utils/transcription/transcriptionNormalizer.js @@ -1,3 +1,5 @@ +import { logError } from '../../config/loggerFunctions.js'; + /** * Converts string numbers to actual numbers * @param {string|number} value - Value to convert @@ -170,7 +172,7 @@ export const normalizeTranscript = async (transcript) => { }, }; } catch (error) { - console.error('Error normalizing transcript:', error); + logError(`Error normalizing transcript: ${error.message}`, error); return { status: 'FAILED', results: { From 0e5288767774d99e424dd4c5d32b90e1ab8d21f8 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 15:35:16 +0100 Subject: [PATCH 31/49] REFACTOR transcription job retrieval logic for more clarity --- server/models/transcriptionModel.js | 10 +++++----- server/services/analysisService.js | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index 48ac846..e87cf76 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -27,16 +27,16 @@ export const updateSingleTranscriptionRequestInDb = async (analysisEntryId) => { export const getSingleTranscriptionJobDetailsFromDb = async (transcriptionJobName) => { const whereClause = { - id: transcriptionJobName, + analysis_entry_id: transcriptionJobName, }; - const transcriptionJobDetailsResult = await prisma.analysisEntry.findUnique({ + const transcriptionJobDetailsResult = await prisma.transcriptionJob.findUnique({ where: whereClause, select: { - analysis_id: true, - transcriptionJob: { + status: true, + AnalysisEntry: { select: { - status: true, + analysis_id: true, }, }, }, diff --git a/server/services/analysisService.js b/server/services/analysisService.js index 176cfa3..4ce2f68 100644 --- a/server/services/analysisService.js +++ b/server/services/analysisService.js @@ -40,13 +40,13 @@ const processSingleCompletedTranscriptionJob = async (transcriptionJob) => { // Delete from AWS Transcribe if already processed - Shouldnt happen if AWS Transcribe job deletion is working properly - if (transcriptionJobDetails.transcriptionJob.status === 'COMPLETED') { // Handle duplicate entries to avoid normalization reprocessing + if (transcriptionJobDetails.status === 'COMPLETED') { // Handle duplicate entries to avoid normalization reprocessing logInfo(`Deleting already processed job: ${transcriptionJob.TranscriptionJobName}`); await deleteCompletedTranscriptionJobFromAWS(transcriptionJob.TranscriptionJobName); } // 2. Construct S3 key and fetch transcription file from AWS - const transcriptionJobResultString = await fetchSingleTranscriptionJob(transcriptionJobDetails.analysis_id, transcriptionJob.TranscriptionJobName); + const transcriptionJobResultString = await fetchSingleTranscriptionJob(transcriptionJobDetails.AnalysisEntry.analysis_id, transcriptionJob.TranscriptionJobName); // 3. Parse the transcription job result (JSON string to object) const transcriptionJobResult = JSON.parse(transcriptionJobResultString); From db31a5c4a42cc609204c1f982d33edd56b9fb457 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 22 Dec 2025 15:39:28 +0100 Subject: [PATCH 32/49] ADD prisma schema formatting --- prisma/schema.prisma | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3e347bd..c75a00b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -125,19 +125,19 @@ model AnalysisEntry { updated_at DateTime @default(now()) @updatedAt transcription_segments Json? full_transcript String? - transcriptionJob TranscriptionJob? + transcriptionJob TranscriptionJob? @@index([analysis_id, status]) } model TranscriptionJob { - id Int @id @unique @default(autoincrement()) + id Int @id @unique @default(autoincrement()) AnalysisEntry AnalysisEntry @relation(fields: [analysis_entry_id], references: [id]) analysis_entry_id String @unique status TranscriptionJobStatus @default(PENDING) created_at DateTime @default(now()) updated_at DateTime @default(now()) @updatedAt - language_code String + language_code String } enum TranscriptionJobStatus { From d25395cb6420abd0c1b73411284d2e8f0223cadc Mon Sep 17 00:00:00 2001 From: Sergio N Date: Tue, 23 Dec 2025 07:41:16 +0100 Subject: [PATCH 33/49] ADD config control for programatic transcription enablement --- .env.example | 1 + server/controllers/analysisEntryController.js | 4 +++- server/cron/jobsContainer.js | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 01f5059..25029d4 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,4 @@ AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= AWS_REGION= AWS_BUCKET= +TRANSCRIPTION_ENABLED= \ No newline at end of file diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index 9fc8c40..38b955c 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -25,7 +25,9 @@ export const updateAnalysisEntry = async (req, res) => { languageCode: 'es-ES', }; - processTranscriptionRequest(transcriptionRequest); // Fire-and-forget + if (process.env.TRANSCRIPTION_ENABLED === 'true') { + processTranscriptionRequest(transcriptionRequest); // Fire-and-forget + } return res.status(200).json({ message: 'Analysis entry updated successfully', diff --git a/server/cron/jobsContainer.js b/server/cron/jobsContainer.js index ba15ceb..1aa17b9 100644 --- a/server/cron/jobsContainer.js +++ b/server/cron/jobsContainer.js @@ -6,7 +6,11 @@ import { markAnalysisEntriesAsCancelledScheduler } from './markAsCancelledAnalys export const startCronJobs = () => { try { deletePasswordResetTokensScheduler.start(); - markAnalysisEntriesAsCancelledScheduler.start(); + + if (process.env.TRANSCRIPTION_ENABLED === true) { + markAnalysisEntriesAsCancelledScheduler.start(); + } + getCompletedTranscriptionJobsScheduler.start(); console.log('Cron jobs started'); From d88d7de01313df637a634752dec77ad50e1e1f02 Mon Sep 17 00:00:00 2001 From: Sergio N <71926587+SergioNR@users.noreply.github.com> Date: Thu, 25 Dec 2025 14:16:12 +0100 Subject: [PATCH 34/49] Fix order of cron job scheduler starts --- server/cron/jobsContainer.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/cron/jobsContainer.js b/server/cron/jobsContainer.js index 1aa17b9..83f9ac1 100644 --- a/server/cron/jobsContainer.js +++ b/server/cron/jobsContainer.js @@ -8,10 +8,11 @@ export const startCronJobs = () => { deletePasswordResetTokensScheduler.start(); if (process.env.TRANSCRIPTION_ENABLED === true) { - markAnalysisEntriesAsCancelledScheduler.start(); + getCompletedTranscriptionJobsScheduler.start(); } - getCompletedTranscriptionJobsScheduler.start(); + markAnalysisEntriesAsCancelledScheduler.start(); + console.log('Cron jobs started'); } catch (error) { From 13c9d59d6e7054cd631ba6d0de95f782ba27a730 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 00:39:52 +0100 Subject: [PATCH 35/49] REMOVE unused code relating to generating presignedurls --- server/controllers/analysisEntryController.js | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index 38b955c..3fa70a5 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -1,4 +1,4 @@ -import { generateS3GetPresignedUrl, generateS3PutPresignedUrl } from '../integrations/aws/s3.js'; +import { generateS3GetPresignedUrl } from '../integrations/aws/s3.js'; import { createAnalysisEntryInDb, getAnalysisEntryDetailsById, markAnalysisEntryAsSubmitted } from '../models/analysisEntryModel.js'; import { processTranscriptionRequest } from '../services/analysisService.js'; @@ -71,17 +71,3 @@ export const getAnalysisEntryDetails = async (req, res) => { transcriptionSegments: analysisEntryDetails.transcription_segments, }); }; - -export const getAnalysisEntryPresignedUploadUrl = async (req, res) => { - const { analysisEntryId, analysisId } = req.body; - - const key = `analysis/${analysisId}/${analysisEntryId}/recording.mp4`; - - const analysisEntryPresignedUrl = await generateS3PutPresignedUrl(key); - - return res.status(200).json({ - success: true, - message: 'PresignedUploadUrl retrieved successfully', - analysisEntryPresignedUploadUrl: analysisEntryPresignedUrl, - }); -}; From 19a87ca951b3897c279ae96af1858595dc768577 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 00:46:49 +0100 Subject: [PATCH 36/49] ADD new minio S3 related variables --- .env.example | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 25029d4..d397c4a 100644 --- a/.env.example +++ b/.env.example @@ -23,4 +23,8 @@ AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= AWS_REGION= AWS_BUCKET= -TRANSCRIPTION_ENABLED= \ No newline at end of file + +TRANSCRIPTION_ENABLED= + +MINIO_ROOT_USER= +MINIO_ROOT_PASSWORD= \ No newline at end of file From af1cbc18b1559d2b968b1822dcbaaff58893ab70 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 09:50:16 +0100 Subject: [PATCH 37/49] UPDATE s3 client config to minIO --- server/integrations/aws/s3.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/integrations/aws/s3.js b/server/integrations/aws/s3.js index 8413349..5b83b20 100644 --- a/server/integrations/aws/s3.js +++ b/server/integrations/aws/s3.js @@ -2,10 +2,12 @@ import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const s3client = new S3Client({ - region: process.env.AWS_REGION, - credentials: { // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/modules/credentials.html - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + region: process.env.AWS_REGION, // irrelevant since miniIO doesnt takei into account + endpoint: 'http://localhost:9000', // Container network endpoint + forcePathStyle: true, // Required for MinIO path-style URLs + credentials: { + accessKeyId: process.env.MINIO_ROOT_USER, + secretAccessKey: process.env.MINIO_ROOT_PASSWORD, }, }); From 876e417000b06a74c2de8c25109204f128b1f5e9 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 10:35:24 +0100 Subject: [PATCH 38/49] RENAME AWS_REGION env variable to S3_REGION --- .env.example | 2 +- .kilocode/rules/memory-bank/tech.md | 2 +- compose.yaml | 80 ++++++++++++++++++----------- server/integrations/aws/s3.js | 2 +- 4 files changed, 52 insertions(+), 34 deletions(-) diff --git a/.env.example b/.env.example index d397c4a..3b1c228 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,7 @@ PORT=3000 AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= -AWS_REGION= +S3_REGION= AWS_BUCKET= TRANSCRIPTION_ENABLED= diff --git a/.kilocode/rules/memory-bank/tech.md b/.kilocode/rules/memory-bank/tech.md index 60bdc49..1c79ee6 100644 --- a/.kilocode/rules/memory-bank/tech.md +++ b/.kilocode/rules/memory-bank/tech.md @@ -89,7 +89,7 @@ PRISMA_POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/uxca # AWS S3 AWS_ACCESS_KEY_ID=your-access-key AWS_SECRET_ACCESS_KEY=your-secret-key -AWS_REGION=eu-west-3 +S3_REGION=eu-west-3 AWS_BUCKET=your-bucket-name # Stripe diff --git a/compose.yaml b/compose.yaml index a2c8073..45d6de1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,38 +1,56 @@ name: uxcaptain services: - server: - container_name: server - build: - context: . - dockerfile: Dockerfile.dev - environment: - NODE_ENV: ${NODE_ENV} - DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT} - POSTHOG_API_KEY: ${POSTHOG_API_KEY} - SESSION_SECRET: ${SESSION_SECRET} - BREVO_API_KEY: ${BREVO_API_KEY} - PRISMA_POSTGRES_DIRECT_URL: ${PRISMA_POSTGRES_DIRECT_URL} - PRISMA_POSTGRES_CONNECTION_STRING: ${PRISMA_POSTGRES_CONNECTION_STRING} - TELEGRAM_BOT_API_KEY: ${TELEGRAM_BOT_API_KEY} - FRONT_WEB_APP_ORIGIN_URL: ${FRONT_WEB_APP_ORIGIN_URL} - STRIPE_API_KEY: ${STRIPE_API_KEY} - STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET} - PORT: ${PORT} - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - AWS_BUCKET: dev-analysis-entry-storage - AWS_REGION: ${AWS_REGION} + # server: + # container_name: server + # build: + # context: . + # dockerfile: Dockerfile.dev + # environment: + # NODE_ENV: ${NODE_ENV} + # DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT} + # POSTHOG_API_KEY: ${POSTHOG_API_KEY} + # SESSION_SECRET: ${SESSION_SECRET} + # BREVO_API_KEY: ${BREVO_API_KEY} + # PRISMA_POSTGRES_DIRECT_URL: ${PRISMA_POSTGRES_DIRECT_URL} + # PRISMA_POSTGRES_CONNECTION_STRING: ${PRISMA_POSTGRES_CONNECTION_STRING} + # TELEGRAM_BOT_API_KEY: ${TELEGRAM_BOT_API_KEY} + # FRONT_WEB_APP_ORIGIN_URL: ${FRONT_WEB_APP_ORIGIN_URL} + # STRIPE_API_KEY: ${STRIPE_API_KEY} + # STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET} + # PORT: ${PORT} + # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + # AWS_BUCKET: dev-analysis-entry-storage + # S3_REGION: ${S3_REGION} + # ports: + # - 3000:3000 + # develop: + # watch: + # - path: . + # action: sync+restart #* Synchronize code changes and restart the server + # target: /usr/src/app/ + # depends_on: #* References the SERVICE name - NOT the container_name + # database: + # condition: service_started + # minio: + # condition: service_started + # networks: + # - uxcaptain-network + + minio: + image: minio/minio:latest + container_name: minio + restart: on-failure + environment: + - MINIO_ROOT_USER=${MINIO_ROOT_USER} + - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + volumes: + - /Users/martaperezsanchez/repos/minio:/data ports: - - 3000:3000 - develop: - watch: - - path: . - action: sync+restart #* Synchronize code changes and restart the server - target: /usr/src/app/ - depends_on: #* References the SERVICE name - NOT the container_name - database: - condition: service_started + - 9000:9000 + - 9001:9001 + command: server /data --console-address ":9001" networks: - uxcaptain-network diff --git a/server/integrations/aws/s3.js b/server/integrations/aws/s3.js index 5b83b20..e1f6771 100644 --- a/server/integrations/aws/s3.js +++ b/server/integrations/aws/s3.js @@ -2,7 +2,7 @@ import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const s3client = new S3Client({ - region: process.env.AWS_REGION, // irrelevant since miniIO doesnt takei into account + region: process.env.S3_REGION, // irrelevant since miniIO doesnt takei into account endpoint: 'http://localhost:9000', // Container network endpoint forcePathStyle: true, // Required for MinIO path-style URLs credentials: { From a87212b684ed373125f0c0cd8cbfe5c8e6da4c6c Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 10:36:14 +0100 Subject: [PATCH 39/49] RENAME AWS_BUCKET env varibale to S3_BUCKET --- .env.example | 2 +- .kilocode/rules/memory-bank/tech.md | 2 +- compose.yaml | 2 +- server/integrations/aws/Transcribe.js | 6 +++--- server/integrations/aws/s3.js | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 3b1c228..7813a8f 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,7 @@ PORT=3000 AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= S3_REGION= -AWS_BUCKET= +S3_BUCKET= TRANSCRIPTION_ENABLED= diff --git a/.kilocode/rules/memory-bank/tech.md b/.kilocode/rules/memory-bank/tech.md index 1c79ee6..32c8cfc 100644 --- a/.kilocode/rules/memory-bank/tech.md +++ b/.kilocode/rules/memory-bank/tech.md @@ -90,7 +90,7 @@ PRISMA_POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/uxca AWS_ACCESS_KEY_ID=your-access-key AWS_SECRET_ACCESS_KEY=your-secret-key S3_REGION=eu-west-3 -AWS_BUCKET=your-bucket-name +S3_BUCKET=your-bucket-name # Stripe STRIPE_API_KEY=sk_test_... diff --git a/compose.yaml b/compose.yaml index 45d6de1..f18bdce 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,7 +20,7 @@ services: # PORT: ${PORT} # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - # AWS_BUCKET: dev-analysis-entry-storage + # S3_BUCKET: dev-analysis-entry-storage # S3_REGION: ${S3_REGION} # ports: diff --git a/server/integrations/aws/Transcribe.js b/server/integrations/aws/Transcribe.js index fe225ea..569f3ee 100644 --- a/server/integrations/aws/Transcribe.js +++ b/server/integrations/aws/Transcribe.js @@ -7,16 +7,16 @@ import { import { getS3Object } from './s3.js'; import { logInfo } from '../../config/loggerFunctions.js'; -const transcribeClient = new TranscribeClient({ region: process.env.AWS_REGION }); +const transcribeClient = new TranscribeClient({ region: process.env.S3_REGION }); export const requestAnalysisEntryTranscriptionToAWSTranscribe = async (transcriptionRequest) => { const command = new StartTranscriptionJobCommand({ TranscriptionJobName: transcriptionRequest.analysisEntryId, LanguageCode: transcriptionRequest.languageCode, Media: { - MediaFileUri: `s3://${process.env.AWS_BUCKET}/analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/recording.mp4`, + MediaFileUri: `s3://${process.env.S3_BUCKET}/analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/recording.mp4`, }, - OutputBucketName: process.env.AWS_BUCKET, + OutputBucketName: process.env.S3_BUCKET, OutputKey: `analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/transcription.json`, }); diff --git a/server/integrations/aws/s3.js b/server/integrations/aws/s3.js index e1f6771..2331837 100644 --- a/server/integrations/aws/s3.js +++ b/server/integrations/aws/s3.js @@ -13,7 +13,7 @@ export const s3client = new S3Client({ export const generateS3GetPresignedUrl = async (key) => { const command = new GetObjectCommand({ - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.S3_BUCKET, Key: key, }); @@ -24,7 +24,7 @@ export const generateS3GetPresignedUrl = async (key) => { export const generateS3PutPresignedUrl = async (key) => { const command = new PutObjectCommand({ - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.S3_BUCKET, Key: key, ContentType: 'video/mp4', }); @@ -38,7 +38,7 @@ export const generateS3PutPresignedUrl = async (key) => { export const getS3Object = async (key) => { const command = new GetObjectCommand({ - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.S3_BUCKET, Key: key, }); From 19d6f370d35863fb174124ae247223b1d78dabed Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 11:44:52 +0100 Subject: [PATCH 40/49] UPDATE s3 endpoint to be an env variable instead of hardcoded --- .env.example | 1 + server/integrations/aws/s3.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 7813a8f..c098c48 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,7 @@ AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= S3_REGION= S3_BUCKET= +S3_ENDPOINT= TRANSCRIPTION_ENABLED= diff --git a/server/integrations/aws/s3.js b/server/integrations/aws/s3.js index 2331837..94c1168 100644 --- a/server/integrations/aws/s3.js +++ b/server/integrations/aws/s3.js @@ -3,7 +3,7 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const s3client = new S3Client({ region: process.env.S3_REGION, // irrelevant since miniIO doesnt takei into account - endpoint: 'http://localhost:9000', // Container network endpoint + endpoint: process.env.S3_ENDPOINT, // Container network endpoint forcePathStyle: true, // Required for MinIO path-style URLs credentials: { accessKeyId: process.env.MINIO_ROOT_USER, From 8a5940ac7326937deec7ec143d4559f5ff42ef5f Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 12:45:35 +0100 Subject: [PATCH 41/49] HOTFIX: add missing minIO env variables to compose example --- compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compose.yaml b/compose.yaml index f18bdce..c033179 100644 --- a/compose.yaml +++ b/compose.yaml @@ -22,6 +22,9 @@ services: # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} # S3_BUCKET: dev-analysis-entry-storage # S3_REGION: ${S3_REGION} + # S3_ENDPOINT: ${S3_ENDPOINT} + # MINIO_ROOT_USER=${MINIO_ROOT_USER} + # MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} # ports: # - 3000:3000 From c2b8b7ea1f5802cc41fbaf2755418c354c030442 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 12:48:40 +0100 Subject: [PATCH 42/49] ADD per-repo docker hub repo Before this change, all tags were being uploaded to the same repo --- .github/workflows/docker:build&push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index 84209b8..97e1010 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -31,4 +31,4 @@ jobs: context: . platforms: linux/amd64 #,linux/arm64 - Not building for ARM, since ubuntu server is just amd64 push: true - tags: sergion14/uxcaptain:server-${{ github.ref_name }} \ No newline at end of file + tags: sergion14/uxcaptain-server-${{ github.ref_name }} \ No newline at end of file From e0303ee66a28dee892122c5a4d05c6c26d887552 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 12:59:43 +0100 Subject: [PATCH 43/49] UPDATE docker build & push GH workflow to generate dated tags too We want to do this to ensure that rollbacks exist --- .github/workflows/docker:build&push.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index 97e1010..ef70c76 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -24,6 +24,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - + name: Generate Docker metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: sergion14/uxcaptain-server + tags: | + type=raw,value=${{ github.ref_name }} + type=raw,value=${{ github.ref_name }}-${{ github.sha_short }} - name: Build and push uses: docker/build-push-action@v6 @@ -31,4 +40,4 @@ jobs: context: . platforms: linux/amd64 #,linux/arm64 - Not building for ARM, since ubuntu server is just amd64 push: true - tags: sergion14/uxcaptain-server-${{ github.ref_name }} \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file From 7dea020fbaa5083b67ac9bc4bd8db344bcec895f Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 13:03:07 +0100 Subject: [PATCH 44/49] ADD temp GH build & push condition to test the workflow --- .github/workflows/docker:build&push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index ef70c76..26b2162 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -2,7 +2,7 @@ name: Build Webapp Docker image and push to Docker Hub on: push: - branches: [ "latest", "next" ] + branches: [ "latest", "next", "add-docker-releases" ] jobs: docker-build-and-push: From 0e0f2910b08f16af178c1943112452a63ac9298b Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 13:05:15 +0100 Subject: [PATCH 45/49] FIX docker metadata-action version --- .github/workflows/docker:build&push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index 26b2162..a0c3ce9 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -27,7 +27,7 @@ jobs: - name: Generate Docker metadata id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@v5 with: images: sergion14/uxcaptain-server tags: | From 0d584782e82db15bb816e7131501dfae67545e6f Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 13:08:00 +0100 Subject: [PATCH 46/49] FIX sha not being generated --- .github/workflows/docker:build&push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index a0c3ce9..12291c9 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -32,7 +32,7 @@ jobs: images: sergion14/uxcaptain-server tags: | type=raw,value=${{ github.ref_name }} - type=raw,value=${{ github.ref_name }}-${{ github.sha_short }} + type=sha,value=${{ github.ref_name }}-${{ github.sha_short }} - name: Build and push uses: docker/build-push-action@v6 From d55e0bafbdeca379892e08f3bba7803dbed5d28b Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 13:15:09 +0100 Subject: [PATCH 47/49] REPLACE sha with date --- .github/workflows/docker:build&push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index 12291c9..868d66d 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -32,7 +32,7 @@ jobs: images: sergion14/uxcaptain-server tags: | type=raw,value=${{ github.ref_name }} - type=sha,value=${{ github.ref_name }}-${{ github.sha_short }} + type=raw,value=${{ github.ref_name }}-date='%Y-%m-%d' - name: Build and push uses: docker/build-push-action@v6 From c3a86fe85d25dae90d262913834ea64357f848b1 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 13:21:33 +0100 Subject: [PATCH 48/49] FIX date generation --- .github/workflows/docker:build&push.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index 868d66d..cd899ea 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -24,6 +24,10 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - + name: Set date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT" - name: Generate Docker metadata id: meta @@ -32,7 +36,7 @@ jobs: images: sergion14/uxcaptain-server tags: | type=raw,value=${{ github.ref_name }} - type=raw,value=${{ github.ref_name }}-date='%Y-%m-%d' + type=raw,value=${{ github.ref_name }}-${{ steps.date.outputs.date }} - name: Build and push uses: docker/build-push-action@v6 From 4369c0358efebdb48a3b85377e2a508b1a963244 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Fri, 26 Dec 2025 13:24:09 +0100 Subject: [PATCH 49/49] REMOVE image generation on non-latest/next branches --- .github/workflows/docker:build&push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index cd899ea..4bb7093 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -2,7 +2,7 @@ name: Build Webapp Docker image and push to Docker Hub on: push: - branches: [ "latest", "next", "add-docker-releases" ] + branches: [ "latest", "next" ] jobs: docker-build-and-push: