diff --git a/.env.example b/.env.example index 9ec90ca..c098c48 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,11 @@ PORT=3000 AWS_SECRET_ACCESS_KEY= AWS_ACCESS_KEY_ID= -AWS_REGION= -AWS_BUCKET= +S3_REGION= +S3_BUCKET= +S3_ENDPOINT= -LAVINMQ_HOST= \ No newline at end of file +TRANSCRIPTION_ENABLED= + +MINIO_ROOT_USER= +MINIO_ROOT_PASSWORD= \ No newline at end of file diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 8967536..b0a4d2d 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -44,5 +44,8 @@ module.exports = { 'import/no-relative-packages': 'off', 'no-case-declarations': 'off', 'max-len': 'off', + 'no-plusplus': 'off', + 'no-await-in-loop': 'off', + 'no-restricted-syntax': 'off', }, }; diff --git a/.github/workflows/docker:build&push.yaml b/.github/workflows/docker:build&push.yaml index 84209b8..4bb7093 100644 --- a/.github/workflows/docker:build&push.yaml +++ b/.github/workflows/docker:build&push.yaml @@ -24,6 +24,19 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - + name: Set date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT" + - + name: Generate Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: sergion14/uxcaptain-server + tags: | + type=raw,value=${{ github.ref_name }} + type=raw,value=${{ github.ref_name }}-${{ steps.date.outputs.date }} - name: Build and push uses: docker/build-push-action@v6 @@ -31,4 +44,4 @@ jobs: context: . platforms: linux/amd64 #,linux/arm64 - Not building for ARM, since ubuntu server is just amd64 push: true - tags: sergion14/uxcaptain:server-${{ github.ref_name }} \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/.kilocode/rules/memory-bank/tech.md b/.kilocode/rules/memory-bank/tech.md index 62bca33..32c8cfc 100644 --- a/.kilocode/rules/memory-bank/tech.md +++ b/.kilocode/rules/memory-bank/tech.md @@ -89,8 +89,8 @@ PRISMA_POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/uxca # AWS S3 AWS_ACCESS_KEY_ID=your-access-key AWS_SECRET_ACCESS_KEY=your-secret-key -AWS_REGION=eu-west-3 -AWS_BUCKET=your-bucket-name +S3_REGION=eu-west-3 +S3_BUCKET=your-bucket-name # Stripe STRIPE_API_KEY=sk_test_... @@ -107,9 +107,6 @@ TELEGRAM_BOT_API_KEY=your-bot-token # Frontend FRONT_WEB_APP_ORIGIN_URL=http://localhost:5173 - -# Message Queue -LAVINMQ_HOST=localhost ``` ### Local Development Commands diff --git a/compose.yaml b/compose.yaml index b310e0a..c033179 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,39 +1,59 @@ name: uxcaptain services: - server: - container_name: server - build: - context: . - dockerfile: Dockerfile.dev - environment: - NODE_ENV: ${NODE_ENV} - DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT} - POSTHOG_API_KEY: ${POSTHOG_API_KEY} - SESSION_SECRET: ${SESSION_SECRET} - BREVO_API_KEY: ${BREVO_API_KEY} - PRISMA_POSTGRES_DIRECT_URL: ${PRISMA_POSTGRES_DIRECT_URL} - PRISMA_POSTGRES_CONNECTION_STRING: ${PRISMA_POSTGRES_CONNECTION_STRING} - TELEGRAM_BOT_API_KEY: ${TELEGRAM_BOT_API_KEY} - FRONT_WEB_APP_ORIGIN_URL: ${FRONT_WEB_APP_ORIGIN_URL} - STRIPE_API_KEY: ${STRIPE_API_KEY} - STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET} - PORT: ${PORT} - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - AWS_BUCKET: dev-analysis-entry-storage - AWS_REGION: ${AWS_REGION} - LAVINMQ_HOST: ${LAVINMQ_HOST} + # server: + # container_name: server + # build: + # context: . + # dockerfile: Dockerfile.dev + # environment: + # NODE_ENV: ${NODE_ENV} + # DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT} + # POSTHOG_API_KEY: ${POSTHOG_API_KEY} + # SESSION_SECRET: ${SESSION_SECRET} + # BREVO_API_KEY: ${BREVO_API_KEY} + # PRISMA_POSTGRES_DIRECT_URL: ${PRISMA_POSTGRES_DIRECT_URL} + # PRISMA_POSTGRES_CONNECTION_STRING: ${PRISMA_POSTGRES_CONNECTION_STRING} + # TELEGRAM_BOT_API_KEY: ${TELEGRAM_BOT_API_KEY} + # FRONT_WEB_APP_ORIGIN_URL: ${FRONT_WEB_APP_ORIGIN_URL} + # STRIPE_API_KEY: ${STRIPE_API_KEY} + # STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET} + # PORT: ${PORT} + # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + # S3_BUCKET: dev-analysis-entry-storage + # S3_REGION: ${S3_REGION} + # S3_ENDPOINT: ${S3_ENDPOINT} + # MINIO_ROOT_USER=${MINIO_ROOT_USER} + # MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + # ports: + # - 3000:3000 + # develop: + # watch: + # - path: . + # action: sync+restart #* Synchronize code changes and restart the server + # target: /usr/src/app/ + # depends_on: #* References the SERVICE name - NOT the container_name + # database: + # condition: service_started + # minio: + # condition: service_started + # networks: + # - uxcaptain-network + + minio: + image: minio/minio:latest + container_name: minio + restart: on-failure + environment: + - MINIO_ROOT_USER=${MINIO_ROOT_USER} + - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + volumes: + - /Users/martaperezsanchez/repos/minio:/data ports: - - 3000:3000 - develop: - watch: - - path: . - action: sync+restart #* Synchronize code changes and restart the server - target: /usr/src/app/ - depends_on: #* References the SERVICE name - NOT the container_name - database: - condition: service_started + - 9000:9000 + - 9001:9001 + command: server /data --console-address ":9001" networks: - uxcaptain-network diff --git a/package-lock.json b/package-lock.json index 8016666..ab8f1da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,8 @@ "license": "Proprietary", "dependencies": { "@aws-sdk/client-s3": "^3.936.0", + "@aws-sdk/client-transcribe": "^3.948.0", "@aws-sdk/s3-request-presigner": "^3.936.0", - "@cloudamqp/amqp-client": "^3.4.0", "@getbrevo/brevo": "^3.0.1", "@prisma/client": "^6.19.0", "@quixo3/prisma-session-store": "^3.1.13", @@ -365,6 +365,412 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/client-transcribe": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe/-/client-transcribe-3.948.0.tgz", + "integrity": "sha512-EOPYaW/lL2UHZbsG6PxPeHu/Pcw8MTsUznrRW6z7svVHCgsQkGUoWJs9gxTr601r+TMPgt8rdv2bv+WgXeN/SQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/credential-provider-node": "3.948.0", + "@aws-sdk/middleware-host-header": "3.936.0", + "@aws-sdk/middleware-logger": "3.936.0", + "@aws-sdk/middleware-recursion-detection": "3.948.0", + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/region-config-resolver": "3.936.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@aws-sdk/util-user-agent-browser": "3.936.0", + "@aws-sdk/util-user-agent-node": "3.947.0", + "@smithy/config-resolver": "^4.4.3", + "@smithy/core": "^3.18.7", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/hash-node": "^4.2.5", + "@smithy/invalid-dependency": "^4.2.5", + "@smithy/middleware-content-length": "^4.2.5", + "@smithy/middleware-endpoint": "^4.3.14", + "@smithy/middleware-retry": "^4.4.14", + "@smithy/middleware-serde": "^4.2.6", + "@smithy/middleware-stack": "^4.2.5", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/url-parser": "^4.2.5", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.13", + "@smithy/util-defaults-mode-node": "^4.2.16", + "@smithy/util-endpoints": "^3.2.5", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-retry": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/client-sso": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.948.0.tgz", + "integrity": "sha512-iWjchXy8bIAVBUsKnbfKYXRwhLgRg3EqCQ5FTr3JbR+QR75rZm4ZOYXlvHGztVTmtAZ+PQVA1Y4zO7v7N87C0A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/middleware-host-header": "3.936.0", + "@aws-sdk/middleware-logger": "3.936.0", + "@aws-sdk/middleware-recursion-detection": "3.948.0", + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/region-config-resolver": "3.936.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@aws-sdk/util-user-agent-browser": "3.936.0", + "@aws-sdk/util-user-agent-node": "3.947.0", + "@smithy/config-resolver": "^4.4.3", + "@smithy/core": "^3.18.7", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/hash-node": "^4.2.5", + "@smithy/invalid-dependency": "^4.2.5", + "@smithy/middleware-content-length": "^4.2.5", + "@smithy/middleware-endpoint": "^4.3.14", + "@smithy/middleware-retry": "^4.4.14", + "@smithy/middleware-serde": "^4.2.6", + "@smithy/middleware-stack": "^4.2.5", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/url-parser": "^4.2.5", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.13", + "@smithy/util-defaults-mode-node": "^4.2.16", + "@smithy/util-endpoints": "^3.2.5", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-retry": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/core": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.947.0.tgz", + "integrity": "sha512-Khq4zHhuAkvCFuFbgcy3GrZTzfSX7ZIjIcW1zRDxXRLZKRtuhnZdonqTUfaWi5K42/4OmxkYNpsO7X7trQOeHw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.936.0", + "@aws-sdk/xml-builder": "3.930.0", + "@smithy/core": "^3.18.7", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/signature-v4": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.947.0.tgz", + "integrity": "sha512-VR2V6dRELmzwAsCpK4GqxUi6UW5WNhAXS9F9AzWi5jvijwJo3nH92YNJUP4quMpgFZxJHEWyXLWgPjh9u0zYOA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.947.0.tgz", + "integrity": "sha512-inF09lh9SlHj63Vmr5d+LmwPXZc2IbK8lAruhOr3KLsZAIHEgHgGPXWDC2ukTEMzg0pkexQ6FOhXXad6klK4RA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/util-stream": "^4.5.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.948.0.tgz", + "integrity": "sha512-Cl//Qh88e8HBL7yYkJNpF5eq76IO6rq8GsatKcfVBm7RFVxCqYEPSSBtkHdbtNwQdRQqAMXc6E/lEB/CZUDxnA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/credential-provider-env": "3.947.0", + "@aws-sdk/credential-provider-http": "3.947.0", + "@aws-sdk/credential-provider-login": "3.948.0", + "@aws-sdk/credential-provider-process": "3.947.0", + "@aws-sdk/credential-provider-sso": "3.948.0", + "@aws-sdk/credential-provider-web-identity": "3.948.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/credential-provider-imds": "^4.2.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-login": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.948.0.tgz", + "integrity": "sha512-gcKO2b6eeTuZGp3Vvgr/9OxajMrD3W+FZ2FCyJox363ZgMoYJsyNid1vuZrEuAGkx0jvveLXfwiVS0UXyPkgtw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.948.0.tgz", + "integrity": "sha512-ep5vRLnrRdcsP17Ef31sNN4g8Nqk/4JBydcUJuFRbGuyQtrZZrVT81UeH2xhz6d0BK6ejafDB9+ZpBjXuWT5/Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.947.0", + "@aws-sdk/credential-provider-http": "3.947.0", + "@aws-sdk/credential-provider-ini": "3.948.0", + "@aws-sdk/credential-provider-process": "3.947.0", + "@aws-sdk/credential-provider-sso": "3.948.0", + "@aws-sdk/credential-provider-web-identity": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/credential-provider-imds": "^4.2.5", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.947.0.tgz", + "integrity": "sha512-WpanFbHe08SP1hAJNeDdBDVz9SGgMu/gc0XJ9u3uNpW99nKZjDpvPRAdW7WLA4K6essMjxWkguIGNOpij6Do2Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.948.0.tgz", + "integrity": "sha512-gqLhX1L+zb/ZDnnYbILQqJ46j735StfWV5PbDjxRzBKS7GzsiYoaf6MyHseEopmWrez5zl5l6aWzig7UpzSeQQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-sso": "3.948.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/token-providers": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.948.0.tgz", + "integrity": "sha512-MvYQlXVoJyfF3/SmnNzOVEtANRAiJIObEUYYyjTqKZTmcRIVVky0tPuG26XnB8LmTYgtESwJIZJj/Eyyc9WURQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.948.0.tgz", + "integrity": "sha512-Qa8Zj+EAqA0VlAVvxpRnpBpIWJI9KUwaioY1vkeNVwXPlNaz9y9zCKVM9iU9OZ5HXpoUg6TnhATAHXHAE8+QsQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.936.0", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/protocol-http": "^5.3.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.947.0.tgz", + "integrity": "sha512-7rpKV8YNgCP2R4F9RjWZFcD2R+SO/0R4VHIbY9iZJdH2MzzJ8ZG7h8dZ2m8QkQd1fjx4wrFJGGPJUTYXPV3baA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@smithy/core": "^3.18.7", + "@smithy/protocol-http": "^5.3.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/nested-clients": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.948.0.tgz", + "integrity": "sha512-zcbJfBsB6h254o3NuoEkf0+UY1GpE9ioiQdENWv7odo69s8iaGBEQ4BDpsIMqcuiiUXw1uKIVNxCB1gUGYz8lw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.947.0", + "@aws-sdk/middleware-host-header": "3.936.0", + "@aws-sdk/middleware-logger": "3.936.0", + "@aws-sdk/middleware-recursion-detection": "3.948.0", + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/region-config-resolver": "3.936.0", + "@aws-sdk/types": "3.936.0", + "@aws-sdk/util-endpoints": "3.936.0", + "@aws-sdk/util-user-agent-browser": "3.936.0", + "@aws-sdk/util-user-agent-node": "3.947.0", + "@smithy/config-resolver": "^4.4.3", + "@smithy/core": "^3.18.7", + "@smithy/fetch-http-handler": "^5.3.6", + "@smithy/hash-node": "^4.2.5", + "@smithy/invalid-dependency": "^4.2.5", + "@smithy/middleware-content-length": "^4.2.5", + "@smithy/middleware-endpoint": "^4.3.14", + "@smithy/middleware-retry": "^4.4.14", + "@smithy/middleware-serde": "^4.2.6", + "@smithy/middleware-stack": "^4.2.5", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/node-http-handler": "^4.4.5", + "@smithy/protocol-http": "^5.3.5", + "@smithy/smithy-client": "^4.9.10", + "@smithy/types": "^4.9.0", + "@smithy/url-parser": "^4.2.5", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.13", + "@smithy/util-defaults-mode-node": "^4.2.16", + "@smithy/util-endpoints": "^3.2.5", + "@smithy/util-middleware": "^4.2.5", + "@smithy/util-retry": "^4.2.5", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/token-providers": { + "version": "3.948.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.948.0.tgz", + "integrity": "sha512-V487/kM4Teq5dcr1t5K6eoUKuqlGr9FRWL3MIMukMERJXHZvio6kox60FZ/YtciRHRI75u14YUqm2Dzddcu3+A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.947.0", + "@aws-sdk/nested-clients": "3.948.0", + "@aws-sdk/types": "3.936.0", + "@smithy/property-provider": "^4.2.5", + "@smithy/shared-ini-file-loader": "^4.4.0", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-transcribe/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.947.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.947.0.tgz", + "integrity": "sha512-+vhHoDrdbb+zerV4noQk1DHaUMNzWFWPpPYjVTwW2186k5BEJIecAMChYkghRrBVJ3KPWP1+JnZwOd72F3d4rQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.947.0", + "@aws-sdk/types": "3.936.0", + "@smithy/node-config-provider": "^4.3.5", + "@smithy/types": "^4.9.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, "node_modules/@aws-sdk/core": { "version": "3.940.0", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.940.0.tgz", @@ -958,9 +1364,9 @@ } }, "node_modules/@aws/lambda-invoke-store": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.1.tgz", - "integrity": "sha512-sIyFcoPZkTtNu9xFeEoynMef3bPJIAbOfUh+ueYcfhVl6xm2VRtMcMclSxmZCMnHHd4hlYKJeq/aggmBEWynww==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.2.tgz", + "integrity": "sha512-C0NBLsIqzDIae8HFw9YIrIBsbc0xTiOtt7fAukGPnqQ/+zZNaq+4jhuccltK0QuWHBnNm/a6kLIRA6GFiM10eg==", "license": "Apache-2.0", "engines": { "node": ">=18.0.0" @@ -1463,15 +1869,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@cloudamqp/amqp-client": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/@cloudamqp/amqp-client/-/amqp-client-3.4.1.tgz", - "integrity": "sha512-A53N3dpS4zEaVL1GMIDg/A4hTy/t/nqVDNaY6m+48VKi2N1cpGkmTmc6RHMAD06UTV6Z0O8CeGqs7076w6VFMQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@emnapi/core": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", @@ -2905,9 +3302,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.18.6", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.18.6.tgz", - "integrity": "sha512-8Q/ugWqfDUEU1Exw71+DoOzlONJ2Cn9QA8VeeDzLLjzO/qruh9UKFzbszy4jXcIYgGofxYiT0t1TT6+CT/GupQ==", + "version": "3.18.7", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.18.7.tgz", + "integrity": "sha512-axG9MvKhMWOhFbvf5y2DuyTxQueO0dkedY9QC3mAfndLosRI/9LJv8WaL0mw7ubNhsO4IuXX9/9dYGPFvHrqlw==", "license": "Apache-2.0", "dependencies": { "@smithy/middleware-serde": "^4.2.6", @@ -3125,12 +3522,12 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.3.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.3.13.tgz", - "integrity": "sha512-X4za1qCdyx1hEVVXuAWlZuK6wzLDv1uw1OY9VtaYy1lULl661+frY7FeuHdYdl7qAARUxH2yvNExU2/SmRFfcg==", + "version": "4.3.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.3.14.tgz", + "integrity": "sha512-v0q4uTKgBM8dsqGjqsabZQyH85nFaTnFcgpWU1uydKFsdyyMzfvOkNum9G7VK+dOP01vUnoZxIeRiJ6uD0kjIg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.18.6", + "@smithy/core": "^3.18.7", "@smithy/middleware-serde": "^4.2.6", "@smithy/node-config-provider": "^4.3.5", "@smithy/shared-ini-file-loader": "^4.4.0", @@ -3144,15 +3541,15 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.4.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.13.tgz", - "integrity": "sha512-RzIDF9OrSviXX7MQeKOm8r/372KTyY8Jmp6HNKOOYlrguHADuM3ED/f4aCyNhZZFLG55lv5beBin7nL0Nzy1Dw==", + "version": "4.4.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.14.tgz", + "integrity": "sha512-Z2DG8Ej7FyWG1UA+7HceINtSLzswUgs2np3sZX0YBBxCt+CXG4QUxv88ZDS3+2/1ldW7LqtSY1UO/6VQ1pND8Q==", "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.5", "@smithy/protocol-http": "^5.3.5", "@smithy/service-error-classification": "^4.2.5", - "@smithy/smithy-client": "^4.9.9", + "@smithy/smithy-client": "^4.9.10", "@smithy/types": "^4.9.0", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", @@ -3319,13 +3716,13 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.9.9", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.9.9.tgz", - "integrity": "sha512-SUnZJMMo5yCmgjopJbiNeo1vlr8KvdnEfIHV9rlD77QuOGdRotIVBcOrBuMr+sI9zrnhtDtLP054bZVbpZpiQA==", + "version": "4.9.10", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.9.10.tgz", + "integrity": "sha512-Jaoz4Jw1QYHc1EFww/E6gVtNjhoDU+gwRKqXP6C3LKYqqH2UQhP8tMP3+t/ePrhaze7fhLE8vS2q6vVxBANFTQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.18.6", - "@smithy/middleware-endpoint": "^4.3.13", + "@smithy/core": "^3.18.7", + "@smithy/middleware-endpoint": "^4.3.14", "@smithy/middleware-stack": "^4.2.5", "@smithy/protocol-http": "^5.3.5", "@smithy/types": "^4.9.0", @@ -3426,13 +3823,13 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.12", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.12.tgz", - "integrity": "sha512-TKc6FnOxFULKxLgTNHYjcFqdOYzXVPFFVm5JhI30F3RdhT7nYOtOsjgaOwfDRmA/3U66O9KaBQ3UHoXwayRhAg==", + "version": "4.3.13", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.13.tgz", + "integrity": "sha512-hlVLdAGrVfyNei+pKIgqDTxfu/ZI2NSyqj4IDxKd5bIsIqwR/dSlkxlPaYxFiIaDVrBy0he8orsFy+Cz119XvA==", "license": "Apache-2.0", "dependencies": { "@smithy/property-provider": "^4.2.5", - "@smithy/smithy-client": "^4.9.9", + "@smithy/smithy-client": "^4.9.10", "@smithy/types": "^4.9.0", "tslib": "^2.6.2" }, @@ -3441,16 +3838,16 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.15", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.15.tgz", - "integrity": "sha512-94NqfQVo+vGc5gsQ9SROZqOvBkGNMQu6pjXbnn8aQvBUhc31kx49gxlkBEqgmaZQHUUfdRUin5gK/HlHKmbAwg==", + "version": "4.2.16", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.16.tgz", + "integrity": "sha512-F1t22IUiJLHrxW9W1CQ6B9PN+skZ9cqSuzB18Eh06HrJPbjsyZ7ZHecAKw80DQtyGTRcVfeukKaCRYebFwclbg==", "license": "Apache-2.0", "dependencies": { "@smithy/config-resolver": "^4.4.3", "@smithy/credential-provider-imds": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/property-provider": "^4.2.5", - "@smithy/smithy-client": "^4.9.9", + "@smithy/smithy-client": "^4.9.10", "@smithy/types": "^4.9.0", "tslib": "^2.6.2" }, diff --git a/package.json b/package.json index 151a5ca..b89d886 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.936.0", + "@aws-sdk/client-transcribe": "^3.948.0", "@aws-sdk/s3-request-presigner": "^3.936.0", - "@cloudamqp/amqp-client": "^3.4.0", "@getbrevo/brevo": "^3.0.1", "@prisma/client": "^6.19.0", "@quixo3/prisma-session-store": "^3.1.13", diff --git a/prisma/migrations/20251222131114_add_transcription_job_table/migration.sql b/prisma/migrations/20251222131114_add_transcription_job_table/migration.sql new file mode 100644 index 0000000..b9a5f8a --- /dev/null +++ b/prisma/migrations/20251222131114_add_transcription_job_table/migration.sql @@ -0,0 +1,23 @@ +-- CreateEnum +CREATE TYPE "TranscriptionJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED'); + +-- CreateTable +CREATE TABLE "TranscriptionJob" ( + "id" SERIAL NOT NULL, + "analysis_entry_id" TEXT NOT NULL, + "status" "TranscriptionJobStatus" NOT NULL DEFAULT 'PENDING', + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "language_code" TEXT NOT NULL, + + CONSTRAINT "TranscriptionJob_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "TranscriptionJob_id_key" ON "TranscriptionJob"("id"); + +-- CreateIndex +CREATE UNIQUE INDEX "TranscriptionJob_analysis_entry_id_key" ON "TranscriptionJob"("analysis_entry_id"); + +-- AddForeignKey +ALTER TABLE "TranscriptionJob" ADD CONSTRAINT "TranscriptionJob_analysis_entry_id_fkey" FOREIGN KEY ("analysis_entry_id") REFERENCES "AnalysisEntry"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b380f13..c75a00b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -44,18 +44,18 @@ model Company { } model ParticipantProfile { - id Int @id @default(autoincrement()) - name String? - last_name String? - age String? // String to allow for prefer not to say option - gender Genders? - invited_by String? - country String? - devices String[] @default([]) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt - User User @relation(fields: [user_id], references: [id]) - user_id String @unique + id Int @id @default(autoincrement()) + name String? + last_name String? + age String? // String to allow for prefer not to say option + gender Genders? + invited_by String? + country String? + devices String[] @default([]) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + User User @relation(fields: [user_id], references: [id]) + user_id String @unique AnalysisEntry AnalysisEntry[] } @@ -85,11 +85,11 @@ model Session { } model Subscription { - id String @id @unique @default(uuid()) - company_id String @unique - Company Company @relation(fields: [company_id], references: [id]) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt + id String @id @unique @default(uuid()) + company_id String @unique + Company Company @relation(fields: [company_id], references: [id]) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt expires_at DateTime? } @@ -105,30 +105,47 @@ model Analysis { tasks Json url String - status AnalysisStatus @default(draft) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt + status AnalysisStatus @default(draft) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt max_number_of_participants Int - AnalysisEntry AnalysisEntry[] + AnalysisEntry AnalysisEntry[] @@index([owner_company_id]) // Not a unique field, but requires an index } model AnalysisEntry { - id String @id @default(uuid()) - Analysis Analysis @relation(fields: [analysis_id], references: [id]) - analysis_id String - ParticipantProfile ParticipantProfile? @relation(fields: [user_id], references: [user_id]) - user_id String? - status AnalysisEntryCompletionStatus @default(in_progress) - created_at DateTime @default(now()) - updated_at DateTime @default(now()) @updatedAt + id String @id @default(uuid()) + Analysis Analysis @relation(fields: [analysis_id], references: [id]) + analysis_id String + ParticipantProfile ParticipantProfile? @relation(fields: [user_id], references: [user_id]) + user_id String? + status AnalysisEntryCompletionStatus @default(in_progress) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt transcription_segments Json? - full_transcript String? + full_transcript String? + transcriptionJob TranscriptionJob? @@index([analysis_id, status]) } +model TranscriptionJob { + id Int @id @unique @default(autoincrement()) + AnalysisEntry AnalysisEntry @relation(fields: [analysis_entry_id], references: [id]) + analysis_entry_id String @unique + status TranscriptionJobStatus @default(PENDING) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + language_code String +} + +enum TranscriptionJobStatus { + PENDING + IN_PROGRESS + COMPLETED +} + enum Genders { male female diff --git a/server/app.js b/server/app.js index 9be7c7d..c0f0065 100644 --- a/server/app.js +++ b/server/app.js @@ -12,7 +12,6 @@ import { slowLimiter } from './middlewares/express-slow-down.js'; import { startCronJobs } from './cron/jobsContainer.js'; import { globalErrorHandler } from './middlewares/globalErrorHandler.js'; import { webhookRouter } from './webhooks/webhooksRouter.js'; -import { connectToMessageBroker } from './config/messageBroker/LavinMQ.js'; const app = express(); const server = createServer(app); @@ -47,10 +46,9 @@ app.use(globalErrorHandler); //* Start the server server.listen(process.env.PORT, () => { // eslint-disable-next-line no-console - console.log(`Server running at http://localhost:${process.env.PORT}/`); + console.log('Server running'); }); -connectToMessageBroker(); startCronJobs(); const gracefulShutdown = () => { diff --git a/server/config/messageBroker/LavinMQ.js b/server/config/messageBroker/LavinMQ.js deleted file mode 100644 index 9e585a0..0000000 --- a/server/config/messageBroker/LavinMQ.js +++ /dev/null @@ -1,131 +0,0 @@ -import { AMQPClient } from '@cloudamqp/amqp-client'; -import { logError, logInfo } from '../loggerFunctions.js'; -import { insertAnalysisEntryTranscriptionInDb } from '../../models/analysisEntryModel.js'; - -let connection; -let channel; -let transcriptionRequestedQueue; -let insightsRequestedQueue; - -// Main AMQP setup function -export const connectToMessageBroker = async () => { - try { - // 1. Establish connection - const amqp = new AMQPClient(process.env.LAVINMQ_HOST); - connection = await amqp.connect(); // Establish connection to the message broker - one connection for all channels - - // 2. Open producer and consumer channels - channel = await connection.channel(); // One channel for producing & consuming messages - - // 3. Declare exchanges & queues & bindings - - // 3.1 Declare exchanges - - const analysisExchange = await channel.exchangeDeclare('analysis_exchange', 'topic', { - durable: true, - passive: false, - autoDelete: false, - internal: false, - }); - - // 3.2 Declare queues - transcriptionRequestedQueue = await channel.queue('transcription_requested_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - const transcriptionCompletedQueue = await channel.queue('transcription_completed_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - insightsRequestedQueue = await channel.queue('insights_requested_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - const insightsCompletedQueue = await channel.queue('insights_completed_queue', { - durable: true, - passive: false, - autoDelete: false, - exclusive: false, - }); - - // 3.3 Bind queues to exchange with routing keys - - await transcriptionRequestedQueue.bind('analysis_exchange', 'analysis.analysisEntry.transcription.requested', { - }); - - await insightsRequestedQueue.bind('analysis_exchange', 'analysis.analysisEntry.insights.requested', { - }); - - await insightsCompletedQueue.bind('analysis_exchange', 'analysis.analysisEntry.insights.completed', { - }); - - // Start consumers after successful connection - - await transcriptionCompletedQueue.subscribe({ noAck: false }, async (msg) => { - try { - const contentStr = msg.bodyToString(msg); - const completedTranscriptionRequest = JSON.parse(contentStr); - - await insertAnalysisEntryTranscriptionInDb(completedTranscriptionRequest); - - /* - const exampleReceivedMessage = { - analysisEntryId: transcriptionJobDetails._id, - transcriptionData: array of objects, each representing a segment of the transcription - fullTranscript: 'lorem ipsum dolor sit amet, consectetur adipiscing elit' - }; - */ - - await msg.ack(); - } catch (error) { - logError('Error processing transcription completed message', error); - await msg.nack(true); // Requeue on failure - } - }); - - await insightsCompletedQueue.subscribe({ noAck: false }, async (msg) => { - try { - await msg.ack(); - } catch (error) { - await msg.nack(true); // Requeue on failure - } - }); - - logInfo('monolith successfully connected to LavinMQ message broker'); - - return { - connection: connection, channel: channel, analysisExchange, transcriptionQueue: transcriptionRequestedQueue, - }; - } catch (e) { - logError('error connecting to message broker', e); - e.connection?.close(); - return setTimeout(connectToMessageBroker, 1000); // will try to reconnect in 1s - } -}; - -// function for publishing to transcription queue - -export const publishToTranscriptionRequestedQueue = async (message) => { - try { - return await transcriptionRequestedQueue.publish(message); - } catch (err) { - return logError('error publishing transcription request', err); - } -}; - -export const publishToInsightsRequestedQueue = async (message) => { - try { - return await insightsRequestedQueue.publish(message); - } catch (err) { - return logError('error publishing transcription request', err); - } -}; diff --git a/server/controllers/analysisController.js b/server/controllers/analysisController.js index 74702ce..3f21a91 100644 --- a/server/controllers/analysisController.js +++ b/server/controllers/analysisController.js @@ -135,6 +135,6 @@ export const participateInAnalysis = async (req, res) => { message: 'Analysis info retrieved successfully', analysisData: analysisData, analysisEntryId: analysisEntry.id, - analysisPresignedUploadUrl: analysisEntryPresignedUploadUrl, + analysisEntryPresignedUploadUrl: analysisEntryPresignedUploadUrl, }); }; diff --git a/server/controllers/analysisEntryController.js b/server/controllers/analysisEntryController.js index 196e00e..3fa70a5 100644 --- a/server/controllers/analysisEntryController.js +++ b/server/controllers/analysisEntryController.js @@ -1,7 +1,6 @@ -import { logError } from '../config/loggerFunctions.js'; -import { publishToTranscriptionRequestedQueue } from '../config/messageBroker/LavinMQ.js'; -import { generateS3GetPresignedUrl, generateS3PutPresignedUrl } from '../integrations/aws/s3.js'; -import { createAnalysisEntryInDb, getAnalysisEntryDetailsById, updateAnalysisEntryInDb } from '../models/analysisEntryModel.js'; +import { generateS3GetPresignedUrl } from '../integrations/aws/s3.js'; +import { createAnalysisEntryInDb, getAnalysisEntryDetailsById, markAnalysisEntryAsSubmitted } from '../models/analysisEntryModel.js'; +import { processTranscriptionRequest } from '../services/analysisService.js'; export const createAnalysisEntry = async (req, res) => { const { analysisId } = req.body; @@ -16,28 +15,21 @@ export const createAnalysisEntry = async (req, res) => { }; export const updateAnalysisEntry = async (req, res) => { - const { analysisEntryId, analysisEntryStatus, analysisId } = req.body; + const { analysisEntryId } = req.body; - await updateAnalysisEntryInDb(analysisEntryId, analysisEntryStatus); + const updatedAnalysisEntry = await markAnalysisEntryAsSubmitted(analysisEntryId); - try { - const message = { - analysisEntryId: analysisEntryId, - analysisId: analysisId, - timestamp: new Date().toISOString(), - mediaType: 'video', - languageCode: 'es-ES', - }; + const transcriptionRequest = { + analysisEntryId: analysisEntryId, + analysisId: updatedAnalysisEntry.analysis_id, + languageCode: 'es-ES', + }; - const stringifiedMessage = JSON.stringify(message); - - await publishToTranscriptionRequestedQueue(stringifiedMessage); - } catch (error) { - logError(`Error sending transcription request analysisEntry ${analysisEntryId}`, error); + if (process.env.TRANSCRIPTION_ENABLED === 'true') { + processTranscriptionRequest(transcriptionRequest); // Fire-and-forget } return res.status(200).json({ - success: true, message: 'Analysis entry updated successfully', }); }; @@ -79,17 +71,3 @@ export const getAnalysisEntryDetails = async (req, res) => { transcriptionSegments: analysisEntryDetails.transcription_segments, }); }; - -export const getAnalysisEntryPresignedUploadUrl = async (req, res) => { - const { analysisEntryId, analysisId } = req.body; - - const key = `analysis/${analysisId}/${analysisEntryId}/recording.mp4`; - - const analysisEntryPresignedUrl = await generateS3PutPresignedUrl(key); - - return res.status(200).json({ - success: true, - message: 'PresignedUploadUrl retrieved successfully', - analysisEntryPresignedUploadUrl: analysisEntryPresignedUrl, - }); -}; diff --git a/server/cron/getCompletedTranscriptionJobsScheduler.js b/server/cron/getCompletedTranscriptionJobsScheduler.js new file mode 100644 index 0000000..36a2f44 --- /dev/null +++ b/server/cron/getCompletedTranscriptionJobsScheduler.js @@ -0,0 +1,12 @@ +import { CronJob } from 'cron'; +import { handleCompletedVideoTranscriptionJobs } from '../services/analysisService.js'; +import { logError, logInfo } from '../config/loggerFunctions.js'; + +export const getCompletedTranscriptionJobsScheduler = new CronJob('15 * * * *', async () => { + try { + logInfo('Checking transcription job status'); + await handleCompletedVideoTranscriptionJobs(); + } catch (error) { + logError('Error checking transcription job status', error); + } +}); diff --git a/server/cron/jobsContainer.js b/server/cron/jobsContainer.js index 054b7ef..83f9ac1 100644 --- a/server/cron/jobsContainer.js +++ b/server/cron/jobsContainer.js @@ -1,11 +1,18 @@ import { logError } from '../config/loggerFunctions.js'; import { deletePasswordResetTokensScheduler } from './deletePasswordResetTokensScheduler.js'; +import { getCompletedTranscriptionJobsScheduler } from './getCompletedTranscriptionJobsScheduler.js'; import { markAnalysisEntriesAsCancelledScheduler } from './markAsCancelledAnalysisEntriesScheduler.js'; export const startCronJobs = () => { try { deletePasswordResetTokensScheduler.start(); + + if (process.env.TRANSCRIPTION_ENABLED === true) { + getCompletedTranscriptionJobsScheduler.start(); + } + markAnalysisEntriesAsCancelledScheduler.start(); + console.log('Cron jobs started'); } catch (error) { diff --git a/server/integrations/aws/Transcribe.js b/server/integrations/aws/Transcribe.js new file mode 100644 index 0000000..569f3ee --- /dev/null +++ b/server/integrations/aws/Transcribe.js @@ -0,0 +1,56 @@ +import { + TranscribeClient, + StartTranscriptionJobCommand, + ListTranscriptionJobsCommand, + DeleteTranscriptionJobCommand, +} from '@aws-sdk/client-transcribe'; +import { getS3Object } from './s3.js'; +import { logInfo } from '../../config/loggerFunctions.js'; + +const transcribeClient = new TranscribeClient({ region: process.env.S3_REGION }); + +export const requestAnalysisEntryTranscriptionToAWSTranscribe = async (transcriptionRequest) => { + const command = new StartTranscriptionJobCommand({ + TranscriptionJobName: transcriptionRequest.analysisEntryId, + LanguageCode: transcriptionRequest.languageCode, + Media: { + MediaFileUri: `s3://${process.env.S3_BUCKET}/analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/recording.mp4`, + }, + OutputBucketName: process.env.S3_BUCKET, + OutputKey: `analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/transcription.json`, + }); + + await transcribeClient.send(command); +}; + +export const listCompletedTranscriptionJobsFromAWS = async () => { + const command = new ListTranscriptionJobsCommand({ + Status: 'COMPLETED', + MaxResults: 10, // Ensure memory is not hogged - if more are available, they will be processed in the next iteration + }); + + const completedTranscriptionJobs = await transcribeClient.send(command); + const completedTranscriptionJobsSummary = completedTranscriptionJobs.TranscriptionJobSummaries; // returns an array + + return completedTranscriptionJobsSummary; +}; + +export const fetchSingleTranscriptionJob = async (analysisId, analysisEntryId) => { + const key = `analysis/${analysisId}/${analysisEntryId}/transcription.json`; + + const transcriptionJobResult = await getS3Object(key); + + return transcriptionJobResult; +}; + +export const deleteCompletedTranscriptionJobFromAWS = async (transcriptionJobName) => { + logInfo(`deleting ${transcriptionJobName} from AWS Transcribe`); + const command = new DeleteTranscriptionJobCommand({ + TranscriptionJobName: transcriptionJobName, + }); + + const deletedTranscriptionJobs = await transcribeClient.send(command); + // returns an array + + return deletedTranscriptionJobs; +}; diff --git a/server/integrations/aws/s3.js b/server/integrations/aws/s3.js index 18f5e5c..94c1168 100644 --- a/server/integrations/aws/s3.js +++ b/server/integrations/aws/s3.js @@ -2,16 +2,18 @@ import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const s3client = new S3Client({ - region: process.env.AWS_REGION, - credentials: { // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/modules/credentials.html - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + region: process.env.S3_REGION, // irrelevant since miniIO doesnt takei into account + endpoint: process.env.S3_ENDPOINT, // Container network endpoint + forcePathStyle: true, // Required for MinIO path-style URLs + credentials: { + accessKeyId: process.env.MINIO_ROOT_USER, + secretAccessKey: process.env.MINIO_ROOT_PASSWORD, }, }); export const generateS3GetPresignedUrl = async (key) => { const command = new GetObjectCommand({ - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.S3_BUCKET, Key: key, }); @@ -22,7 +24,7 @@ export const generateS3GetPresignedUrl = async (key) => { export const generateS3PutPresignedUrl = async (key) => { const command = new PutObjectCommand({ - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.S3_BUCKET, Key: key, ContentType: 'video/mp4', }); @@ -33,3 +35,17 @@ export const generateS3PutPresignedUrl = async (key) => { return analysisEntryPutPresignedUrl; }; + +export const getS3Object = async (key) => { + const command = new GetObjectCommand({ + Bucket: process.env.S3_BUCKET, + Key: key, + }); + + const s3Object = await s3client.send(command); + + // Read the response body as a stream and convert to string so it is workable + const responseBody = await s3Object.Body.transformToString(); + + return responseBody; +}; diff --git a/server/models/analysisEntryModel.js b/server/models/analysisEntryModel.js index 4bc7844..9ccb29b 100644 --- a/server/models/analysisEntryModel.js +++ b/server/models/analysisEntryModel.js @@ -43,7 +43,7 @@ export const createAnalysisEntryInDb = async (analysisId) => { return createAnalysisEntryQuery; }; -export const updateAnalysisEntryInDb = async (analysisEntryId, status) => { +export const markAnalysisEntryAsSubmitted = async (analysisEntryId) => { const whereClause = { id: analysisEntryId, }; @@ -51,10 +51,10 @@ export const updateAnalysisEntryInDb = async (analysisEntryId, status) => { const analysisEntryUpdateQuery = await prisma.analysisEntry.update({ where: whereClause, data: { - status: status, + status: 'submitted', }, select: { - status: true, + analysis_id: true, }, }); diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js new file mode 100644 index 0000000..e87cf76 --- /dev/null +++ b/server/models/transcriptionModel.js @@ -0,0 +1,65 @@ +import { PrismaClient } from '../config/generated/prisma/client/index.js'; + +const prisma = new PrismaClient(); + +export const insertTranscriptionRequestInDb = async (transcriptionRequest) => { + await prisma.transcriptionJob.create({ + data: { + analysis_entry_id: transcriptionRequest.analysisEntryId, + status: 'PENDING', + language_code: transcriptionRequest.languageCode, + }, + }); +}; + +export const updateSingleTranscriptionRequestInDb = async (analysisEntryId) => { + const whereClause = { + analysis_entry_id: analysisEntryId, + }; + + await prisma.transcriptionJob.update({ + where: whereClause, + data: { + status: 'IN_PROGRESS', + }, + }); +}; + +export const getSingleTranscriptionJobDetailsFromDb = async (transcriptionJobName) => { + const whereClause = { + analysis_entry_id: transcriptionJobName, + }; + + const transcriptionJobDetailsResult = await prisma.transcriptionJob.findUnique({ + where: whereClause, + select: { + status: true, + AnalysisEntry: { + select: { + analysis_id: true, + }, + }, + }, + }); + + return transcriptionJobDetailsResult; +}; + +export const storeNormalizedTranscriptionInDb = async (transcriptionJobName, normalizedTranscriptionJob, transcriptionJobResult) => { + const whereClause = { + id: transcriptionJobName, + }; + + await prisma.analysisEntry.update({ + where: whereClause, + data: { + full_transcript: transcriptionJobResult.results.transcripts[0].transcript, + transcription_segments: normalizedTranscriptionJob.results.segments, + transcriptionJob: { + update: { + status: 'COMPLETED', + }, + }, + }, + }); +}; diff --git a/server/services/analysisService.js b/server/services/analysisService.js new file mode 100644 index 0000000..4ce2f68 --- /dev/null +++ b/server/services/analysisService.js @@ -0,0 +1,90 @@ +import { logError, logInfo } from '../config/loggerFunctions.js'; +import { + insertTranscriptionRequestInDb, + updateSingleTranscriptionRequestInDb, + getSingleTranscriptionJobDetailsFromDb, + storeNormalizedTranscriptionInDb, +} from '../models/transcriptionModel.js'; +import { + requestAnalysisEntryTranscriptionToAWSTranscribe, deleteCompletedTranscriptionJobFromAWS, + fetchSingleTranscriptionJob, + listCompletedTranscriptionJobsFromAWS, +} from '../integrations/aws/Transcribe.js'; +import { normalizeTranscript } from '../utils/transcription/transcriptionNormalizer.js'; + +export const processTranscriptionRequest = async (transcriptionRequest) => { + try { + await insertTranscriptionRequestInDb(transcriptionRequest); + logInfo('Transcription request stored in DB', transcriptionRequest); + + try { // Handle errors gracefully - errors will be picked up by a cron job if failed + await requestAnalysisEntryTranscriptionToAWSTranscribe(transcriptionRequest); + logInfo('Transcription request sent to AWS Transcribe', transcriptionRequest); + + await updateSingleTranscriptionRequestInDb(transcriptionRequest.analysisEntryId); + logInfo('Transcription request updated in DB', transcriptionRequest); + } catch (error) { + logError('Error requesting transcription to AWS Transcribe', error); + } + } catch (error) { + logError('Error storing transcription request in DB', error); + } +}; + +const processSingleCompletedTranscriptionJob = async (transcriptionJob) => { + logInfo(`Processing completed transcription job: ${transcriptionJob.TranscriptionJobName}`); + + try { + // 1. Get transcription job details from database + const transcriptionJobDetails = await getSingleTranscriptionJobDetailsFromDb(transcriptionJob.TranscriptionJobName); + + // Delete from AWS Transcribe if already processed - Shouldnt happen if AWS Transcribe job deletion is working properly + + if (transcriptionJobDetails.status === 'COMPLETED') { // Handle duplicate entries to avoid normalization reprocessing + logInfo(`Deleting already processed job: ${transcriptionJob.TranscriptionJobName}`); + await deleteCompletedTranscriptionJobFromAWS(transcriptionJob.TranscriptionJobName); + } + + // 2. Construct S3 key and fetch transcription file from AWS + const transcriptionJobResultString = await fetchSingleTranscriptionJob(transcriptionJobDetails.AnalysisEntry.analysis_id, transcriptionJob.TranscriptionJobName); + + // 3. Parse the transcription job result (JSON string to object) + const transcriptionJobResult = JSON.parse(transcriptionJobResultString); + + // 4. Normalize transcription job result + const normalizedTranscriptionJob = await normalizeTranscript(transcriptionJobResult); + + // 5. Store normalized transcript in DB and update status to COMPLETED + await storeNormalizedTranscriptionInDb(transcriptionJob.TranscriptionJobName, normalizedTranscriptionJob, transcriptionJobResult); + + try { + await deleteCompletedTranscriptionJobFromAWS(transcriptionJob.TranscriptionJobName); + } catch (error) { + logError(`Error deleting transcription job ${transcriptionJob.TranscriptionJobName} from AWS Transcribe`, error); + // AWS Transcribe deletion failing is not an issue since it will be caught by a CRON-based retry mechanism + } + + logInfo(`Successfully processed transcription job: ${transcriptionJob.TranscriptionJobName}`); + } catch (error) { + logError(`Error processing transcription job ${transcriptionJob.TranscriptionJobName}`, error); + } +}; + +export const handleCompletedVideoTranscriptionJobs = async () => { + try { + // Get the completed Jobs from AWS Transcribe + logInfo('Retrieving completed transcription jobs'); + const completedTranscriptionJobsSummary = await listCompletedTranscriptionJobsFromAWS(); + + if (completedTranscriptionJobsSummary.length === 0) { + logInfo('no completed transcription jobs available to process'); + return; + } + + for (const transcriptionJob of completedTranscriptionJobsSummary) { + await processSingleCompletedTranscriptionJob(transcriptionJob); + } + } catch (error) { + logError('Error processing transcription jobs', error); + } +}; diff --git a/server/utils/transcription/transcriptionNormalizer.js b/server/utils/transcription/transcriptionNormalizer.js new file mode 100644 index 0000000..500fd11 --- /dev/null +++ b/server/utils/transcription/transcriptionNormalizer.js @@ -0,0 +1,183 @@ +import { logError } from '../../config/loggerFunctions.js'; + +/** + * Converts string numbers to actual numbers + * @param {string|number} value - Value to convert + * @returns {number} Converted number or 0 if invalid + */ +const convertToNumber = (value) => { + if (value === undefined || value === null) { + return 0; + } + + const num = parseFloat(value); + return Number.isNaN(num) ? 0 : num; +}; + +/** + * Groups transcription items into meaningful segments with reduced cluttering + * @param {Array} items - Array of transcription items from AWS Transcribe + * @returns {Array} Array of segments with start_time, end_time, and transcript + */ +const createSegmentsFromItems = (items) => { + if (!items || items.length === 0) { + return []; + } + + const segments = []; + let currentSegment = null; + const SEGMENT_GAP_THRESHOLD = 2.0; // seconds - gap for natural pauses to start new segment + const SEGMENT_MAX_DURATION = 25.0; // seconds - maximum duration to avoid overly long segments + + for (let i = 0; i < items.length; i += 1) { + const item = items[i]; + + // Skip items without alternatives + if (!item.alternatives || item.alternatives.length === 0) { + // Skip this iteration + } else { + const alternative = item.alternatives[0]; + const content = alternative.content || ''; + + // Handle items without timing information (like some punctuation) + if (!item.start_time || !item.end_time) { + if (currentSegment && item.type === 'punctuation') { + currentSegment.transcript += content; + } + } else { + const startTime = convertToNumber(item.start_time); + const endTime = convertToNumber(item.end_time); + + // Start a new segment or continue current segment + if (!currentSegment) { + // Start first segment + currentSegment = { + start_time: startTime, + end_time: endTime, + transcript: content, + }; + } else { + // Check if we should start a new segment based on time gap or max duration + const timeGap = startTime - currentSegment.end_time; + const segmentDuration = startTime - currentSegment.start_time; + + // Start new segment if there's a significant gap OR if we've reached max duration + if (timeGap > SEGMENT_GAP_THRESHOLD || segmentDuration >= SEGMENT_MAX_DURATION) { + // Significant gap or max duration reached - finalize current segment and start new one + segments.push(currentSegment); + currentSegment = { + start_time: startTime, + end_time: endTime, + transcript: content, + }; + } else { + // Continue current segment + // Improved logic for adding spaces around punctuation + const currentText = currentSegment.transcript; + const newContent = content.trim(); + + // Don't add space if current text is empty + if (currentText.length === 0) { + currentSegment.transcript += newContent; + } else { + // Get the last character of current text and first character of new content + const lastChar = currentText[currentText.length - 1]; + const firstChar = newContent[0]; + + // Determine if we need to add space + let shouldAddSpace = false; + + // Add space if: + // 1. Current text doesn't end with punctuation and new content doesn't start with punctuation + // 2. Current text ends with punctuation (except quotes/brackets) and new content starts with a letter/number + // 3. Current text ends with letter/number and new content starts with punctuation (.,!?;:) + if (!/[.,!?;:)\]}'"]$/.test(lastChar) && !/^[.,!?;:([{'"]/.test(firstChar)) { + // Neither ends nor starts with punctuation - add space + shouldAddSpace = true; + } else if (/[.,!?;:)]'?]*$/.test(lastChar) && /^[A-Za-zÁÉÍÓÚÑÜáéíóúñü0-9]/.test(firstChar)) { + // Ends with punctuation and starts with letter/number - add space + shouldAddSpace = true; + } else if (/[A-Za-zÁÉÍÓÚÑÜáéíóúñü0-9]$/.test(lastChar) && /^[.,!?;:]/.test(firstChar)) { + // Ends with letter/number and starts with punctuation - don't add space + shouldAddSpace = false; + } else if (/[)\]}'"]$/.test(lastChar) && /^[A-Za-zÁÉÍÓÚÑÜáéíóúñü0-9]/.test(firstChar)) { + // Ends with closing bracket/quote and starts with letter/number - add space + shouldAddSpace = true; + } + + // Special handling for common Spanish patterns + // Add space after periods followed by capital letters (sentence boundaries) + if (/[.] $/.test(lastChar) && /^[A-ZÁÉÍÓÚÑÜ]/.test(firstChar)) { + shouldAddSpace = true; + } + + // Add space after commas, semicolons, and colons + if (/[,;:]$/.test(lastChar)) { + shouldAddSpace = true; + } + + if (shouldAddSpace) { + currentSegment.transcript += ' '; + } + + currentSegment.transcript += newContent; + } + currentSegment.end_time = endTime; + } + } + } + } + } + + // Add the last segment if it exists + if (currentSegment) { + segments.push(currentSegment); + } + + return segments; +}; + +/** + * Normalizes AWS Transcribe output to the required format + * @param {object} transcript - AWS Transcribe output as parsed object + * @returns {Promise} Normalized transcription data + */ +export const normalizeTranscript = async (transcript) => { + try { + // Handle missing or malformed data + if (!transcript) { + return { + status: 'FAILED', + results: { + segments: [], + }, + }; + } + + // Extract status + const status = transcript.status || 'UNKNOWN'; + + // Extract results section + const results = transcript.results || {}; + + // Extract items array and create segments + const items = results.items || []; + const segments = createSegmentsFromItems(items); + + // Return normalized structure matching the required format + return { + status, + results: { + segments, + }, + }; + } catch (error) { + logError(`Error normalizing transcript: ${error.message}`, error); + return { + status: 'FAILED', + results: { + segments: [], + }, + }; + } +}; diff --git a/swagger.yaml b/swagger.yaml index c76d21a..bc56fb1 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1488,16 +1488,10 @@ paths: schema: type: object required: - - analysisId - - status - analysisEntryId properties: analysisEntryId: $ref: "#/components/schemas/AnalysisEntryId" - analysisEntryStatus: - $ref: "#/components/schemas/AnalysisEntryCompletionStatus" - analysisId: - $ref: "#/components/schemas/AnalysisId" responses: 200: description: Success @@ -1506,8 +1500,6 @@ paths: schema: type: object properties: - success: - $ref: "#/components/schemas/SuccessTrueStatus" message: type: string example: Analysis entry updated successfully