diff --git a/.env.example b/.env.example index 886df5796..227d7d00b 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,11 @@ AUTH_SECRET="" # openssl rand -base64 32 DATABASE_URL="" # Format: "postgresql://postgres:pass@127.0.0.1:5432/comp" RESEND_DOMAIN="" # Domain configured in Resend, e.g. mail.trycomp.ai RESEND_API_KEY="" # API key from Resend for email authentication / invites -RESEND_FROM_MARKETING="Lewis Carhart " -RESEND_FROM_SYSTEM="Comp AI " -RESEND_FROM_DEFAULT="Comp AI " -RESEND_TO_TEST="mail@mail.trycomp.ai" -RESEND_REPLY_TO_MARKETING="lewis@mail.trycomp.ai" +RESEND_FROM_MARKETING="" +RESEND_FROM_SYSTEM="" +RESEND_FROM_DEFAULT="" +RESEND_TO_TEST="" +RESEND_REPLY_TO_MARKETING="" REVALIDATION_SECRET="" # openssl rand -base64 32 NEXT_PUBLIC_PORTAL_URL="http://localhost:3002" # The employee portal uses port 3002 by default diff --git a/.github/workflows/trigger-api-tasks-deploy-main.yml b/.github/workflows/trigger-api-tasks-deploy-main.yml new file mode 100644 index 000000000..57dd23ec2 --- /dev/null +++ b/.github/workflows/trigger-api-tasks-deploy-main.yml @@ -0,0 +1,42 @@ +name: Deploy API to Trigger.dev (dev) +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + deploy: + runs-on: warp-ubuntu-latest-arm64-4x + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: "22.x" # Updated to match Node.js w/ Vercel + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + - name: Clear cache + run: rm -rf node_modules .bun + - name: Install dependencies + run: bun install --frozen-lockfile || bun install --frozen-lockfile --ignore-scripts + - name: Install DB package dependencies + working-directory: ./packages/db + run: bun install --frozen-lockfile --ignore-scripts + - name: Build DB package + working-directory: ./packages/db + run: bun run build + - name: Copy schema to api and generate client + working-directory: ./apps/api + run: | + mkdir -p prisma + cp ../../packages/db/dist/schema.prisma prisma/schema.prisma + bunx prisma generate + - name: 🚀 Deploy Trigger.dev + working-directory: ./apps/api + timeout-minutes: 20 + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + run: bunx trigger.dev@4.0.6 deploy --env staging --log-level debug diff --git a/.github/workflows/trigger-api-tasks-deploy-release.yml b/.github/workflows/trigger-api-tasks-deploy-release.yml new file mode 100644 index 000000000..8ba396896 --- /dev/null +++ b/.github/workflows/trigger-api-tasks-deploy-release.yml @@ -0,0 +1,46 @@ +name: Deploy API to Trigger.dev (prod) + +on: + push: + branches: + - release + +permissions: + contents: read + +jobs: + deploy: + runs-on: warp-ubuntu-latest-arm64-4x + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: "20.x" + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile || bun install --frozen-lockfile --ignore-scripts + - name: Install DB package dependencies + working-directory: ./packages/db + run: bun install --frozen-lockfile --ignore-scripts + + - name: Build DB package + working-directory: ./packages/db + run: bun run build + + - name: Copy schema to api and generate client + working-directory: ./apps/api + run: | + mkdir -p prisma + cp ../../packages/db/dist/schema.prisma prisma/schema.prisma + bunx prisma generate + + - name: 🚀 Deploy Trigger.dev + working-directory: ./apps/api + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + run: bunx trigger.dev@4.0.6 deploy diff --git a/SELF_HOSTING.md b/SELF_HOSTING.md index 858cd7f43..35ea1b239 100644 --- a/SELF_HOSTING.md +++ b/SELF_HOSTING.md @@ -46,7 +46,7 @@ App (`apps/app`): - **APP_AWS_REGION**, **APP_AWS_ACCESS_KEY_ID**, **APP_AWS_SECRET_ACCESS_KEY**, **APP_AWS_BUCKET_NAME**: AWS S3 credentials for file storage (attachments, general uploads). - **APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET**: AWS S3 bucket name specifically for questionnaire file uploads. Required for the Security Questionnaire feature. If not set, users will see an error when trying to parse questionnaires. - **APP_AWS_KNOWLEDGE_BASE_BUCKET**: AWS S3 bucket name specifically for knowledge base documents. Required for the Knowledge Base feature in Security Questionnaire. If not set, users will see an error when trying to upload knowledge base documents. -- **APP_AWS_ORG_ASSETS_BUCKET**: AWS S3 bucket name for organization static assets (e.g., company logos). Required for logo uploads in organization settings. If not set, logo upload will fail. +- **APP_AWS_ORG_ASSETS_BUCKET**: AWS S3 bucket name for organization static assets (e.g., company logos, compliance certificates). Required for logo uploads in organization settings and Trust Portal compliance certificate uploads. If not set, these features will fail. - **OPENAI_API_KEY**: Enables AI features that call OpenAI models. - **UPSTASH_REDIS_REST_URL**, **UPSTASH_REDIS_REST_TOKEN**: Optional Redis (Upstash) used for rate limiting/queues/caching. - **NEXT_PUBLIC_POSTHOG_KEY**, **NEXT_PUBLIC_POSTHOG_HOST**: Client analytics via PostHog; leave unset to disable. @@ -59,6 +59,17 @@ App (`apps/app`): - **GA4_API_SECRET**, **GA4_MEASUREMENT_ID**: Google Analytics 4 server/client tracking. - **NEXT_PUBLIC_API_URL**: Override client API base URL (defaults to same origin). +API (`apps/api`): + +- **APP_AWS_REGION**, **APP_AWS_ACCESS_KEY_ID**, **APP_AWS_SECRET_ACCESS_KEY**, **APP_AWS_BUCKET_NAME**: AWS S3 credentials for file storage (attachments, general uploads). +- **APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET**: AWS S3 bucket name specifically for questionnaire file uploads. Required for the Security Questionnaire feature. +- **APP_AWS_KNOWLEDGE_BASE_BUCKET**: AWS S3 bucket name specifically for knowledge base documents. Required for the Knowledge Base feature in Security Questionnaire. +- **APP_AWS_ORG_ASSETS_BUCKET**: AWS S3 bucket name for organization static assets (e.g., company logos, compliance certificates). Required for Trust Portal compliance certificate uploads and organization logo uploads. If not set, these features will fail. +- **OPENAI_API_KEY**: Enables AI features that call OpenAI models. +- **UPSTASH_VECTOR_REST_URL**, **UPSTASH_VECTOR_REST_TOKEN**: Required for vector database operations (questionnaire auto-answer, SOA auto-fill, knowledge base search). +- **BETTER_AUTH_URL**: URL of the Better Auth instance (usually the same as the app URL). +- **DATABASE_URL**: PostgreSQL database connection string. + Portal (`apps/portal`): - **NEXT_PUBLIC_POSTHOG_KEY**, **NEXT_PUBLIC_POSTHOG_HOST**: Client analytics via PostHog for portal. diff --git a/apps/api/.env.example b/apps/api/.env.example new file mode 100644 index 000000000..ed970d912 --- /dev/null +++ b/apps/api/.env.example @@ -0,0 +1,26 @@ +BASE_URL="http://localhost:3333" +BETTER_AUTH_URL="http://localhost:3000" +PORT="3333" + +APP_AWS_BUCKET_NAME= +APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET= +APP_AWS_KNOWLEDGE_BASE_BUCKET= +APP_AWS_REGION= +APP_AWS_ACCESS_KEY_ID= +APP_AWS_SECRET_ACCESS_KEY= +APP_AWS_ORG_ASSETS_BUCKET= + +DATABASE_URL= + + +# Upstash +UPSTASH_REDIS_REST_URL= +UPSTASH_REDIS_REST_TOKEN= + +UPSTASH_VECTOR_REST_URL= +UPSTASH_VECTOR_REST_TOKEN= + +# Trigger +TRIGGER_SECRET_KEY= + +OPENAI_API_KEY= \ No newline at end of file diff --git a/apps/api/customPrismaExtension.ts b/apps/api/customPrismaExtension.ts new file mode 100644 index 000000000..f1b5991b5 --- /dev/null +++ b/apps/api/customPrismaExtension.ts @@ -0,0 +1,298 @@ +import { binaryForRuntime, BuildContext, BuildExtension, BuildManifest } from '@trigger.dev/build'; +import assert from 'node:assert'; +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { cp, mkdir } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; + +export type PrismaExtensionOptions = { + version?: string; + migrate?: boolean; + directUrlEnvVarName?: string; + /** + * The version of the @trycompai/db package to use + */ + dbPackageVersion?: string; +}; + +type ExtendedBuildContext = BuildContext & { workspaceDir?: string }; +type SchemaResolution = { + path?: string; + searched: string[]; +}; + +export function prismaExtension(options: PrismaExtensionOptions = {}): PrismaExtension { + return new PrismaExtension(options); +} + +export class PrismaExtension implements BuildExtension { + moduleExternals: string[]; + public readonly name = 'PrismaExtension'; + private _resolvedSchemaPath?: string; + + constructor(private options: PrismaExtensionOptions) { + this.moduleExternals = [ + '@prisma/client', + '@prisma/engines', + '@trycompai/db', // Add the published package to externals + ]; + } + + externalsForTarget(target: any) { + if (target === 'dev') { + return []; + } + return this.moduleExternals; + } + + async onBuildStart(context: BuildContext) { + if (context.target === 'dev') { + return; + } + + const resolution = this.tryResolveSchemaPath(context as ExtendedBuildContext); + + if (!resolution.path) { + context.logger.debug( + 'Prisma schema not found during build start, likely before dependencies are installed.', + { searched: resolution.searched }, + ); + return; + } + + this._resolvedSchemaPath = resolution.path; + context.logger.debug(`Resolved prisma schema to ${resolution.path}`); + await this.ensureLocalPrismaClient(context as ExtendedBuildContext, resolution.path); + } + + async onBuildComplete(context: BuildContext, manifest: BuildManifest) { + if (context.target === 'dev') { + return; + } + + if (!this._resolvedSchemaPath || !existsSync(this._resolvedSchemaPath)) { + const resolution = this.tryResolveSchemaPath(context as ExtendedBuildContext); + + if (!resolution.path) { + throw new Error( + [ + 'PrismaExtension could not find the prisma schema. Make sure @trycompai/db is installed', + `with version ${this.options.dbPackageVersion || 'latest'} and that its dist files are built.`, + 'Searched the following locations:', + ...resolution.searched.map((candidate) => ` - ${candidate}`), + ].join('\n'), + ); + } + + this._resolvedSchemaPath = resolution.path; + } + + assert(this._resolvedSchemaPath, 'Resolved schema path is not set'); + const schemaPath = this._resolvedSchemaPath; + + await this.ensureLocalPrismaClient(context as ExtendedBuildContext, schemaPath); + + context.logger.debug('Looking for @prisma/client in the externals', { + externals: manifest.externals, + }); + + const prismaExternal = manifest.externals?.find( + (external) => external.name === '@prisma/client', + ); + const version = prismaExternal?.version ?? this.options.version; + + if (!version) { + throw new Error( + `PrismaExtension could not determine the version of @prisma/client. It's possible that the @prisma/client was not used in the project. If this isn't the case, please provide a version in the PrismaExtension options.`, + ); + } + + context.logger.debug( + `PrismaExtension is generating the Prisma client for version ${version} from @trycompai/db package`, + ); + + const commands: string[] = []; + const env: Record = {}; + + // Copy the prisma schema from the published package to the build output path + const schemaDestinationPath = join(manifest.outputPath, 'prisma', 'schema.prisma'); + const schemaDestinationDir = dirname(schemaDestinationPath); + context.logger.debug( + `Copying the prisma schema from ${schemaPath} to ${schemaDestinationPath}`, + ); + await mkdir(schemaDestinationDir, { recursive: true }); + await cp(schemaPath, schemaDestinationPath); + + // Add prisma generate command to generate the client from the copied schema + commands.push( + `${binaryForRuntime(manifest.runtime)} node_modules/prisma/build/index.js generate --schema=./prisma/schema.prisma`, + ); + + // Only handle migrations if requested + if (this.options.migrate) { + context.logger.debug( + 'Migration support not implemented for published package - please handle migrations separately', + ); + // You could add migration commands here if needed + // commands.push(`${binaryForRuntime(manifest.runtime)} npx prisma migrate deploy`); + } + + // Set up environment variables + env.DATABASE_URL = manifest.deploy.env?.DATABASE_URL; + + if (this.options.directUrlEnvVarName) { + env[this.options.directUrlEnvVarName] = + manifest.deploy.env?.[this.options.directUrlEnvVarName] ?? + process.env[this.options.directUrlEnvVarName]; + if (!env[this.options.directUrlEnvVarName]) { + context.logger.warn( + `prismaExtension could not resolve the ${this.options.directUrlEnvVarName} environment variable. Make sure you add it to your environment variables or provide it as an environment variable to the deploy CLI command. See our docs for more info: https://trigger.dev/docs/deploy-environment-variables`, + ); + } + } else { + env.DIRECT_URL = manifest.deploy.env?.DIRECT_URL; + env.DIRECT_DATABASE_URL = manifest.deploy.env?.DIRECT_DATABASE_URL; + } + + if (!env.DATABASE_URL) { + context.logger.warn( + 'prismaExtension could not resolve the DATABASE_URL environment variable. Make sure you add it to your environment variables. See our docs for more info: https://trigger.dev/docs/deploy-environment-variables', + ); + } + + context.logger.debug('Adding the prisma layer with the following commands', { + commands, + env, + dependencies: { + prisma: version, + '@trycompai/db': this.options.dbPackageVersion || 'latest', + }, + }); + + context.addLayer({ + id: 'prisma', + commands, + dependencies: { + prisma: version, + '@trycompai/db': this.options.dbPackageVersion || 'latest', + }, + build: { + env, + }, + }); + } + + private async ensureLocalPrismaClient( + context: ExtendedBuildContext, + schemaSourcePath: string, + ): Promise { + const schemaDir = resolve(context.workingDir, 'prisma'); + const schemaDestinationPath = resolve(schemaDir, 'schema.prisma'); + + await mkdir(schemaDir, { recursive: true }); + await cp(schemaSourcePath, schemaDestinationPath); + + const clientEntryPoint = resolve(context.workingDir, 'node_modules/.prisma/client/default.js'); + + if (existsSync(clientEntryPoint) && !process.env.TRIGGER_PRISMA_FORCE_GENERATE) { + context.logger.debug('Prisma client already generated locally, skipping regenerate.'); + return; + } + + const prismaBinary = this.resolvePrismaBinary(context.workingDir); + + if (!prismaBinary) { + context.logger.debug( + 'Prisma CLI not available yet, skipping local generate until install finishes.', + ); + return; + } + + context.logger.log('Prisma client missing. Generating before Trigger indexing.'); + await this.runPrismaGenerate(context, prismaBinary, schemaDestinationPath); + } + + private runPrismaGenerate( + context: ExtendedBuildContext, + prismaBinary: string, + schemaPath: string, + ): Promise { + return new Promise((resolvePromise, rejectPromise) => { + const child = spawn(prismaBinary, ['generate', `--schema=${schemaPath}`], { + cwd: context.workingDir, + env: { + ...process.env, + PRISMA_HIDE_UPDATE_MESSAGE: '1', + }, + }); + + child.stdout?.on('data', (data: Buffer) => { + context.logger.debug(data.toString().trim()); + }); + + child.stderr?.on('data', (data: Buffer) => { + context.logger.warn(data.toString().trim()); + }); + + child.on('error', (error) => { + rejectPromise(error); + }); + + child.on('close', (code) => { + if (code === 0) { + resolvePromise(); + } else { + rejectPromise(new Error(`prisma generate exited with code ${code}`)); + } + }); + }); + } + + private resolvePrismaBinary(workingDir: string): string | undefined { + const binDir = resolve(workingDir, 'node_modules', '.bin'); + const executable = process.platform === 'win32' ? 'prisma.cmd' : 'prisma'; + const binaryPath = resolve(binDir, executable); + + if (!existsSync(binaryPath)) { + return undefined; + } + + return binaryPath; + } + + private tryResolveSchemaPath(context: ExtendedBuildContext): SchemaResolution { + const candidates = this.buildSchemaCandidates(context); + const path = candidates.find((candidate) => existsSync(candidate)); + return { path, searched: candidates }; + } + + private buildSchemaCandidates(context: ExtendedBuildContext): string[] { + const candidates = new Set(); + + const addNodeModuleCandidates = (start: string | undefined) => { + if (!start) { + return; + } + + let current = start; + while (true) { + candidates.add(resolve(current, 'node_modules/@trycompai/db/dist/schema.prisma')); + const parent = dirname(current); + if (parent === current) { + break; + } + current = parent; + } + }; + + addNodeModuleCandidates(context.workingDir); + addNodeModuleCandidates(context.workspaceDir); + + candidates.add(resolve(context.workingDir, '../../packages/db/dist/schema.prisma')); + candidates.add(resolve(context.workingDir, '../packages/db/dist/schema.prisma')); + + return Array.from(candidates); + } +} + + diff --git a/apps/api/package.json b/apps/api/package.json index 640a1c08e..cdc922874 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,6 +5,10 @@ "author": "", "dependencies": { "@ai-sdk/openai": "^2.0.65", + "@prisma/instrumentation": "^6.13.0", + "@trigger.dev/build": "4.0.6", + "@trigger.dev/sdk": "4.0.6", + "@upstash/vector": "^1.2.2", "@aws-sdk/client-s3": "^3.859.0", "ai": "^5.0.60", "@aws-sdk/s3-request-presigner": "^3.859.0", @@ -24,6 +28,7 @@ "dotenv": "^17.2.3", "jose": "^6.0.12", "jspdf": "^3.0.3", + "mammoth": "^1.8.0", "nanoid": "^5.1.6", "pdf-lib": "^1.17.1", "prisma": "^6.13.0", @@ -33,6 +38,7 @@ "resend": "^6.4.2", "rxjs": "^7.8.1", "swagger-ui-express": "^5.0.1", + "xlsx": "^0.18.5", "zod": "^4.0.14" }, "devDependencies": { @@ -44,6 +50,7 @@ "@types/archiver": "^6.0.3", "@types/express": "^5.0.0", "@types/jest": "^30.0.0", + "@types/multer": "^1.4.12", "@types/node": "^24.0.3", "@types/supertest": "^6.0.2", "@types/swagger-ui-express": "^4.1.8", @@ -87,7 +94,10 @@ "db:generate": "bun run db:getschema && bunx prisma generate", "db:getschema": "node ../../packages/db/scripts/combine-schemas.js && cp ../../packages/db/dist/schema.prisma prisma/schema.prisma", "db:migrate": "cd ../../packages/db && bunx prisma migrate dev && cd ../../apps/api", - "dev": "nest start --watch", + "deploy:trigger-prod": "npx trigger.dev@4.0.6 deploy", + "dev": "bunx concurrently --kill-others --names \"nest,trigger\" --prefix-colors \"green,blue\" \"nest start --watch\" \"bunx trigger.dev@4.0.6 dev\"", + "dev:nest": "nest start --watch", + "dev:trigger": "bunx trigger.dev@4.0.6 dev", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "prebuild": "bun run db:generate", diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 22e1f6794..ed53b359e 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -19,6 +19,10 @@ import { VendorsModule } from './vendors/vendors.module'; import { ContextModule } from './context/context.module'; import { TrustPortalModule } from './trust-portal/trust-portal.module'; import { TaskTemplateModule } from './framework-editor/task-template/task-template.module'; +import { QuestionnaireModule } from './questionnaire/questionnaire.module'; +import { VectorStoreModule } from './vector-store/vector-store.module'; +import { KnowledgeBaseModule } from './knowledge-base/knowledge-base.module'; +import { SOAModule } from './soa/soa.module'; @Module({ imports: [ @@ -40,13 +44,16 @@ import { TaskTemplateModule } from './framework-editor/task-template/task-templa DevicesModule, PoliciesModule, DeviceAgentModule, - DevicesModule, AttachmentsModule, TasksModule, CommentsModule, HealthModule, TrustPortalModule, TaskTemplateModule, + QuestionnaireModule, + VectorStoreModule, + KnowledgeBaseModule, + SOAModule, ], controllers: [AppController], providers: [AppService], diff --git a/apps/api/src/app/s3.ts b/apps/api/src/app/s3.ts new file mode 100644 index 000000000..e22e8b6b0 --- /dev/null +++ b/apps/api/src/app/s3.ts @@ -0,0 +1,147 @@ +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { Logger } from '@nestjs/common'; +import '../config/load-env'; + +const logger = new Logger('S3'); + +const APP_AWS_REGION = process.env.APP_AWS_REGION; +const APP_AWS_ACCESS_KEY_ID = process.env.APP_AWS_ACCESS_KEY_ID; +const APP_AWS_SECRET_ACCESS_KEY = process.env.APP_AWS_SECRET_ACCESS_KEY; + +export const BUCKET_NAME = process.env.APP_AWS_BUCKET_NAME; +export const APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET = + process.env.APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET; +export const APP_AWS_KNOWLEDGE_BASE_BUCKET = + process.env.APP_AWS_KNOWLEDGE_BASE_BUCKET; +export const APP_AWS_ORG_ASSETS_BUCKET = process.env.APP_AWS_ORG_ASSETS_BUCKET; + +let s3ClientInstance: S3Client | null = null; + +try { + if ( + !APP_AWS_ACCESS_KEY_ID || + !APP_AWS_SECRET_ACCESS_KEY || + !BUCKET_NAME || + !APP_AWS_REGION + ) { + logger.error( + '[S3] AWS S3 credentials or configuration missing. Check environment variables.', + ); + throw new Error( + 'AWS S3 credentials or configuration missing. Check environment variables.', + ); + } + + s3ClientInstance = new S3Client({ + region: APP_AWS_REGION, + credentials: { + accessKeyId: APP_AWS_ACCESS_KEY_ID, + secretAccessKey: APP_AWS_SECRET_ACCESS_KEY, + }, + }); +} catch (error) { + logger.error( + 'FAILED TO INITIALIZE S3 CLIENT', + error instanceof Error ? error.stack : error, + ); + s3ClientInstance = null; + logger.error( + '[S3] Creating dummy S3 client - file uploads will fail until credentials are fixed', + ); +} + +export const s3Client = s3ClientInstance; + +function isValidS3Host(host: string): boolean { + const normalizedHost = host.toLowerCase(); + + if (!normalizedHost.endsWith('.amazonaws.com')) { + return false; + } + + return /^([\w.-]+\.)?(s3|s3-[\w-]+|s3-website[\w.-]+|s3-accesspoint|s3-control)(\.[\w-]+)?\.amazonaws\.com$/.test( + normalizedHost, + ); +} + +export function extractS3KeyFromUrl(url: string): string { + if (!url || typeof url !== 'string') { + throw new Error('Invalid input: URL must be a non-empty string'); + } + + let parsedUrl: URL | null = null; + try { + parsedUrl = new URL(url); + } catch { + // not a URL, continue + } + + if (parsedUrl) { + if (!isValidS3Host(parsedUrl.host)) { + throw new Error('Invalid URL: Not a valid S3 endpoint'); + } + + const key = decodeURIComponent(parsedUrl.pathname.substring(1)); + + if (key.includes('../') || key.includes('..\\')) { + throw new Error('Invalid S3 key: Path traversal detected'); + } + + if (!key) { + throw new Error('Invalid S3 key: Key cannot be empty'); + } + + return key; + } + + // Reject inputs that look like URLs or domains but weren't parsed as valid S3 URLs above + // This catches malformed URLs and prevents URL injection attacks + const lowerInput = url.toLowerCase(); + if (lowerInput.includes('://')) { + throw new Error('Invalid input: Malformed URL detected'); + } + + // Check for domain-like patterns (e.g., "example.com", "sub.example.com") + // S3 keys should not contain domain patterns + const domainPattern = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*\.[a-z]{2,}(\/|$)/i; + if (domainPattern.test(url)) { + throw new Error('Invalid input: Domain-like pattern detected in S3 key'); + } + + if (url.includes('../') || url.includes('..\\')) { + throw new Error('Invalid S3 key: Path traversal detected'); + } + + const key = url.startsWith('/') ? url.substring(1) : url; + + if (!key) { + throw new Error('Invalid S3 key: Key cannot be empty'); + } + + return key; +} + +export async function getFleetAgent({ + os, +}: { + os: 'macos' | 'windows' | 'linux'; +}) { + if (!s3Client) { + throw new Error('S3 client not configured'); + } + + const fleetBucketName = process.env.FLEET_AGENT_BUCKET_NAME; + const fleetAgentFileName = 'Comp AI Agent-1.0.0-arm64.dmg'; + + if (!fleetBucketName) { + throw new Error('FLEET_AGENT_BUCKET_NAME is not defined.'); + } + + const getFleetAgentCommand = new GetObjectCommand({ + Bucket: fleetBucketName, + Key: `${os}/${fleetAgentFileName}`, + }); + + const response = await s3Client.send(getFleetAgentCommand); + return response.Body; +} diff --git a/apps/api/src/attachments/attachments.service.ts b/apps/api/src/attachments/attachments.service.ts index 8dcf4b0ee..76be8bb16 100644 --- a/apps/api/src/attachments/attachments.service.ts +++ b/apps/api/src/attachments/attachments.service.ts @@ -27,7 +27,10 @@ export class AttachmentsService { // Safe to access environment variables directly since they're validated this.bucketName = process.env.APP_AWS_BUCKET_NAME!; - if (!process.env.APP_AWS_ACCESS_KEY_ID || !process.env.APP_AWS_SECRET_ACCESS_KEY) { + if ( + !process.env.APP_AWS_ACCESS_KEY_ID || + !process.env.APP_AWS_SECRET_ACCESS_KEY + ) { console.warn('AWS credentials are missing, S3 client may fail'); } diff --git a/apps/api/src/config/load-env.ts b/apps/api/src/config/load-env.ts new file mode 100644 index 000000000..65f442948 --- /dev/null +++ b/apps/api/src/config/load-env.ts @@ -0,0 +1,36 @@ +import { config } from 'dotenv'; +import { existsSync } from 'fs'; +import path from 'path'; + +let envLoaded = false; + +const searchPaths = [ + // When compiled to dist/src (Nest build output) + path.join(__dirname, '..', '..', '.env'), + // When running with ts-node directly from src + path.join(__dirname, '..', '.env'), + // Fallback to current working directory + path.join(process.cwd(), '.env'), +]; + +function loadEnv(): void { + for (const envPath of searchPaths) { + if (existsSync(envPath)) { + config({ path: envPath, override: true }); + envLoaded = true; + return; + } + } + envLoaded = true; +} + +if (!envLoaded) { + loadEnv(); +} + +export function ensureEnvLoaded(): void { + if (!envLoaded) { + loadEnv(); + } +} + diff --git a/apps/api/src/knowledge-base/dto/delete-all-manual-answers.dto.ts b/apps/api/src/knowledge-base/dto/delete-all-manual-answers.dto.ts new file mode 100644 index 000000000..f67de1646 --- /dev/null +++ b/apps/api/src/knowledge-base/dto/delete-all-manual-answers.dto.ts @@ -0,0 +1,6 @@ +import { IsString } from 'class-validator'; + +export class DeleteAllManualAnswersDto { + @IsString() + organizationId!: string; +} diff --git a/apps/api/src/knowledge-base/dto/delete-document.dto.ts b/apps/api/src/knowledge-base/dto/delete-document.dto.ts new file mode 100644 index 000000000..ca6b4421f --- /dev/null +++ b/apps/api/src/knowledge-base/dto/delete-document.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class DeleteDocumentDto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; +} diff --git a/apps/api/src/knowledge-base/dto/delete-manual-answer.dto.ts b/apps/api/src/knowledge-base/dto/delete-manual-answer.dto.ts new file mode 100644 index 000000000..6cf6c3646 --- /dev/null +++ b/apps/api/src/knowledge-base/dto/delete-manual-answer.dto.ts @@ -0,0 +1,6 @@ +import { IsString } from 'class-validator'; + +export class DeleteManualAnswerDto { + @IsString() + organizationId!: string; +} diff --git a/apps/api/src/knowledge-base/dto/get-document-url.dto.ts b/apps/api/src/knowledge-base/dto/get-document-url.dto.ts new file mode 100644 index 000000000..01439af68 --- /dev/null +++ b/apps/api/src/knowledge-base/dto/get-document-url.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class GetDocumentUrlDto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; +} diff --git a/apps/api/src/knowledge-base/dto/process-documents.dto.ts b/apps/api/src/knowledge-base/dto/process-documents.dto.ts new file mode 100644 index 000000000..150f4d2b5 --- /dev/null +++ b/apps/api/src/knowledge-base/dto/process-documents.dto.ts @@ -0,0 +1,12 @@ +import { IsArray, IsString, MinLength, ArrayMinSize } from 'class-validator'; + +export class ProcessDocumentsDto { + @IsString() + organizationId!: string; + + @IsArray() + @ArrayMinSize(1, { message: 'At least one document ID is required' }) + @IsString({ each: true }) + @MinLength(1, { each: true }) + documentIds!: string[]; +} diff --git a/apps/api/src/knowledge-base/dto/upload-document.dto.ts b/apps/api/src/knowledge-base/dto/upload-document.dto.ts new file mode 100644 index 000000000..524052117 --- /dev/null +++ b/apps/api/src/knowledge-base/dto/upload-document.dto.ts @@ -0,0 +1,19 @@ +import { IsOptional, IsString } from 'class-validator'; + +export class UploadDocumentDto { + @IsString() + organizationId!: string; + + @IsString() + fileName!: string; + + @IsString() + fileType!: string; + + @IsString() + fileData!: string; // base64 encoded + + @IsOptional() + @IsString() + description?: string; +} diff --git a/apps/api/src/knowledge-base/knowledge-base.controller.ts b/apps/api/src/knowledge-base/knowledge-base.controller.ts new file mode 100644 index 000000000..f23c7a7f9 --- /dev/null +++ b/apps/api/src/knowledge-base/knowledge-base.controller.ts @@ -0,0 +1,237 @@ +import { + Controller, + Get, + Post, + Body, + Param, + Query, + HttpCode, + HttpStatus, +} from '@nestjs/common'; +import { + ApiTags, + ApiOperation, + ApiOkResponse, + ApiConsumes, +} from '@nestjs/swagger'; +import { KnowledgeBaseService } from './knowledge-base.service'; +import { UploadDocumentDto } from './dto/upload-document.dto'; +import { DeleteDocumentDto } from './dto/delete-document.dto'; +import { GetDocumentUrlDto } from './dto/get-document-url.dto'; +import { ProcessDocumentsDto } from './dto/process-documents.dto'; +import { DeleteManualAnswerDto } from './dto/delete-manual-answer.dto'; +import { DeleteAllManualAnswersDto } from './dto/delete-all-manual-answers.dto'; + +@Controller({ path: 'knowledge-base', version: '1' }) +@ApiTags('Knowledge Base') +export class KnowledgeBaseController { + constructor(private readonly knowledgeBaseService: KnowledgeBaseService) {} + + @Get('documents') + @ApiOperation({ + summary: 'List all knowledge base documents for an organization', + }) + @ApiOkResponse({ + description: 'List of knowledge base documents', + schema: { + type: 'array', + items: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + description: { type: 'string', nullable: true }, + s3Key: { type: 'string' }, + fileType: { type: 'string' }, + fileSize: { type: 'number' }, + processingStatus: { + type: 'string', + enum: ['pending', 'processing', 'completed', 'failed'], + }, + createdAt: { type: 'string', format: 'date-time' }, + updatedAt: { type: 'string', format: 'date-time' }, + }, + }, + }, + }) + async listDocuments(@Query('organizationId') organizationId: string) { + return this.knowledgeBaseService.listDocuments(organizationId); + } + + @Post('documents/upload') + @ApiOperation({ summary: 'Upload a knowledge base document' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document uploaded successfully', + schema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + s3Key: { type: 'string' }, + }, + }, + }) + async uploadDocument(@Body() dto: UploadDocumentDto) { + return this.knowledgeBaseService.uploadDocument(dto); + } + + @Post('documents/:documentId/download') + @ApiOperation({ + summary: 'Get a signed download URL for a knowledge base document', + }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Signed download URL generated', + schema: { + type: 'object', + properties: { + signedUrl: { type: 'string' }, + fileName: { type: 'string' }, + }, + }, + }) + async getDownloadUrl( + @Param('documentId') documentId: string, + @Body() dto: Omit, + ) { + return this.knowledgeBaseService.getDownloadUrl({ + ...dto, + documentId, + }); + } + + @Post('documents/:documentId/view') + @ApiOperation({ + summary: 'Get a signed view URL for a knowledge base document', + }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Signed view URL generated', + schema: { + type: 'object', + properties: { + signedUrl: { type: 'string' }, + fileName: { type: 'string' }, + fileType: { type: 'string' }, + viewableInBrowser: { type: 'boolean' }, + }, + }, + }) + async getViewUrl( + @Param('documentId') documentId: string, + @Body() dto: Omit, + ) { + return this.knowledgeBaseService.getViewUrl({ + ...dto, + documentId, + }); + } + + @Post('documents/:documentId/delete') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Delete a knowledge base document' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document deleted successfully', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + vectorDeletionRunId: { type: 'string', nullable: true }, + publicAccessToken: { type: 'string', nullable: true }, + }, + }, + }) + async deleteDocument( + @Param('documentId') documentId: string, + @Body() dto: Omit, + ) { + return this.knowledgeBaseService.deleteDocument({ + ...dto, + documentId, + }); + } + + @Post('documents/process') + @ApiOperation({ summary: 'Trigger processing of knowledge base documents' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document processing triggered', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + runId: { type: 'string' }, + publicAccessToken: { type: 'string', nullable: true }, + message: { type: 'string' }, + }, + }, + }) + async processDocuments(@Body() dto: ProcessDocumentsDto) { + return this.knowledgeBaseService.processDocuments(dto); + } + + @Post('runs/:runId/token') + @ApiOperation({ summary: 'Create a public access token for a Trigger.dev run' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Public access token created', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + token: { type: 'string', nullable: true }, + }, + }, + }) + async createRunToken(@Param('runId') runId: string) { + const token = await this.knowledgeBaseService.createRunReadToken(runId); + return { + success: !!token, + token, + }; + } + + @Post('manual-answers/:manualAnswerId/delete') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Delete a manual answer' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Manual answer deleted successfully', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + error: { type: 'string', nullable: true }, + }, + }, + }) + async deleteManualAnswer( + @Param('manualAnswerId') manualAnswerId: string, + @Body() dto: DeleteManualAnswerDto, + ) { + return this.knowledgeBaseService.deleteManualAnswer({ + ...dto, + manualAnswerId, + }); + } + + @Post('manual-answers/delete-all') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Delete all manual answers for an organization' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'All manual answers deleted successfully', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + error: { type: 'string', nullable: true }, + }, + }, + }) + async deleteAllManualAnswers(@Body() dto: DeleteAllManualAnswersDto) { + return this.knowledgeBaseService.deleteAllManualAnswers(dto); + } +} diff --git a/apps/api/src/knowledge-base/knowledge-base.module.ts b/apps/api/src/knowledge-base/knowledge-base.module.ts new file mode 100644 index 000000000..9742370d9 --- /dev/null +++ b/apps/api/src/knowledge-base/knowledge-base.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { KnowledgeBaseController } from './knowledge-base.controller'; +import { KnowledgeBaseService } from './knowledge-base.service'; + +@Module({ + controllers: [KnowledgeBaseController], + providers: [KnowledgeBaseService], +}) +export class KnowledgeBaseModule {} diff --git a/apps/api/src/knowledge-base/knowledge-base.service.ts b/apps/api/src/knowledge-base/knowledge-base.service.ts new file mode 100644 index 000000000..cebd047ec --- /dev/null +++ b/apps/api/src/knowledge-base/knowledge-base.service.ts @@ -0,0 +1,355 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { db } from '@db'; +import { tasks, auth } from '@trigger.dev/sdk'; +import { UploadDocumentDto } from './dto/upload-document.dto'; +import { DeleteDocumentDto } from './dto/delete-document.dto'; +import { GetDocumentUrlDto } from './dto/get-document-url.dto'; +import { ProcessDocumentsDto } from './dto/process-documents.dto'; +import { DeleteManualAnswerDto } from './dto/delete-manual-answer.dto'; +import { DeleteAllManualAnswersDto } from './dto/delete-all-manual-answers.dto'; +import { processKnowledgeBaseDocumentTask } from '@/vector-store/jobs/process-knowledge-base-document'; +import { processKnowledgeBaseDocumentsOrchestratorTask } from '@/vector-store/jobs/process-knowledge-base-documents-orchestrator'; +import { deleteKnowledgeBaseDocumentTask } from '@/vector-store/jobs/delete-knowledge-base-document'; +import { deleteManualAnswerTask } from '@/vector-store/jobs/delete-manual-answer'; +import { deleteAllManualAnswersOrchestratorTask } from '@/vector-store/jobs/delete-all-manual-answers-orchestrator'; +import { isViewableInBrowser } from './utils/constants'; +import { + uploadToS3, + generateDownloadUrl, + generateViewUrl, + deleteFromS3, +} from './utils/s3-operations'; + +@Injectable() +export class KnowledgeBaseService { + private readonly logger = new Logger(KnowledgeBaseService.name); + + async listDocuments(organizationId: string) { + return db.knowledgeBaseDocument.findMany({ + where: { organizationId }, + select: { + id: true, + name: true, + description: true, + s3Key: true, + fileType: true, + fileSize: true, + processingStatus: true, + createdAt: true, + updatedAt: true, + }, + orderBy: { createdAt: 'desc' }, + }); + } + + async uploadDocument(dto: UploadDocumentDto) { + // Upload to S3 + const { s3Key, fileSize } = await uploadToS3( + dto.organizationId, + dto.fileName, + dto.fileType, + dto.fileData, + ); + + // Create database record + const document = await db.knowledgeBaseDocument.create({ + data: { + name: dto.fileName, + description: dto.description || null, + s3Key, + fileType: dto.fileType, + fileSize, + organizationId: dto.organizationId, + processingStatus: 'pending', + }, + }); + + return { + id: document.id, + name: document.name, + s3Key: document.s3Key, + }; + } + + async getDownloadUrl(dto: GetDocumentUrlDto) { + const document = await this.findDocument(dto.documentId, dto.organizationId); + + const { signedUrl } = await generateDownloadUrl(document.s3Key, document.name); + + return { + signedUrl, + fileName: document.name, + }; + } + + async getViewUrl(dto: GetDocumentUrlDto) { + const document = await this.findDocument(dto.documentId, dto.organizationId); + + const { signedUrl } = await generateViewUrl( + document.s3Key, + document.name, + document.fileType, + ); + + return { + signedUrl, + fileName: document.name, + fileType: document.fileType, + viewableInBrowser: isViewableInBrowser(document.fileType), + }; + } + + async deleteDocument(dto: DeleteDocumentDto) { + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: dto.documentId, + organizationId: dto.organizationId, + }, + }); + + if (!document) { + throw new Error('Document not found'); + } + + // Delete embeddings from vector database (async, non-blocking) + const vectorDeletionRunId = await this.triggerVectorDeletion( + document.id, + dto.organizationId, + ); + + // Create public access token for the deletion run + const publicAccessToken = vectorDeletionRunId + ? await this.createRunReadToken(vectorDeletionRunId) + : undefined; + + // Delete from S3 (non-blocking) + const s3Deleted = await deleteFromS3(document.s3Key); + if (!s3Deleted) { + this.logger.warn('Error deleting file from S3', { documentId: document.id }); + } + + // Delete from database + await db.knowledgeBaseDocument.delete({ + where: { id: dto.documentId }, + }); + + return { + success: true, + vectorDeletionRunId, + publicAccessToken, + }; + } + + async processDocuments(dto: ProcessDocumentsDto) { + let runId: string | undefined; + + if (dto.documentIds.length > 1) { + const handle = await tasks.trigger< + typeof processKnowledgeBaseDocumentsOrchestratorTask + >('process-knowledge-base-documents-orchestrator', { + documentIds: dto.documentIds, + organizationId: dto.organizationId, + }); + runId = handle.id; + } else { + const handle = await tasks.trigger< + typeof processKnowledgeBaseDocumentTask + >('process-knowledge-base-document', { + documentId: dto.documentIds[0], + organizationId: dto.organizationId, + }); + runId = handle.id; + } + + // Create public access token for the run + const publicAccessToken = runId + ? await this.createRunReadToken(runId) + : undefined; + + return { + success: true, + runId, + publicAccessToken, + message: + dto.documentIds.length > 1 + ? `Processing ${dto.documentIds.length} documents in parallel...` + : 'Processing document...', + }; + } + + /** + * Creates a public access token for reading a specific run + */ + async createRunReadToken(runId: string): Promise { + try { + const token = await auth.createPublicToken({ + scopes: { + read: { + runs: [runId], + }, + }, + expirationTime: '1hr', + }); + return token; + } catch (error) { + this.logger.warn('Failed to create run read token', { + runId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return undefined; + } + } + + async deleteManualAnswer( + dto: DeleteManualAnswerDto & { manualAnswerId: string }, + ) { + const manualAnswer = await db.securityQuestionnaireManualAnswer.findUnique({ + where: { + id: dto.manualAnswerId, + organizationId: dto.organizationId, + }, + }); + + if (!manualAnswer) { + return { success: false, error: 'Manual answer not found' }; + } + + // Trigger vector DB deletion (async) + await this.triggerManualAnswerVectorDeletion( + dto.manualAnswerId, + dto.organizationId, + ); + + // Delete from main DB + await db.securityQuestionnaireManualAnswer.delete({ + where: { id: dto.manualAnswerId }, + }); + + return { success: true }; + } + + async deleteAllManualAnswers(dto: DeleteAllManualAnswersDto) { + // Get all manual answer IDs before deletion + const manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ + where: { organizationId: dto.organizationId }, + select: { id: true }, + }); + + this.logger.log('Found manual answers to delete', { + organizationId: dto.organizationId, + count: manualAnswers.length, + ids: manualAnswers.map((ma) => ma.id), + }); + + // Trigger orchestrator for batch vector deletion + if (manualAnswers.length > 0) { + await this.triggerBatchManualAnswerDeletion( + dto.organizationId, + manualAnswers.map((ma) => ma.id), + ); + } else { + this.logger.log('No manual answers to delete', { + organizationId: dto.organizationId, + }); + } + + // Delete all from main DB + await db.securityQuestionnaireManualAnswer.deleteMany({ + where: { organizationId: dto.organizationId }, + }); + + return { success: true }; + } + + // Private helper methods + + private async findDocument(documentId: string, organizationId: string) { + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: documentId, + organizationId, + }, + select: { + s3Key: true, + name: true, + fileType: true, + }, + }); + + if (!document) { + throw new Error('Document not found'); + } + + return document; + } + + private async triggerVectorDeletion( + documentId: string, + organizationId: string, + ): Promise { + try { + const handle = await tasks.trigger< + typeof deleteKnowledgeBaseDocumentTask + >('delete-knowledge-base-document-from-vector', { + documentId, + organizationId, + }); + return handle.id; + } catch (error) { + this.logger.warn('Failed to trigger vector deletion task', { + documentId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return undefined; + } + } + + private async triggerManualAnswerVectorDeletion( + manualAnswerId: string, + organizationId: string, + ): Promise { + try { + await tasks.trigger( + 'delete-manual-answer-from-vector', + { manualAnswerId, organizationId }, + ); + this.logger.log('Triggered delete manual answer from vector DB task', { + manualAnswerId, + organizationId, + }); + } catch (error) { + this.logger.warn( + 'Failed to trigger delete manual answer from vector DB task', + { + manualAnswerId, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }, + ); + } + } + + private async triggerBatchManualAnswerDeletion( + organizationId: string, + manualAnswerIds: string[], + ): Promise { + try { + await tasks.trigger( + 'delete-all-manual-answers-orchestrator', + { organizationId, manualAnswerIds }, + ); + this.logger.log('Triggered delete all manual answers orchestrator task', { + organizationId, + count: manualAnswerIds.length, + }); + } catch (error) { + this.logger.warn( + 'Failed to trigger delete all manual answers orchestrator', + { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }, + ); + } + } +} diff --git a/apps/api/src/knowledge-base/utils/constants.ts b/apps/api/src/knowledge-base/utils/constants.ts new file mode 100644 index 000000000..d0f76da60 --- /dev/null +++ b/apps/api/src/knowledge-base/utils/constants.ts @@ -0,0 +1,63 @@ +/** + * Knowledge Base module constants + */ + +// File size limits +export const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; // 10MB + +// Signed URL expiration +export const SIGNED_URL_EXPIRATION_SECONDS = 3600; // 1 hour + +// MIME types that can be viewed inline in browser +export const VIEWABLE_MIME_TYPES = [ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/gif', + 'image/webp', + 'image/svg+xml', + 'text/plain', + 'text/html', + 'text/csv', + 'text/markdown', +]; + +/** + * Checks if a file type can be viewed inline in browser + */ +export function isViewableInBrowser(fileType: string): boolean { + return VIEWABLE_MIME_TYPES.includes(fileType); +} + +/** + * Sanitizes a filename for safe use in S3 keys + */ +export function sanitizeFileName(fileName: string): string { + return fileName.replace(/[^a-zA-Z0-9.-]/g, '_'); +} + +/** + * Sanitizes a filename for use in S3 metadata (ASCII only) + */ +export function sanitizeMetadataFileName(fileName: string): string { + return Buffer.from(fileName, 'utf8') + .toString('ascii') + .replace(/[\x00-\x1F\x7F]/g, '') + .replace(/\?/g, '_') + .trim() + .substring(0, 1024); +} + +/** + * Generates a unique S3 key for a knowledge base document + */ +export function generateS3Key( + organizationId: string, + fileId: string, + sanitizedFileName: string, +): string { + const timestamp = Date.now(); + return `${organizationId}/knowledge-base-documents/${timestamp}-${fileId}-${sanitizedFileName}`; +} + diff --git a/apps/api/src/knowledge-base/utils/s3-operations.ts b/apps/api/src/knowledge-base/utils/s3-operations.ts new file mode 100644 index 000000000..221a281a6 --- /dev/null +++ b/apps/api/src/knowledge-base/utils/s3-operations.ts @@ -0,0 +1,152 @@ +import { + PutObjectCommand, + DeleteObjectCommand, + GetObjectCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { randomBytes } from 'crypto'; +import { s3Client, APP_AWS_KNOWLEDGE_BASE_BUCKET } from '@/app/s3'; +import { + MAX_FILE_SIZE_BYTES, + SIGNED_URL_EXPIRATION_SECONDS, + sanitizeFileName, + sanitizeMetadataFileName, + generateS3Key, +} from './constants'; + +export interface UploadResult { + s3Key: string; + fileSize: number; +} + +export interface SignedUrlResult { + signedUrl: string; +} + +/** + * Validates that S3 is configured + */ +export function validateS3Config(): void { + if (!s3Client) { + throw new Error('S3 client not configured'); + } + + if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { + throw new Error( + 'Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable.', + ); + } +} + +/** + * Uploads a document to S3 + */ +export async function uploadToS3( + organizationId: string, + fileName: string, + fileType: string, + fileData: string, +): Promise { + validateS3Config(); + + // Convert base64 to buffer + const fileBuffer = Buffer.from(fileData, 'base64'); + + // Validate file size + if (fileBuffer.length > MAX_FILE_SIZE_BYTES) { + throw new Error( + `File exceeds the ${MAX_FILE_SIZE_BYTES / (1024 * 1024)}MB limit`, + ); + } + + // Generate unique file key + const fileId = randomBytes(16).toString('hex'); + const sanitized = sanitizeFileName(fileName); + const s3Key = generateS3Key(organizationId, fileId, sanitized); + + // Upload to S3 + const putCommand = new PutObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET!, + Key: s3Key, + Body: fileBuffer, + ContentType: fileType, + Metadata: { + originalFileName: sanitizeMetadataFileName(fileName), + organizationId, + }, + }); + + await s3Client!.send(putCommand); + + return { + s3Key, + fileSize: fileBuffer.length, + }; +} + +/** + * Generates a signed URL for downloading a document + */ +export async function generateDownloadUrl( + s3Key: string, + fileName: string, +): Promise { + validateS3Config(); + + const command = new GetObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET!, + Key: s3Key, + ResponseContentDisposition: `attachment; filename="${encodeURIComponent(fileName)}"`, + }); + + const signedUrl = await getSignedUrl(s3Client!, command, { + expiresIn: SIGNED_URL_EXPIRATION_SECONDS, + }); + + return { signedUrl }; +} + +/** + * Generates a signed URL for viewing a document in browser + */ +export async function generateViewUrl( + s3Key: string, + fileName: string, + fileType: string, +): Promise { + validateS3Config(); + + const command = new GetObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET!, + Key: s3Key, + ResponseContentDisposition: `inline; filename="${encodeURIComponent(fileName)}"`, + ResponseContentType: fileType || 'application/octet-stream', + }); + + const signedUrl = await getSignedUrl(s3Client!, command, { + expiresIn: SIGNED_URL_EXPIRATION_SECONDS, + }); + + return { signedUrl }; +} + +/** + * Deletes a document from S3 + * Returns true if successful, false if error (non-throwing) + */ +export async function deleteFromS3(s3Key: string): Promise { + try { + validateS3Config(); + + const deleteCommand = new DeleteObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET!, + Key: s3Key, + }); + + await s3Client!.send(deleteCommand); + return true; + } catch { + return false; + } +} + diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 44e218727..85476644d 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -1,27 +1,13 @@ +import './config/load-env'; import type { INestApplication } from '@nestjs/common'; import { ValidationPipe, VersioningType } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import type { OpenAPIObject } from '@nestjs/swagger'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import * as express from 'express'; -import { config } from 'dotenv'; import path from 'path'; import { AppModule } from './app.module'; -import { existsSync, mkdirSync, writeFileSync } from 'fs'; - -// Load .env file from apps/api directory before anything else -// This ensures .env values override any shell environment variables -// __dirname in compiled code is dist/src, so go up two levels to apps/api -const envPath = path.join(__dirname, '..', '..', '.env'); -if (existsSync(envPath)) { - config({ path: envPath, override: true }); -} else { - // Fallback: try current working directory (when run from apps/api) - const cwdEnvPath = path.join(process.cwd(), '.env'); - if (existsSync(cwdEnvPath)) { - config({ path: cwdEnvPath, override: true }); - } -} +import { mkdirSync, writeFileSync, existsSync } from 'fs'; async function bootstrap(): Promise { const app: INestApplication = await NestFactory.create(AppModule); @@ -69,6 +55,7 @@ async function bootstrap(): Promise { .setTitle('API Documentation') .setDescription('The API documentation for this application') .setVersion('1.0') + .addServer('http://localhost:3333', 'Local API Server') .addApiKey( { type: 'apiKey', diff --git a/apps/api/src/policies/dto/ai-suggest-policy.dto.ts b/apps/api/src/policies/dto/ai-suggest-policy.dto.ts index 6c9a8b277..351a83801 100644 --- a/apps/api/src/policies/dto/ai-suggest-policy.dto.ts +++ b/apps/api/src/policies/dto/ai-suggest-policy.dto.ts @@ -4,7 +4,8 @@ import { IsString, IsOptional, IsArray } from 'class-validator'; export class AISuggestPolicyRequestDto { @ApiProperty({ description: 'User instructions about what changes to make to the policy', - example: 'Update the data retention section to specify a 7-year retention period', + example: + 'Update the data retention section to specify a 7-year retention period', }) @IsString() instructions: string; diff --git a/apps/api/src/questionnaire/dto/answer-single-question.dto.ts b/apps/api/src/questionnaire/dto/answer-single-question.dto.ts new file mode 100644 index 000000000..88ba6a107 --- /dev/null +++ b/apps/api/src/questionnaire/dto/answer-single-question.dto.ts @@ -0,0 +1,21 @@ +import { IsInt, IsOptional, IsString, Min } from 'class-validator'; + +export class AnswerSingleQuestionDto { + @IsString() + question!: string; + + @IsInt() + @Min(0) + questionIndex!: number; + + @IsInt() + @Min(1) + totalQuestions!: number; + + @IsString() + organizationId!: string; + + @IsOptional() + @IsString() + questionnaireId?: string; +} diff --git a/apps/api/src/questionnaire/dto/auto-answer.dto.ts b/apps/api/src/questionnaire/dto/auto-answer.dto.ts new file mode 100644 index 000000000..fe2c250f1 --- /dev/null +++ b/apps/api/src/questionnaire/dto/auto-answer.dto.ts @@ -0,0 +1,37 @@ +import { Type } from 'class-transformer'; +import { + IsArray, + IsInt, + IsOptional, + IsString, + Min, + ValidateNested, +} from 'class-validator'; + +class AutoAnswerQuestionDto { + @IsString() + question!: string; + + @IsOptional() + @IsString() + answer?: string | null; + + @IsOptional() + @IsInt() + @Min(0) + _originalIndex?: number; +} + +export class AutoAnswerDto { + @IsString() + organizationId!: string; + + @IsOptional() + @IsString() + questionnaireId?: string; + + @IsArray() + @ValidateNested({ each: true }) + @Type(() => AutoAnswerQuestionDto) + questionsAndAnswers!: AutoAnswerQuestionDto[]; +} diff --git a/apps/api/src/questionnaire/dto/delete-answer.dto.ts b/apps/api/src/questionnaire/dto/delete-answer.dto.ts new file mode 100644 index 000000000..11da01ad2 --- /dev/null +++ b/apps/api/src/questionnaire/dto/delete-answer.dto.ts @@ -0,0 +1,12 @@ +import { IsString } from 'class-validator'; + +export class DeleteAnswerDto { + @IsString() + questionnaireId!: string; + + @IsString() + organizationId!: string; + + @IsString() + questionAnswerId!: string; +} diff --git a/apps/api/src/questionnaire/dto/export-by-id.dto.ts b/apps/api/src/questionnaire/dto/export-by-id.dto.ts new file mode 100644 index 000000000..23645be34 --- /dev/null +++ b/apps/api/src/questionnaire/dto/export-by-id.dto.ts @@ -0,0 +1,12 @@ +import { IsIn, IsString } from 'class-validator'; + +export class ExportByIdDto { + @IsString() + questionnaireId!: string; + + @IsString() + organizationId!: string; + + @IsIn(['xlsx', 'csv', 'pdf']) + format!: 'xlsx' | 'csv' | 'pdf'; +} diff --git a/apps/api/src/questionnaire/dto/export-questionnaire.dto.ts b/apps/api/src/questionnaire/dto/export-questionnaire.dto.ts new file mode 100644 index 000000000..4ae1e35b4 --- /dev/null +++ b/apps/api/src/questionnaire/dto/export-questionnaire.dto.ts @@ -0,0 +1,16 @@ +import { IsIn, IsOptional, IsString } from 'class-validator'; +import { ParseQuestionnaireDto } from './parse-questionnaire.dto'; + +export type QuestionnaireExportFormat = 'pdf' | 'csv' | 'xlsx'; + +export class ExportQuestionnaireDto extends ParseQuestionnaireDto { + @IsString() + organizationId!: string; + + @IsIn(['pdf', 'csv', 'xlsx']) + format!: QuestionnaireExportFormat; + + @IsOptional() + @IsIn(['internal', 'external']) + source?: 'internal' | 'external'; +} diff --git a/apps/api/src/questionnaire/dto/parse-questionnaire-upload.dto.ts b/apps/api/src/questionnaire/dto/parse-questionnaire-upload.dto.ts new file mode 100644 index 000000000..e9f664421 --- /dev/null +++ b/apps/api/src/questionnaire/dto/parse-questionnaire-upload.dto.ts @@ -0,0 +1,11 @@ +import { IsOptional, IsString } from 'class-validator'; + +export class ParseQuestionnaireUploadDto { + @IsOptional() + @IsString() + vendorName?: string; + + @IsOptional() + @IsString() + fileName?: string; +} diff --git a/apps/api/src/questionnaire/dto/parse-questionnaire.dto.ts b/apps/api/src/questionnaire/dto/parse-questionnaire.dto.ts new file mode 100644 index 000000000..af0630fd1 --- /dev/null +++ b/apps/api/src/questionnaire/dto/parse-questionnaire.dto.ts @@ -0,0 +1,17 @@ +import { IsOptional, IsString } from 'class-validator'; + +export class ParseQuestionnaireDto { + @IsString() + fileData!: string; // base64 encoded content + + @IsString() + fileType!: string; // MIME type + + @IsOptional() + @IsString() + fileName?: string; + + @IsOptional() + @IsString() + vendorName?: string; +} diff --git a/apps/api/src/questionnaire/dto/save-answer.dto.ts b/apps/api/src/questionnaire/dto/save-answer.dto.ts new file mode 100644 index 000000000..fa4e69922 --- /dev/null +++ b/apps/api/src/questionnaire/dto/save-answer.dto.ts @@ -0,0 +1,65 @@ +import { + IsArray, + IsIn, + IsInt, + IsOptional, + IsString, + Min, + ValidateNested, +} from 'class-validator'; +import { Type } from 'class-transformer'; + +class AnswerSourceDto { + @IsString() + sourceType!: string; + + @IsOptional() + @IsString() + sourceName?: string; + + @IsOptional() + @IsString() + sourceId?: string; + + @IsOptional() + @IsString() + policyName?: string; + + @IsOptional() + @IsString() + documentName?: string; + + @IsOptional() + @IsInt() + score?: number; +} + +export class SaveAnswerDto { + @IsString() + questionnaireId!: string; + + @IsString() + organizationId!: string; + + @IsOptional() + @IsString() + questionAnswerId?: string; + + @IsOptional() + @IsInt() + @Min(0) + questionIndex?: number; + + @IsOptional() + @IsString() + answer?: string | null; + + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => AnswerSourceDto) + sources?: AnswerSourceDto[]; + + @IsIn(['generated', 'manual']) + status!: 'generated' | 'manual'; +} diff --git a/apps/api/src/questionnaire/dto/upload-and-parse.dto.ts b/apps/api/src/questionnaire/dto/upload-and-parse.dto.ts new file mode 100644 index 000000000..76c400c5c --- /dev/null +++ b/apps/api/src/questionnaire/dto/upload-and-parse.dto.ts @@ -0,0 +1,19 @@ +import { IsIn, IsOptional, IsString } from 'class-validator'; + +export class UploadAndParseDto { + @IsString() + organizationId!: string; + + @IsString() + fileName!: string; + + @IsString() + fileType!: string; + + @IsString() + fileData!: string; // base64 encoded + + @IsOptional() + @IsIn(['internal', 'external']) + source?: 'internal' | 'external'; +} diff --git a/apps/api/src/questionnaire/questionnaire.controller.ts b/apps/api/src/questionnaire/questionnaire.controller.ts new file mode 100644 index 000000000..3eba0981e --- /dev/null +++ b/apps/api/src/questionnaire/questionnaire.controller.ts @@ -0,0 +1,657 @@ +import { + BadRequestException, + Body, + Controller, + Post, + Query, + Res, + UploadedFile, + UseInterceptors, + Logger, +} from '@nestjs/common'; +import type { Response } from 'express'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { + ApiBody, + ApiConsumes, + ApiOkResponse, + ApiProduces, + ApiQuery, + ApiTags, +} from '@nestjs/swagger'; +import { ParseQuestionnaireDto } from './dto/parse-questionnaire.dto'; +import { ExportQuestionnaireDto } from './dto/export-questionnaire.dto'; +import { AnswerSingleQuestionDto } from './dto/answer-single-question.dto'; +import { AutoAnswerDto } from './dto/auto-answer.dto'; +import { SaveAnswerDto } from './dto/save-answer.dto'; +import { DeleteAnswerDto } from './dto/delete-answer.dto'; +import { UploadAndParseDto } from './dto/upload-and-parse.dto'; +import { ExportByIdDto } from './dto/export-by-id.dto'; +import { + QuestionnaireService, + type ParsedQuestionnaireResult, +} from './questionnaire.service'; +import { syncOrganizationEmbeddings, findSimilarContentBatch } from '@/vector-store/lib'; +import { generateAnswerFromContent } from './vendors/answer-question-helpers'; +import { TrustAccessService } from '../trust-portal/trust-access.service'; +import { + createSafeSSESender, + setupSSEHeaders, + sanitizeErrorMessage, +} from '../utils/sse-utils'; + +@ApiTags('Questionnaire') +@Controller({ + path: 'questionnaire', + version: '1', +}) +export class QuestionnaireController { + private readonly logger = new Logger(QuestionnaireController.name); + + constructor( + private readonly questionnaireService: QuestionnaireService, + private readonly trustAccessService: TrustAccessService, + ) {} + + @Post('parse') + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Parsed questionnaire content', + type: Object, + }) + async parseQuestionnaire( + @Body() dto: ParseQuestionnaireDto, + ): Promise { + return this.questionnaireService.parseQuestionnaire(dto); + } + + @Post('answer-single') + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Generated single answer result', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + data: { + type: 'object', + properties: { + questionIndex: { type: 'number' }, + question: { type: 'string' }, + answer: { type: 'string', nullable: true }, + sources: { type: 'array', items: { type: 'object' } }, + error: { type: 'string', nullable: true }, + }, + }, + }, + }, + }) + async answerSingleQuestion(@Body() dto: AnswerSingleQuestionDto) { + const result = await this.questionnaireService.answerSingleQuestion(dto); + return { + success: result.success, + data: { + questionIndex: result.questionIndex, + question: result.question, + answer: result.answer, + sources: result.sources, + error: result.error, + }, + }; + } + + @Post('save-answer') + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Save manual or generated answer', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + error: { type: 'string', nullable: true }, + }, + }, + }) + async saveAnswer(@Body() dto: SaveAnswerDto) { + return this.questionnaireService.saveAnswer(dto); + } + + @Post('delete-answer') + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Delete questionnaire answer', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + error: { type: 'string', nullable: true }, + }, + }, + }) + async deleteAnswer(@Body() dto: DeleteAnswerDto) { + return this.questionnaireService.deleteAnswer(dto); + } + + @Post('export') + @ApiConsumes('application/json') + @ApiProduces( + 'application/pdf', + 'text/csv', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ) + @ApiOkResponse({ + description: 'Export questionnaire by ID to specified format', + }) + async exportById( + @Body() dto: ExportByIdDto, + @Res({ passthrough: true }) res: Response, + ): Promise { + const result = await this.questionnaireService.exportById(dto); + + res.setHeader('Content-Type', result.mimeType); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${result.filename}"`, + ); + + res.send(result.fileBuffer); + } + + @Post('upload-and-parse') + @ApiConsumes('application/json') + @ApiOkResponse({ + description: + 'Upload file, parse questions (no answers), save to DB, return questionnaireId', + schema: { + type: 'object', + properties: { + questionnaireId: { type: 'string' }, + totalQuestions: { type: 'number' }, + }, + }, + }) + async uploadAndParse(@Body() dto: UploadAndParseDto) { + return this.questionnaireService.uploadAndParse(dto); + } + + @Post('upload-and-parse/upload') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + @ApiBody({ + schema: { + type: 'object', + properties: { + file: { + type: 'string', + format: 'binary', + description: 'Questionnaire file (PDF, image, XLSX, CSV, TXT)', + }, + organizationId: { + type: 'string', + description: 'Organization ID', + }, + source: { + type: 'string', + enum: ['internal', 'external'], + default: 'internal', + description: 'Source of the upload', + }, + }, + required: ['file', 'organizationId'], + }, + }) + @ApiOkResponse({ + description: + 'Upload file, parse questions (no answers), save to DB, return questionnaireId', + schema: { + type: 'object', + properties: { + questionnaireId: { type: 'string' }, + totalQuestions: { type: 'number' }, + }, + }, + }) + async uploadAndParseUpload( + @UploadedFile() file: Express.Multer.File, + @Body() + body: { + organizationId: string; + source?: 'internal' | 'external'; + }, + ) { + if (!file) { + throw new BadRequestException('file is required'); + } + if (!body.organizationId) { + throw new BadRequestException('organizationId is required'); + } + + const dto: UploadAndParseDto = { + organizationId: body.organizationId, + fileName: file.originalname, + fileType: file.mimetype, + fileData: file.buffer.toString('base64'), + source: body.source || 'internal', + }; + + return this.questionnaireService.uploadAndParse(dto); + } + + @Post('parse/upload') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + @ApiBody({ + schema: { + type: 'object', + properties: { + file: { + type: 'string', + format: 'binary', + description: 'Questionnaire file (PDF, image, XLSX, CSV, TXT)', + }, + organizationId: { + type: 'string', + description: 'Organization to use for generating answers', + }, + format: { + type: 'string', + enum: ['pdf', 'csv', 'xlsx'], + default: 'xlsx', + description: 'Output format (defaults to XLSX)', + }, + source: { + type: 'string', + enum: ['internal', 'external'], + default: 'internal', + description: + 'Indicates if the request originated from our UI (internal) or trust portal (external).', + }, + }, + required: ['file', 'organizationId'], + }, + }) + @ApiProduces( + 'application/pdf', + 'text/csv', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ) + async parseQuestionnaireUpload( + @UploadedFile() file: Express.Multer.File, + @Body() + body: { + organizationId: string; + format?: 'pdf' | 'csv' | 'xlsx'; + source?: 'internal' | 'external'; + }, + @Res({ passthrough: true }) res: Response, + ): Promise { + if (!file) { + throw new BadRequestException('file is required'); + } + if (!body.organizationId) { + throw new BadRequestException('organizationId is required'); + } + + const dto: ExportQuestionnaireDto = { + fileData: file.buffer.toString('base64'), + fileType: file.mimetype, + organizationId: body.organizationId, + fileName: file.originalname, + vendorName: undefined, + format: body.format || 'xlsx', + source: body.source || 'internal', + }; + + const result = await this.questionnaireService.autoAnswerAndExport(dto); + + res.setHeader('Content-Type', result.mimeType); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${result.filename}"`, + ); + res.setHeader( + 'X-Question-Count', + String(result.questionsAndAnswers.length), + ); + + res.send(result.fileBuffer); + } + + @Post('parse/upload/token') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + @ApiQuery({ + name: 'token', + required: true, + description: 'Trust access token for authentication', + }) + @ApiBody({ + schema: { + type: 'object', + properties: { + file: { + type: 'string', + format: 'binary', + description: 'Questionnaire file (PDF, image, XLSX, CSV, TXT)', + }, + format: { + type: 'string', + enum: ['pdf', 'csv', 'xlsx'], + default: 'xlsx', + description: 'Output format (defaults to XLSX)', + }, + }, + required: ['file'], + }, + }) + @ApiProduces( + 'application/pdf', + 'text/csv', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ) + async parseQuestionnaireUploadByToken( + @UploadedFile() file: Express.Multer.File, + @Query('token') token: string, + @Body() + body: { + format?: 'pdf' | 'csv' | 'xlsx'; + }, + @Res({ passthrough: true }) res: Response, + ): Promise { + if (!file) { + throw new BadRequestException('file is required'); + } + if (!token) { + throw new BadRequestException('token is required'); + } + + // Validate token and get organizationId + const organizationId = + await this.trustAccessService.validateAccessTokenAndGetOrganizationId( + token, + ); + + const dto: ExportQuestionnaireDto = { + fileData: file.buffer.toString('base64'), + fileType: file.mimetype, + organizationId, + fileName: file.originalname, + vendorName: undefined, + format: body.format || 'xlsx', + source: 'external', // Always external for token-based access + }; + + const result = await this.questionnaireService.autoAnswerAndExport(dto); + + res.setHeader('Content-Type', result.mimeType); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${result.filename}"`, + ); + res.setHeader( + 'X-Question-Count', + String(result.questionsAndAnswers.length), + ); + + res.send(result.fileBuffer); + } + + @Post('answers/export') + @ApiConsumes('application/json') + @ApiProduces( + 'application/pdf', + 'text/csv', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ) + async autoAnswerAndExport( + @Body() dto: ExportQuestionnaireDto, + @Res({ passthrough: true }) res: Response, + ): Promise { + const result = await this.questionnaireService.autoAnswerAndExport(dto); + + res.setHeader('Content-Type', result.mimeType); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${result.filename}"`, + ); + res.setHeader( + 'X-Question-Count', + String(result.questionsAndAnswers.length), + ); + + res.send(result.fileBuffer); + } + + @Post('answers/export/upload') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + @ApiBody({ + schema: { + type: 'object', + properties: { + file: { + type: 'string', + format: 'binary', + description: 'Questionnaire file (PDF, image, XLSX, CSV, TXT)', + }, + organizationId: { + type: 'string', + description: 'Organization to use for answer generation', + }, + format: { + type: 'string', + enum: ['pdf', 'csv', 'xlsx'], + default: 'xlsx', + description: 'Output format (defaults to XLSX)', + }, + }, + required: ['file', 'organizationId'], + }, + }) + @ApiProduces( + 'application/pdf', + 'text/csv', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ) + async autoAnswerAndExportUpload( + @UploadedFile() file: Express.Multer.File, + @Body() body: { organizationId: string; format?: 'pdf' | 'csv' | 'xlsx' }, + @Res({ passthrough: true }) res: Response, + ): Promise { + if (!file) { + throw new BadRequestException('file is required'); + } + if (!body.organizationId) { + throw new BadRequestException('organizationId is required'); + } + + const dto: ExportQuestionnaireDto = { + fileData: file.buffer.toString('base64'), + fileType: file.mimetype, + organizationId: body.organizationId, + fileName: file.originalname, + vendorName: undefined, + format: body.format || 'xlsx', + }; + + const result = await this.questionnaireService.autoAnswerAndExport(dto); + + res.setHeader('Content-Type', result.mimeType); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${result.filename}"`, + ); + res.setHeader( + 'X-Question-Count', + String(result.questionsAndAnswers.length), + ); + + res.send(result.fileBuffer); + } + + @Post('auto-answer') + @ApiConsumes('application/json') + @ApiProduces('text/event-stream') + async autoAnswer( + @Body() dto: AutoAnswerDto, + @Res() res: Response, + ): Promise { + setupSSEHeaders(res); + const send = createSafeSSESender(res); + + try { + // Step 1: Sync organization embeddings once + try { + await syncOrganizationEmbeddings(dto.organizationId); + } catch (error) { + this.logger.warn('Failed to sync organization embeddings', { + organizationId: dto.organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + + const questionsToAnswer = dto.questionsAndAnswers + .map((qa, index) => ({ + question: qa.question, + answer: qa.answer ?? null, + index: qa._originalIndex ?? index, + })) + .filter( + (qa) => !qa.answer || (qa.answer && qa.answer.trim().length === 0), + ); + + if (questionsToAnswer.length === 0) { + send({ + type: 'complete', + total: 0, + answered: 0, + answers: [], + }); + return; + } + + send({ + type: 'progress', + total: questionsToAnswer.length, + completed: 0, + remaining: questionsToAnswer.length, + phase: 'searching', + }); + + // Step 2: Batch search - generates all embeddings in ONE API call (saves ~5-10 seconds) + const searchStartTime = Date.now(); + const allSimilarContent = await findSimilarContentBatch( + questionsToAnswer.map((qa) => qa.question), + dto.organizationId, + ); + const searchTime = Date.now() - searchStartTime; + + this.logger.log( + `Batch search completed in ${searchTime}ms for ${questionsToAnswer.length} questions`, + ); + + send({ + type: 'progress', + total: questionsToAnswer.length, + completed: 0, + remaining: questionsToAnswer.length, + phase: 'generating', + searchTimeMs: searchTime, + }); + + const results: Array<{ + questionIndex: number; + question: string; + answer: string | null; + sources?: unknown; + error?: string; + }> = []; + + // Step 3: Generate answers in parallel using pre-fetched content (still streams!) + await Promise.all( + questionsToAnswer.map(async (qa, i) => { + try { + const similarContent = allSimilarContent[i] || []; + const result = await generateAnswerFromContent( + qa.question, + similarContent, + ); + + // Save answer to database if questionnaireId is provided + if (dto.questionnaireId && result.answer) { + try { + await this.questionnaireService.saveGeneratedAnswerPublic({ + questionnaireId: dto.questionnaireId, + questionIndex: qa.index, + answer: result.answer, + sources: result.sources, + }); + } catch (saveError) { + this.logger.warn('Failed to save answer to database', { + questionnaireId: dto.questionnaireId, + questionIndex: qa.index, + error: + saveError instanceof Error + ? saveError.message + : 'Unknown error', + }); + } + } + + send({ + type: 'answer', + questionIndex: qa.index, + question: qa.question, + answer: result.answer, + sources: result.sources, + success: result.answer !== null, + }); + + results.push({ + questionIndex: qa.index, + question: qa.question, + answer: result.answer, + sources: result.sources, + }); + } catch (error) { + const errorPayload = { + questionIndex: qa.index, + question: qa.question, + answer: null, + sources: [], + error: sanitizeErrorMessage(error), + }; + + send({ + type: 'answer', + ...errorPayload, + success: false, + }); + + results.push(errorPayload); + } + }), + ); + + send({ + type: 'complete', + total: questionsToAnswer.length, + answered: results.filter((r) => r.answer).length, + answers: results, + searchTimeMs: searchTime, + }); + } catch (error) { + const safeErrorMessage = sanitizeErrorMessage(error); + this.logger.error('Error in auto-answer stream', { + organizationId: dto.organizationId, + error: safeErrorMessage, + }); + send({ + type: 'error', + error: safeErrorMessage, + }); + } finally { + res.end(); + } + } +} diff --git a/apps/api/src/questionnaire/questionnaire.module.ts b/apps/api/src/questionnaire/questionnaire.module.ts new file mode 100644 index 000000000..9de28e7fb --- /dev/null +++ b/apps/api/src/questionnaire/questionnaire.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { QuestionnaireController } from './questionnaire.controller'; +import { QuestionnaireService } from './questionnaire.service'; +import { TrustPortalModule } from '../trust-portal/trust-portal.module'; + +@Module({ + imports: [TrustPortalModule], + controllers: [QuestionnaireController], + providers: [QuestionnaireService], +}) +export class QuestionnaireModule {} diff --git a/apps/api/src/questionnaire/questionnaire.service.ts b/apps/api/src/questionnaire/questionnaire.service.ts new file mode 100644 index 000000000..2cedf4e11 --- /dev/null +++ b/apps/api/src/questionnaire/questionnaire.service.ts @@ -0,0 +1,507 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { AnswerQuestionResult } from './vendors/answer-question'; +import { answerQuestion } from './vendors/answer-question'; +import { generateAnswerWithRAGBatch } from './vendors/answer-question-helpers'; +import { ParseQuestionnaireDto } from './dto/parse-questionnaire.dto'; +import { + ExportQuestionnaireDto, + type QuestionnaireExportFormat, +} from './dto/export-questionnaire.dto'; +import { AnswerSingleQuestionDto } from './dto/answer-single-question.dto'; +import { SaveAnswerDto } from './dto/save-answer.dto'; +import { DeleteAnswerDto } from './dto/delete-answer.dto'; +import { UploadAndParseDto } from './dto/upload-and-parse.dto'; +import { ExportByIdDto } from './dto/export-by-id.dto'; +import { db, Prisma } from '@db'; +import { syncManualAnswerToVector, syncOrganizationEmbeddings } from '@/vector-store/lib'; + +// Import shared utilities +import { extractContentFromFile, type ContentExtractionLogger } from './utils/content-extractor'; +import { parseQuestionsAndAnswers, type QuestionAnswer as ParsedQA } from './utils/question-parser'; +import { generateExportFile, type ExportFormat } from './utils/export-generator'; +import { + updateAnsweredCount, + persistQuestionnaireResult, + uploadQuestionnaireFile, + saveGeneratedAnswer, + type StorageLogger, +} from './utils/questionnaire-storage'; + +export interface QuestionnaireAnswer { + question: string; + answer: string | null; + sources?: AnswerQuestionResult['sources']; +} + +export interface ParsedQuestionnaireResult { + vendorName?: string; + fileName?: string; + totalQuestions: number; + questionsAndAnswers: QuestionnaireAnswer[]; +} + +export interface QuestionnaireExportResult { + fileBuffer: Buffer; + mimeType: string; + filename: string; + questionsAndAnswers: QuestionnaireAnswer[]; +} + +@Injectable() +export class QuestionnaireService { + private readonly logger = new Logger(QuestionnaireService.name); + + private get contentLogger(): ContentExtractionLogger { + return { + info: (msg, meta) => this.logger.log(msg, meta), + warn: (msg, meta) => this.logger.warn(msg, meta), + error: (msg, meta) => this.logger.error(msg, meta), + }; + } + + private get storageLogger(): StorageLogger { + return { + log: (msg, meta) => this.logger.log(msg, meta), + error: (msg, meta) => this.logger.error(msg, meta), + }; + } + + async parseQuestionnaire( + dto: ParseQuestionnaireDto, + ): Promise { + const content = await extractContentFromFile( + dto.fileData, + dto.fileType, + this.contentLogger, + ); + const questionsAndAnswers = await parseQuestionsAndAnswers(content, this.contentLogger); + + return { + vendorName: dto.vendorName, + fileName: dto.fileName, + totalQuestions: questionsAndAnswers.length, + questionsAndAnswers: this.convertParsedToQuestionnaireAnswers(questionsAndAnswers), + }; + } + + async autoAnswerAndExport( + dto: ExportQuestionnaireDto, + ): Promise { + let uploadInfo: { s3Key: string; fileSize: number } | null = null; + if (dto.fileData) { + uploadInfo = await uploadQuestionnaireFile({ + organizationId: dto.organizationId, + fileName: dto.fileName || dto.vendorName || 'questionnaire', + fileType: dto.fileType, + fileData: dto.fileData, + source: dto.source || 'internal', + }); + } else { + this.logger.warn( + 'No fileData provided for autoAnswerAndExport; original file will not be saved.', + { organizationId: dto.organizationId }, + ); + } + + const parsed = await this.parseQuestionnaire(dto); + const answered = await this.generateAnswersForQuestions( + parsed.questionsAndAnswers, + dto.organizationId, + ); + + const vendorName = + dto.vendorName || dto.fileName || parsed.vendorName || 'questionnaire'; + const exportFile = generateExportFile( + answered.map((a) => ({ question: a.question, answer: a.answer })), + dto.format as ExportFormat, + vendorName, + ); + + await persistQuestionnaireResult( + { + organizationId: dto.organizationId, + fileName: dto.fileName || vendorName, + fileType: dto.fileType, + fileSize: + uploadInfo?.fileSize ?? + (dto.fileData ? Buffer.from(dto.fileData, 'base64').length : 0), + s3Key: uploadInfo?.s3Key ?? null, + questionsAndAnswers: answered, + source: dto.source || 'internal', + }, + this.storageLogger, + ); + + return { + ...exportFile, + questionsAndAnswers: answered, + }; + } + + async uploadAndParse( + dto: UploadAndParseDto, + ): Promise<{ questionnaireId: string; totalQuestions: number }> { + const uploadInfo = await uploadQuestionnaireFile({ + organizationId: dto.organizationId, + fileName: dto.fileName, + fileType: dto.fileType, + fileData: dto.fileData, + source: dto.source || 'internal', + }); + + const content = await extractContentFromFile( + dto.fileData, + dto.fileType, + this.contentLogger, + ); + const questionsAndAnswers = await parseQuestionsAndAnswers(content, this.contentLogger); + + const questionnaireId = await persistQuestionnaireResult( + { + organizationId: dto.organizationId, + fileName: dto.fileName, + fileType: dto.fileType, + fileSize: uploadInfo?.fileSize ?? Buffer.from(dto.fileData, 'base64').length, + s3Key: uploadInfo?.s3Key ?? null, + questionsAndAnswers: questionsAndAnswers.map((qa) => ({ + question: qa.question, + answer: null, + sources: undefined, + })), + source: dto.source || 'internal', + }, + this.storageLogger, + ); + + if (!questionnaireId) { + throw new Error('Failed to save questionnaire'); + } + + return { + questionnaireId, + totalQuestions: questionsAndAnswers.length, + }; + } + + async answerSingleQuestion( + dto: AnswerSingleQuestionDto, + options?: { skipSync?: boolean }, + ): Promise { + const result = await answerQuestion( + { + question: dto.question, + organizationId: dto.organizationId, + questionIndex: dto.questionIndex, + totalQuestions: dto.totalQuestions, + }, + { useMetadata: false, skipSync: options?.skipSync }, + ); + + if (result.success && result.answer && dto.questionnaireId) { + await saveGeneratedAnswer({ + questionnaireId: dto.questionnaireId, + questionIndex: dto.questionIndex, + answer: result.answer, + sources: result.sources, + }); + } + + return result; + } + + async saveAnswer( + dto: SaveAnswerDto, + ): Promise<{ success: boolean; error?: string }> { + if (!dto.questionAnswerId && dto.questionIndex === undefined) { + return { + success: false, + error: 'questionIndex or questionAnswerId is required', + }; + } + + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: dto.questionnaireId, + organizationId: dto.organizationId, + }, + include: { + questions: { + where: dto.questionAnswerId + ? { id: dto.questionAnswerId } + : { questionIndex: dto.questionIndex }, + }, + }, + }); + + if (!questionnaire) { + return { success: false, error: 'Questionnaire not found' }; + } + + let existingQuestion: Awaited< + ReturnType + > = questionnaire.questions[0] ?? null; + let questionIndex = dto.questionIndex; + + if (!existingQuestion && dto.questionAnswerId) { + existingQuestion = await db.questionnaireQuestionAnswer.findUnique({ + where: { + id: dto.questionAnswerId, + questionnaireId: dto.questionnaireId, + }, + }); + } + + if (!existingQuestion && questionIndex !== undefined) { + existingQuestion = await db.questionnaireQuestionAnswer.findFirst({ + where: { + questionnaireId: dto.questionnaireId, + questionIndex, + }, + }); + } + + if (!existingQuestion) { + return { success: false, error: 'Question answer not found' }; + } + + if (questionIndex === undefined) { + questionIndex = existingQuestion.questionIndex; + } + + const normalizedAnswer = dto.answer?.trim() || null; + + await db.questionnaireQuestionAnswer.update({ + where: { id: existingQuestion.id }, + data: { + answer: normalizedAnswer, + status: dto.status === 'generated' ? 'generated' : 'manual', + sources: dto.sources + ? (dto.sources as unknown as Prisma.InputJsonValue) + : Prisma.JsonNull, + generatedAt: dto.status === 'generated' ? new Date() : null, + updatedBy: null, + updatedAt: new Date(), + }, + }); + + // Sync manual answer to vector DB + if ( + dto.status === 'manual' && + normalizedAnswer && + existingQuestion.question && + existingQuestion.question.trim().length > 0 + ) { + try { + const manualAnswer = await db.securityQuestionnaireManualAnswer.upsert({ + where: { + organizationId_question: { + organizationId: dto.organizationId, + question: existingQuestion.question.trim(), + }, + }, + create: { + question: existingQuestion.question.trim(), + answer: normalizedAnswer, + tags: [], + organizationId: dto.organizationId, + sourceQuestionnaireId: dto.questionnaireId, + createdBy: null, + updatedBy: null, + }, + update: { + answer: normalizedAnswer, + sourceQuestionnaireId: dto.questionnaireId, + updatedBy: null, + updatedAt: new Date(), + }, + }); + + await syncManualAnswerToVector(manualAnswer.id, dto.organizationId); + } catch (error) { + this.logger.error('Error saving manual answer to vector DB', { + organizationId: dto.organizationId, + questionIndex, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + + await updateAnsweredCount(dto.questionnaireId); + + return { success: true }; + } + + async exportById( + dto: ExportByIdDto, + ): Promise<{ fileBuffer: Buffer; mimeType: string; filename: string }> { + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: dto.questionnaireId, + organizationId: dto.organizationId, + }, + include: { + questions: { + orderBy: { questionIndex: 'asc' }, + }, + }, + }); + + if (!questionnaire) { + throw new Error('Questionnaire not found'); + } + + const questionsAndAnswers = questionnaire.questions.map((q) => ({ + question: q.question, + answer: q.answer, + })); + + this.logger.log('Exporting questionnaire', { + questionnaireId: dto.questionnaireId, + originalFilename: questionnaire.filename, + format: dto.format, + }); + + return generateExportFile( + questionsAndAnswers, + dto.format as ExportFormat, + questionnaire.filename, + ); + } + + async deleteAnswer( + dto: DeleteAnswerDto, + ): Promise<{ success: boolean; error?: string }> { + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: dto.questionnaireId, + organizationId: dto.organizationId, + }, + }); + + if (!questionnaire) { + return { success: false, error: 'Questionnaire not found' }; + } + + const questionAnswer = await db.questionnaireQuestionAnswer.findUnique({ + where: { + id: dto.questionAnswerId, + questionnaireId: dto.questionnaireId, + }, + }); + + if (!questionAnswer) { + return { success: false, error: 'Question answer not found' }; + } + + await db.questionnaireQuestionAnswer.update({ + where: { id: questionAnswer.id }, + data: { + answer: null, + status: 'untouched', + sources: Prisma.JsonNull, + generatedAt: null, + updatedBy: null, + updatedAt: new Date(), + }, + }); + + await updateAnsweredCount(dto.questionnaireId); + + return { success: true }; + } + + /** + * Public wrapper for saving generated answers (used by controller) + */ + async saveGeneratedAnswerPublic(params: { + questionnaireId: string; + questionIndex: number; + answer: string; + sources?: AnswerQuestionResult['sources']; + }): Promise { + await saveGeneratedAnswer(params); + } + + // Private helper methods + + private convertParsedToQuestionnaireAnswers( + parsed: ParsedQA[], + ): QuestionnaireAnswer[] { + return parsed.map((qa) => ({ + question: qa.question, + answer: qa.answer, + })); + } + + private async generateAnswersForQuestions( + questionsAndAnswers: QuestionnaireAnswer[], + organizationId: string, + ): Promise { + const questionsNeedingAnswers = questionsAndAnswers + .map((qa, index) => ({ ...qa, index })) + .filter((qa) => !qa.answer || qa.answer.trim().length === 0); + + if (questionsNeedingAnswers.length === 0) { + return questionsAndAnswers; + } + + this.logger.log( + `Generating answers for ${questionsNeedingAnswers.length} of ${questionsAndAnswers.length} questions`, + ); + + // Sync organization embeddings before generating answers + try { + await syncOrganizationEmbeddings(organizationId); + } catch (error) { + this.logger.error('Failed to sync organization embeddings', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + + // Use batch processing for efficiency + const startTime = Date.now(); + const questionsToAnswer = questionsNeedingAnswers.map((qa) => qa.question); + + const batchResults = await generateAnswerWithRAGBatch( + questionsToAnswer, + organizationId, + ); + + // Map batch results + const results: AnswerQuestionResult[] = questionsNeedingAnswers.map( + ({ question, index }, i) => ({ + success: batchResults[i]?.answer !== null, + questionIndex: index, + question, + answer: batchResults[i]?.answer ?? null, + sources: batchResults[i]?.sources ?? [], + }), + ); + + const answeredCount = results.filter((r) => r.answer !== null).length; + const totalTime = Date.now() - startTime; + + this.logger.log( + `Batch answer generation completed: ${answeredCount}/${questionsNeedingAnswers.length} answered in ${totalTime}ms`, + ); + + const answeredMap = new Map(); + results.forEach((result) => { + answeredMap.set(result.questionIndex, result); + }); + + return questionsAndAnswers.map((qa, index) => { + const generated = answeredMap.get(index); + if (!generated || !generated.success || !generated.answer) { + return qa; + } + + return { + question: qa.question, + answer: generated.answer, + sources: generated.sources, + }; + }); + } +} diff --git a/apps/api/src/questionnaire/utils/constants.ts b/apps/api/src/questionnaire/utils/constants.ts new file mode 100644 index 000000000..9877e8e3d --- /dev/null +++ b/apps/api/src/questionnaire/utils/constants.ts @@ -0,0 +1,41 @@ +/** + * Shared constants for questionnaire module + */ + +// Chunk sizes for question-aware parsing +export const MAX_CHUNK_SIZE_CHARS = 80_000; +export const MIN_CHUNK_SIZE_CHARS = 5_000; +export const MAX_QUESTIONS_PER_CHUNK = 1; + +// File size limits +export const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; // 10MB + +// LLM Model identifiers +export const PARSING_MODEL = 'gpt-5-mini'; +export const ANSWER_MODEL = 'gpt-4o-mini'; + +// System prompts for answer generation +export const ANSWER_SYSTEM_PROMPT = `You are an expert at answering security and compliance questions for vendor questionnaires. + +Your task is to answer questions based ONLY on the provided context from the organization's policies and documentation. + +CRITICAL RULES: +1. Answer based ONLY on the provided context. Do not make up facts or use general knowledge. +2. If the context does not contain enough information to answer the question, respond with exactly: "N/A - no evidence found" +3. BE CONCISE. Give SHORT, direct answers. Do NOT provide detailed explanations or elaborate unnecessarily. +4. Use enterprise-ready language appropriate for vendor questionnaires. +5. If multiple sources provide information, synthesize them into ONE concise answer. +6. Do not include disclaimers or notes about the source unless specifically relevant. +7. Format your answer as a clear, professional response suitable for a vendor questionnaire. +8. Always write in first person plural (we, our, us) as if speaking on behalf of the organization. +9. Keep answers to 1-3 sentences maximum unless the question explicitly requires more detail.`; + +export const QUESTION_PARSING_SYSTEM_PROMPT = `You parse vendor questionnaires. Return only genuine question text paired with its answer. +- Ignore table headers, column labels, metadata rows, or placeholder words such as "Question", "Company Name", "Department", "Assessment Date", "Name of Assessor". +- A valid question is a meaningful sentence (usually ends with '?' or starts with interrogatives like What/Why/How/When/Where/Is/Are/Do/Does/Can/Will/Should). +- Do not fabricate answers; if no answer is provided, set answer to null. +- Keep the original question wording but trim whitespace.`; + +// Vision extraction prompt for PDFs and images +export const VISION_EXTRACTION_PROMPT = `Extract all text and identify question-answer pairs. Look for columns/sections labeled "Question", "Q", "Answer", "A". Match questions (ending with "?" or starting with What/How/Why/When/Is/Can/Do) to nearby answers. Preserve order. Return only Question → Answer pairs.`; + diff --git a/apps/api/src/questionnaire/utils/content-extractor.ts b/apps/api/src/questionnaire/utils/content-extractor.ts new file mode 100644 index 000000000..bdafece6d --- /dev/null +++ b/apps/api/src/questionnaire/utils/content-extractor.ts @@ -0,0 +1,201 @@ +import { openai } from '@ai-sdk/openai'; +import { generateText } from 'ai'; +import * as XLSX from 'xlsx'; +import { PARSING_MODEL, VISION_EXTRACTION_PROMPT } from './constants'; + +export interface ContentExtractionLogger { + info: (message: string, meta?: Record) => void; + warn: (message: string, meta?: Record) => void; + error: (message: string, meta?: Record) => void; +} + +// Default no-op logger for when no logger is provided +const defaultLogger: ContentExtractionLogger = { + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +/** + * Extracts content from a file based on its MIME type + * Supports: Excel, CSV, text, PDF, and images + */ +export async function extractContentFromFile( + fileData: string, + fileType: string, + logger: ContentExtractionLogger = defaultLogger, +): Promise { + const fileBuffer = Buffer.from(fileData, 'base64'); + + // Handle Excel files (.xlsx, .xls) + if (isExcelFile(fileType)) { + return extractFromExcel(fileBuffer, fileType, logger); + } + + // Handle CSV files + if (isCsvFile(fileType)) { + return extractFromCsv(fileBuffer); + } + + // Handle plain text files + if (isTextFile(fileType)) { + return fileBuffer.toString('utf-8'); + } + + // Handle Word documents - not directly supported + if (isWordDocument(fileType)) { + throw new Error( + 'Word documents (.docx) are best converted to PDF or image format for parsing. Alternatively, use a URL to view the document.', + ); + } + + // For images and PDFs, use OpenAI vision API + if (isImageOrPdf(fileType)) { + return extractFromVision(fileData, fileType, logger); + } + + throw new Error( + `Unsupported file type: ${fileType}. Supported formats: PDF, images (PNG, JPG, etc.), Excel (.xlsx, .xls), CSV, text files (.txt).`, + ); +} + +// File type detection helpers +function isExcelFile(fileType: string): boolean { + return ( + fileType === 'application/vnd.ms-excel' || + fileType === + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || + fileType === 'application/vnd.ms-excel.sheet.macroEnabled.12' + ); +} + +function isCsvFile(fileType: string): boolean { + return fileType === 'text/csv' || fileType === 'text/comma-separated-values'; +} + +function isTextFile(fileType: string): boolean { + return fileType === 'text/plain' || fileType.startsWith('text/'); +} + +function isWordDocument(fileType: string): boolean { + return ( + fileType === 'application/msword' || + fileType === + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ); +} + +function isImageOrPdf(fileType: string): boolean { + return fileType.startsWith('image/') || fileType === 'application/pdf'; +} + +// Content extraction functions +function extractFromExcel( + fileBuffer: Buffer, + fileType: string, + logger: ContentExtractionLogger, +): string { + const excelStartTime = Date.now(); + const fileSizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2); + + logger.info('Processing Excel file', { fileType, fileSizeMB }); + + const workbook = XLSX.read(fileBuffer, { type: 'buffer' }); + const sheets: string[] = []; + + for (const sheetName of workbook.SheetNames) { + const worksheet = workbook.Sheets[sheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet, { + header: 1, + defval: '', + }); + + const sheetText = (jsonData as unknown[][]) + .map((row) => { + if (Array.isArray(row)) { + return row + .filter((cell) => cell !== null && cell !== undefined && cell !== '') + .join(' | '); + } + return String(row); + }) + .filter((line) => line.trim() !== '') + .join('\n'); + + if (sheetText.trim()) { + sheets.push(`Sheet: ${sheetName}\n${sheetText}`); + } + } + + const extractionTime = ((Date.now() - excelStartTime) / 1000).toFixed(2); + logger.info('Excel file processed', { + fileSizeMB, + totalSheets: workbook.SheetNames.length, + extractedLength: sheets.join('\n\n').length, + extractionTimeSeconds: extractionTime, + }); + + return sheets.join('\n\n'); +} + +function extractFromCsv(fileBuffer: Buffer): string { + const text = fileBuffer.toString('utf-8'); + return text + .split('\n') + .filter((line) => line.trim() !== '') + .join('\n'); +} + +async function extractFromVision( + fileData: string, + fileType: string, + logger: ContentExtractionLogger, +): Promise { + const fileSizeMB = ( + Buffer.from(fileData, 'base64').length / + (1024 * 1024) + ).toFixed(2); + + logger.info('Extracting content from PDF/image using vision API', { + fileType, + fileSizeMB, + }); + + const startTime = Date.now(); + + try { + const { text } = await generateText({ + model: openai(PARSING_MODEL), + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: VISION_EXTRACTION_PROMPT }, + { type: 'image', image: `data:${fileType};base64,${fileData}` }, + ], + }, + ], + }); + + const extractionTime = ((Date.now() - startTime) / 1000).toFixed(2); + logger.info('Content extracted from PDF/image', { + fileType, + extractedLength: text.length, + extractionTimeSeconds: extractionTime, + }); + + return text; + } catch (error) { + const extractionTime = ((Date.now() - startTime) / 1000).toFixed(2); + logger.error('Failed to extract content from PDF/image', { + fileType, + fileSizeMB, + extractionTimeSeconds: extractionTime, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw new Error( + `Failed to extract content: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } +} + diff --git a/apps/api/src/questionnaire/utils/deduplicate-sources.ts b/apps/api/src/questionnaire/utils/deduplicate-sources.ts new file mode 100644 index 000000000..5b9f348c6 --- /dev/null +++ b/apps/api/src/questionnaire/utils/deduplicate-sources.ts @@ -0,0 +1,122 @@ +export interface Source { + sourceType: string; + sourceName?: string; + sourceId?: string; + policyName?: string; + documentName?: string; + manualAnswerQuestion?: string; + score: number; +} + +/** + * Deduplicates an array of sources based on source type and content. + * Mirrored from the Next.js implementation so Trigger/Nest tasks can reuse it. + * + * - Policies: policyName + * - Context: grouped into single "Context Q&A" + * - Manual Answers: sourceId (each answer is separate) + * - Knowledge Base Documents: sourceId + * - Others: sourceId + * + * Keeps the highest score when duplicates appear. + */ +export function deduplicateSources(sources: Source[]): Source[] { + if (!sources || sources.length === 0) { + return []; + } + + const sourceMap = new Map(); + + for (const source of sources) { + if (!source.sourceType) { + continue; + } + + let deduplicationKey: string; + + if (source.sourceType === 'policy' && source.policyName) { + deduplicationKey = `policy:${source.policyName}`; + } else if (source.sourceType === 'context') { + deduplicationKey = 'context:all'; + } else if (source.sourceType === 'manual_answer') { + deduplicationKey = `manual_answer:${source.sourceId || 'unknown'}`; + } else if (source.sourceType === 'knowledge_base_document') { + deduplicationKey = `knowledge_base_document:${source.sourceId || 'unknown'}`; + } else { + deduplicationKey = source.sourceId || `unknown:${source.sourceType}`; + } + + const existing = sourceMap.get(deduplicationKey); + if (!existing || source.score > existing.score) { + const normalizedSource: Source = { + ...source, + documentName: source.documentName || existing?.documentName, + manualAnswerQuestion: + source.manualAnswerQuestion || existing?.manualAnswerQuestion, + sourceName: undefined, + }; + normalizedSource.sourceName = getSourceDisplayName(normalizedSource); + sourceMap.set(deduplicationKey, normalizedSource); + } else if (existing) { + let needsUpdate = false; + if (source.documentName && !existing.documentName) { + existing.documentName = source.documentName; + needsUpdate = true; + } + if (source.manualAnswerQuestion && !existing.manualAnswerQuestion) { + existing.manualAnswerQuestion = source.manualAnswerQuestion; + needsUpdate = true; + } + if ( + needsUpdate || + !existing.sourceName || + existing.sourceType === 'manual_answer' + ) { + existing.sourceName = getSourceDisplayName(existing); + } + } + } + + return Array.from(sourceMap.values()).sort((a, b) => b.score - a.score); +} + +function getSourceDisplayName(source: Source): string { + if (source.sourceType === 'policy' && source.policyName) { + return `Policy: ${source.policyName}`; + } + + if (source.sourceType === 'context') { + return 'Context Q&A'; + } + + if (source.sourceType === 'manual_answer') { + if (source.manualAnswerQuestion) { + const preview = + source.manualAnswerQuestion.length > 50 + ? `${source.manualAnswerQuestion.substring(0, 50)}...` + : source.manualAnswerQuestion; + return `Manual Answer (${preview})`; + } + if (source.sourceId) { + const shortId = + source.sourceId.length > 8 + ? source.sourceId.substring(source.sourceId.length - 8) + : source.sourceId; + return `Manual Answer (${shortId})`; + } + return 'Manual Answer'; + } + + if (source.sourceType === 'knowledge_base_document') { + if (source.documentName) { + return `Knowledge Base Document (${source.documentName})`; + } + return 'Knowledge Base Document'; + } + + if (source.sourceName) { + return source.sourceName; + } + + return source.sourceType || 'Unknown Source'; +} diff --git a/apps/api/src/questionnaire/utils/export-generator.ts b/apps/api/src/questionnaire/utils/export-generator.ts new file mode 100644 index 000000000..f5d27b7fc --- /dev/null +++ b/apps/api/src/questionnaire/utils/export-generator.ts @@ -0,0 +1,142 @@ +import * as XLSX from 'xlsx'; +import { jsPDF } from 'jspdf'; +import type { QuestionAnswer } from './question-parser'; + +export type ExportFormat = 'pdf' | 'csv' | 'xlsx'; + +export interface ExportResult { + fileBuffer: Buffer; + mimeType: string; + filename: string; +} + +/** + * Generates an export file in the specified format + */ +export function generateExportFile( + questionsAndAnswers: QuestionAnswer[], + format: ExportFormat, + vendorName: string, +): ExportResult { + // Remove original extension if present and get base name + const baseName = vendorName.replace(/\.[^/.]+$/, ''); + // Keep the original name but sanitize only dangerous characters for filenames + const sanitizedBaseName = baseName.replace(/[<>:"/\\|?*]/g, '_'); + + switch (format) { + case 'xlsx': + return { + fileBuffer: generateXLSX(questionsAndAnswers), + mimeType: + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + filename: `${sanitizedBaseName}.xlsx`, + }; + + case 'csv': + return { + fileBuffer: Buffer.from(generateCSV(questionsAndAnswers), 'utf-8'), + mimeType: 'text/csv', + filename: `${sanitizedBaseName}.csv`, + }; + + case 'pdf': + default: + return { + fileBuffer: generatePDF(questionsAndAnswers, baseName), + mimeType: 'application/pdf', + filename: `${sanitizedBaseName}.pdf`, + }; + } +} + +/** + * Generates an XLSX file buffer from questions and answers + */ +export function generateXLSX(questionsAndAnswers: QuestionAnswer[]): Buffer { + const workbook = XLSX.utils.book_new(); + const worksheetData = [ + ['#', 'Question', 'Answer'], + ...questionsAndAnswers.map((qa, index) => [ + index + 1, + qa.question, + qa.answer || '', + ]), + ]; + const worksheet = XLSX.utils.aoa_to_sheet(worksheetData); + worksheet['!cols'] = [{ wch: 5 }, { wch: 60 }, { wch: 60 }]; + XLSX.utils.book_append_sheet(workbook, worksheet, 'Questionnaire'); + return XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }); +} + +/** + * Generates a CSV string from questions and answers + */ +export function generateCSV(questionsAndAnswers: QuestionAnswer[]): string { + const rows = [ + ['#', 'Question', 'Answer'], + ...questionsAndAnswers.map((qa, index) => [ + String(index + 1), + qa.question.replace(/"/g, '""'), + (qa.answer || '').replace(/"/g, '""'), + ]), + ]; + return rows.map((row) => row.map((cell) => `"${cell}"`).join(',')).join('\n'); +} + +/** + * Generates a PDF buffer from questions and answers + */ +export function generatePDF( + questionsAndAnswers: QuestionAnswer[], + vendorName?: string, +): Buffer { + const doc = new jsPDF(); + const pageWidth = doc.internal.pageSize.getWidth(); + const pageHeight = doc.internal.pageSize.getHeight(); + const margin = 20; + const contentWidth = pageWidth - 2 * margin; + let yPosition = margin; + const lineHeight = 7; + + // Title + doc.setFontSize(16); + doc.setFont('helvetica', 'bold'); + const title = vendorName ? `Questionnaire: ${vendorName}` : 'Questionnaire'; + doc.text(title, margin, yPosition); + yPosition += lineHeight * 2; + + // Generated date + doc.setFontSize(10); + doc.setFont('helvetica', 'normal'); + doc.text(`Generated: ${new Date().toLocaleDateString()}`, margin, yPosition); + yPosition += lineHeight * 2; + + // Questions and answers + doc.setFontSize(11); + questionsAndAnswers.forEach((qa, index) => { + if (yPosition > pageHeight - 40) { + doc.addPage(); + yPosition = margin; + } + + // Question + doc.setFont('helvetica', 'bold'); + const questionText = `Q${index + 1}: ${qa.question}`; + const questionLines = doc.splitTextToSize(questionText, contentWidth); + doc.text(questionLines, margin, yPosition); + yPosition += questionLines.length * lineHeight + 2; + + // Answer + doc.setFont('helvetica', 'normal'); + const answerText = qa.answer || 'No answer provided'; + const answerLines = doc.splitTextToSize( + `A${index + 1}: ${answerText}`, + contentWidth, + ); + doc.text(answerLines, margin, yPosition); + yPosition += answerLines.length * lineHeight + 4; + }); + + return Buffer.from(doc.output('arraybuffer')); +} + diff --git a/apps/api/src/questionnaire/utils/question-parser.ts b/apps/api/src/questionnaire/utils/question-parser.ts new file mode 100644 index 000000000..04cf16912 --- /dev/null +++ b/apps/api/src/questionnaire/utils/question-parser.ts @@ -0,0 +1,285 @@ +import { openai } from '@ai-sdk/openai'; +import { generateObject, jsonSchema } from 'ai'; +import { + MAX_CHUNK_SIZE_CHARS, + MIN_CHUNK_SIZE_CHARS, + MAX_QUESTIONS_PER_CHUNK, + PARSING_MODEL, + QUESTION_PARSING_SYSTEM_PROMPT, +} from './constants'; + +export interface QuestionAnswer { + question: string; + answer: string | null; +} + +export interface ChunkInfo { + content: string; + questionCount: number; +} + +export interface QuestionParserLogger { + info: (message: string, meta?: Record) => void; + warn: (message: string, meta?: Record) => void; + error: (message: string, meta?: Record) => void; +} + +// Default no-op logger +const defaultLogger: QuestionParserLogger = { + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +/** + * Parses questions and answers from extracted content using LLM + * Handles large content by chunking and processing in parallel + */ +export async function parseQuestionsAndAnswers( + content: string, + logger: QuestionParserLogger = defaultLogger, +): Promise { + const chunkInfos = buildQuestionAwareChunks(content, { + maxChunkChars: MAX_CHUNK_SIZE_CHARS, + minChunkChars: MIN_CHUNK_SIZE_CHARS, + maxQuestionsPerChunk: MAX_QUESTIONS_PER_CHUNK, + }); + + if (chunkInfos.length === 0) { + logger.warn('No content found after preprocessing, returning empty result'); + return []; + } + + if (chunkInfos.length === 1) { + logger.info('Processing content as a single chunk', { + contentLength: chunkInfos[0].content.length, + estimatedQuestions: chunkInfos[0].questionCount, + }); + return parseChunkQuestionsAndAnswers(chunkInfos[0].content, 0, 1); + } + + logger.info('Chunking content by individual questions for parallel processing', { + contentLength: content.length, + totalChunks: chunkInfos.length, + questionsPerChunk: 1, + }); + + // Process all chunks in parallel for maximum speed + const parseStartTime = Date.now(); + const allPromises = chunkInfos.map((chunk, index) => + parseChunkQuestionsAndAnswers(chunk.content, index, chunkInfos.length), + ); + + const allResults = await Promise.all(allPromises); + const parseTime = ((Date.now() - parseStartTime) / 1000).toFixed(2); + + const totalRawQuestions = allResults.reduce( + (sum, chunk) => sum + chunk.length, + 0, + ); + + logger.info('All chunks processed in parallel', { + totalChunks: chunkInfos.length, + parseTimeSeconds: parseTime, + totalQuestions: totalRawQuestions, + }); + + // Deduplicate questions (same question might appear in multiple chunks) + const seenQuestions = new Map(); + + for (const qaArray of allResults) { + for (const qa of qaArray) { + const normalizedQuestion = qa.question.toLowerCase().trim(); + if (!seenQuestions.has(normalizedQuestion)) { + seenQuestions.set(normalizedQuestion, qa); + } + } + } + + const uniqueResults = Array.from(seenQuestions.values()); + + logger.info('Parsing complete', { + totalQuestions: uniqueResults.length, + duplicatesRemoved: totalRawQuestions - uniqueResults.length, + }); + + return uniqueResults; +} + +/** + * Parses questions and answers from a single chunk of content + */ +export async function parseChunkQuestionsAndAnswers( + chunk: string, + chunkIndex: number, + totalChunks: number, +): Promise { + const { object } = await generateObject({ + model: openai(PARSING_MODEL), + mode: 'json', + schema: jsonSchema({ + type: 'object', + properties: { + questionsAndAnswers: { + type: 'array', + items: { + type: 'object', + properties: { + question: { type: 'string', description: 'The question text' }, + answer: { + anyOf: [{ type: 'string' }, { type: 'null' }], + description: 'The answer to the question. Use null if no answer is provided.', + }, + }, + required: ['question'], + }, + }, + }, + required: ['questionsAndAnswers'], + }), + system: QUESTION_PARSING_SYSTEM_PROMPT, + prompt: buildParsingPrompt(chunk, chunkIndex, totalChunks), + }); + + const parsed = (object as { questionsAndAnswers: QuestionAnswer[] }) + .questionsAndAnswers; + + // Post-process to ensure empty strings are converted to null + return parsed.map((qa) => ({ + question: qa.question, + answer: qa.answer && qa.answer.trim() !== '' ? qa.answer : null, + })); +} + +function buildParsingPrompt( + chunk: string, + chunkIndex: number, + totalChunks: number, +): string { + if (totalChunks > 1) { + return `Chunk ${chunkIndex + 1} of ${totalChunks}. +Instructions: +- Extract only question → answer pairs that represent real questions. +- Ignore rows or cells that contain only headers/labels (e.g. "Company Name", "Department", "Assessment Date", "Question", "Answer") or other metadata. +- If an answer is blank, set it to null. + +Chunk content: +${chunk}`; + } + + return `Instructions: +- Extract all meaningful question → answer pairs from the following content. +- Ignore rows or cells that contain only headers/labels (e.g. "Company Name", "Department", "Assessment Date", "Question", "Answer", "Name of Assessor"). +- Keep only entries that are actual questions (end with '?' or start with interrogative words). +- If an answer is blank, set it to null. + +Content: +${chunk}`; +} + +/** + * Builds question-aware chunks from content + * Each chunk contains exactly one question for parallel processing + */ +export function buildQuestionAwareChunks( + content: string, + options: { + maxChunkChars: number; + minChunkChars: number; + maxQuestionsPerChunk: number; + }, +): ChunkInfo[] { + const trimmedContent = content.trim(); + if (!trimmedContent) { + return []; + } + + const chunks: ChunkInfo[] = []; + const lines = trimmedContent.split(/\r?\n/); + let currentChunk: string[] = []; + let currentQuestionFound = false; + + const pushChunk = () => { + const chunkText = currentChunk.join('\n').trim(); + if (!chunkText) { + return; + } + chunks.push({ + content: chunkText, + questionCount: 1, + }); + currentChunk = []; + currentQuestionFound = false; + }; + + for (const line of lines) { + const trimmedLine = line.trim(); + const isEmpty = trimmedLine.length === 0; + const looksLikeQuestion = !isEmpty && looksLikeQuestionLine(trimmedLine); + + // If we find a new question and we already have a question in the current chunk, start a new chunk + if (looksLikeQuestion && currentQuestionFound && currentChunk.length > 0) { + pushChunk(); + } + + // Add line to current chunk (including empty lines for context) + if (!isEmpty || currentChunk.length > 0) { + currentChunk.push(line); + } + + // Mark that we've found a question in this chunk + if (looksLikeQuestion) { + currentQuestionFound = true; + } + } + + // Push the last chunk if it has content + if (currentChunk.length > 0) { + pushChunk(); + } + + // If no questions were detected, return the entire content as a single chunk + return chunks.length > 0 + ? chunks + : [ + { + content: trimmedContent, + questionCount: estimateQuestionCount(trimmedContent), + }, + ]; +} + +/** + * Checks if a line looks like a question + */ +export function looksLikeQuestionLine(line: string): boolean { + const questionSuffix = /[?？]\s*$/; + const explicitQuestionPrefix = /^(?:\d+\s*[\).\]]\s*)?(?:question|q)\b/i; + const interrogativePrefix = + /^(?:what|why|how|when|where|is|are|does|do|can|will|should|list|describe|explain)\b/i; + + return ( + questionSuffix.test(line) || + explicitQuestionPrefix.test(line) || + interrogativePrefix.test(line) + ); +} + +/** + * Estimates the number of questions in a text + */ +export function estimateQuestionCount(text: string): number { + const questionMarks = text.match(/[?？]/g)?.length ?? 0; + if (questionMarks > 0) { + return questionMarks; + } + const lines = text + .split(/\r?\n/) + .filter((line) => looksLikeQuestionLine(line.trim())); + if (lines.length > 0) { + return lines.length; + } + // Fallback heuristic: assume roughly one question per 1200 chars + return Math.max(1, Math.floor(text.length / 1200)); +} + diff --git a/apps/api/src/questionnaire/utils/questionnaire-storage.ts b/apps/api/src/questionnaire/utils/questionnaire-storage.ts new file mode 100644 index 000000000..846595011 --- /dev/null +++ b/apps/api/src/questionnaire/utils/questionnaire-storage.ts @@ -0,0 +1,209 @@ +import { db, Prisma } from '@db'; +import { + s3Client, + APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET, + BUCKET_NAME, +} from '../../app/s3'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { randomBytes } from 'crypto'; +import { MAX_FILE_SIZE_BYTES } from './constants'; + +export interface QuestionnaireAnswerData { + question: string; + answer: string | null; + sources?: unknown; +} + +export interface StorageLogger { + log: (message: string, meta?: Record) => void; + error: (message: string, meta?: Record) => void; +} + +const defaultLogger: StorageLogger = { + log: () => {}, + error: () => {}, +}; + +/** + * Updates the answered questions count for a questionnaire + */ +export async function updateAnsweredCount(questionnaireId: string): Promise { + const answeredCount = await db.questionnaireQuestionAnswer.count({ + where: { + questionnaireId, + answer: { not: null }, + }, + }); + + await db.questionnaire.update({ + where: { id: questionnaireId }, + data: { + answeredQuestions: answeredCount, + updatedAt: new Date(), + }, + }); +} + +/** + * Persists a questionnaire result to the database + */ +export async function persistQuestionnaireResult( + params: { + organizationId: string; + fileName: string; + fileType: string; + fileSize: number; + questionsAndAnswers: QuestionnaireAnswerData[]; + source: 'internal' | 'external'; + s3Key: string | null; + }, + logger: StorageLogger = defaultLogger, +): Promise { + try { + const answeredCount = params.questionsAndAnswers.filter( + (qa) => qa.answer && qa.answer.trim().length > 0, + ).length; + + const questionnaire = await db.questionnaire.create({ + data: { + filename: params.fileName, + s3Key: params.s3Key ?? `api-upload-${params.source}`, + fileType: params.fileType, + fileSize: params.fileSize, + organizationId: params.organizationId, + status: 'completed', + parsedAt: new Date(), + totalQuestions: params.questionsAndAnswers.length, + answeredQuestions: answeredCount, + questions: { + create: params.questionsAndAnswers.map((qa, index) => ({ + question: qa.question, + answer: qa.answer, + questionIndex: index, + status: qa.answer ? 'generated' : 'untouched', + generatedAt: qa.answer ? new Date() : undefined, + sources: qa.sources + ? (qa.sources as Prisma.InputJsonValue) + : Prisma.JsonNull, + })), + }, + }, + }); + + logger.log('Saved questionnaire result', { + questionnaireId: questionnaire.id, + organizationId: params.organizationId, + source: params.source, + }); + + return questionnaire.id; + } catch (error) { + logger.error('Failed to save questionnaire result', { + error: error instanceof Error ? error.message : 'Unknown error', + organizationId: params.organizationId, + }); + return null; + } +} + +/** + * Uploads a questionnaire file to S3 + */ +export async function uploadQuestionnaireFile(params: { + organizationId: string; + fileName: string; + fileType: string; + fileData: string; + source: 'internal' | 'external'; +}): Promise<{ s3Key: string; fileSize: number } | null> { + if (!s3Client) { + throw new Error('S3 client not configured for questionnaire uploads'); + } + + const bucket = APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET || BUCKET_NAME; + if (!bucket) { + throw new Error( + 'APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET or APP_AWS_BUCKET_NAME must be configured for questionnaire uploads', + ); + } + + const fileBuffer = Buffer.from(params.fileData, 'base64'); + + if (fileBuffer.length > MAX_FILE_SIZE_BYTES) { + throw new Error( + `File exceeds the ${MAX_FILE_SIZE_BYTES / (1024 * 1024)}MB limit`, + ); + } + + const fileId = randomBytes(16).toString('hex'); + const sanitizedFileName = params.fileName.replace(/[^a-zA-Z0-9.-]/g, '_'); + const timestamp = Date.now(); + const s3Key = `${params.organizationId}/questionnaire-uploads/${timestamp}-${fileId}-${sanitizedFileName}`; + + const putCommand = new PutObjectCommand({ + Bucket: bucket, + Key: s3Key, + Body: fileBuffer, + ContentType: params.fileType, + Metadata: { + originalFileName: params.fileName, + organizationId: params.organizationId, + source: params.source, + }, + }); + + await s3Client.send(putCommand); + + return { + s3Key, + fileSize: fileBuffer.length, + }; +} + +/** + * Saves a generated answer to the database + */ +export async function saveGeneratedAnswer(params: { + questionnaireId: string; + questionIndex: number; + answer: string; + sources?: unknown; +}): Promise { + const question = await db.questionnaireQuestionAnswer.findFirst({ + where: { + questionnaireId: params.questionnaireId, + questionIndex: params.questionIndex, + }, + }); + + if (question) { + await db.questionnaireQuestionAnswer.update({ + where: { id: question.id }, + data: { + answer: params.answer, + status: 'generated', + sources: params.sources + ? (params.sources as Prisma.InputJsonValue) + : Prisma.JsonNull, + generatedAt: new Date(), + }, + }); + } else { + await db.questionnaireQuestionAnswer.create({ + data: { + questionnaireId: params.questionnaireId, + questionIndex: params.questionIndex, + question: '', + answer: params.answer, + status: 'generated', + sources: params.sources + ? (params.sources as Prisma.InputJsonValue) + : Prisma.JsonNull, + generatedAt: new Date(), + }, + }); + } + + await updateAnsweredCount(params.questionnaireId); +} + diff --git a/apps/api/src/questionnaire/vendors/answer-question-helpers.ts b/apps/api/src/questionnaire/vendors/answer-question-helpers.ts new file mode 100644 index 000000000..05f437e1c --- /dev/null +++ b/apps/api/src/questionnaire/vendors/answer-question-helpers.ts @@ -0,0 +1,293 @@ +import { findSimilarContent, findSimilarContentBatch } from '@/vector-store/lib'; +import type { SimilarContentResult } from '@/vector-store/lib'; +import { openai } from '@ai-sdk/openai'; +import { logger } from '@trigger.dev/sdk'; +import { generateText } from 'ai'; +import { deduplicateSources, type Source } from '@/questionnaire/utils/deduplicate-sources'; +import { ANSWER_MODEL, ANSWER_SYSTEM_PROMPT } from '@/questionnaire/utils/constants'; + +export interface AnswerWithSources { + answer: string | null; + sources: Array<{ + sourceType: string; + sourceName?: string; + score: number; + }>; +} + +/** + * Extracts source information from similar content results and deduplicates them + */ +function extractAndDeduplicateSources(similarContent: SimilarContentResult[]): Source[] { + const sourcesBeforeDedup = similarContent.map((result) => { + const r = result as SimilarContentResult; + let sourceName: string | undefined; + + if (r.policyName) { + sourceName = `Policy: ${r.policyName}`; + } else if (r.contextQuestion) { + sourceName = 'Context Q&A'; + } else if (r.sourceType === 'manual_answer') { + // Don't set sourceName here - let deduplicateSources handle it with manualAnswerQuestion + sourceName = undefined; + } + // Don't set sourceName for knowledge_base_document - let deduplication function handle it with filename + + return { + sourceType: r.sourceType, + sourceName, + sourceId: r.sourceId, + policyName: r.policyName, + documentName: r.documentName, + manualAnswerQuestion: r.manualAnswerQuestion, + score: r.score, + }; + }); + + return deduplicateSources(sourcesBeforeDedup); +} + +/** + * Builds context string from similar content for LLM prompt + */ +function buildContextFromContent(similarContent: SimilarContentResult[]): string { + const contextParts = similarContent.map((result, index) => { + const r = result as SimilarContentResult; + let sourceInfo = ''; + + if (r.policyName) { + sourceInfo = `Source: Policy "${r.policyName}"`; + } else if (r.contextQuestion) { + sourceInfo = `Source: Context Q&A`; + } else if (r.sourceType === 'knowledge_base_document') { + sourceInfo = r.documentName + ? `Source: Knowledge Base Document "${r.documentName}"` + : `Source: Knowledge Base Document`; + } else if (r.sourceType === 'manual_answer') { + sourceInfo = `Source: Manual Answer`; + } else { + sourceInfo = `Source: ${r.sourceType}`; + } + + return `[${index + 1}] ${sourceInfo}\n${r.content}`; + }); + + return contextParts.join('\n\n'); +} + +/** + * Generates answer using LLM with the provided context + */ +async function generateAnswerWithLLM( + question: string, + context: string, +): Promise { + const { text } = await generateText({ + model: openai(ANSWER_MODEL), + system: ANSWER_SYSTEM_PROMPT, + prompt: `Based on the following context from our organization's policies and documentation, answer this question: + +Question: ${question} + +Context: +${context} + +Answer the question based ONLY on the provided context, using first person plural (we, our, us). If the context doesn't contain enough information, respond with exactly "N/A - no evidence found".`, + }); + + return text.trim(); +} + +/** + * Checks if an answer indicates no evidence was found + */ +function isNoEvidenceAnswer(answer: string): boolean { + const lowerAnswer = answer.toLowerCase(); + return ( + lowerAnswer.includes('n/a') || + lowerAnswer.includes('no evidence') || + lowerAnswer.includes('not found in the context') + ); +} + +/** + * Generates an answer for a question using RAG (Retrieval-Augmented Generation) + */ +export async function generateAnswerWithRAG( + question: string, + organizationId: string, +): Promise { + try { + // Find similar content from vector database + const similarContent = await findSimilarContent(question, organizationId); + + logger.info('Vector search results', { + question: question.substring(0, 100), + organizationId, + resultCount: similarContent.length, + results: similarContent.map((r) => ({ + sourceType: r.sourceType, + score: r.score, + sourceId: r.sourceId.substring(0, 50), + })), + }); + + // If no relevant content found, return null + if (similarContent.length === 0) { + logger.warn('No similar content found in vector database', { + question: question.substring(0, 100), + organizationId, + }); + return { answer: null, sources: [] }; + } + + // Extract and deduplicate sources + const sources = extractAndDeduplicateSources(similarContent); + + logger.info('Sources extracted and deduplicated', { + question: question.substring(0, 100), + organizationId, + similarContentCount: similarContent.length, + sourcesAfterDedupCount: sources.length, + sources: sources.map((s) => ({ + type: s.sourceType, + name: s.sourceName, + score: s.score, + sourceId: s.sourceId?.substring(0, 30), + })), + }); + + // Build context and generate answer + const context = buildContextFromContent(similarContent); + const answer = await generateAnswerWithLLM(question, context); + + // Check if the answer indicates no evidence + if (isNoEvidenceAnswer(answer)) { + logger.warn('Answer indicates no evidence found', { + question: question.substring(0, 100), + answer: answer.substring(0, 100), + sourcesCount: sources.length, + }); + return { answer: null, sources: [] }; + } + + // Safety check: if we have an answer but no sources, log a warning + if (sources.length === 0 && answer) { + logger.warn( + 'Answer generated but no sources found - this may indicate LLM used general knowledge', + { + question: question.substring(0, 100), + answer: answer.substring(0, 100), + similarContentCount: similarContent.length, + }, + ); + } + + return { answer, sources }; + } catch (error) { + logger.error('Failed to generate answer with RAG', { + question: question.substring(0, 100), + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { answer: null, sources: [] }; + } +} + +/** + * Batch version of generateAnswerWithRAG - processes multiple questions efficiently + * Uses batch embedding generation for significant speedup + */ +export async function generateAnswerWithRAGBatch( + questions: string[], + organizationId: string, +): Promise { + if (questions.length === 0) { + return []; + } + + const startTime = Date.now(); + + try { + logger.info('Starting batch RAG generation', { + questionCount: questions.length, + organizationId, + }); + + // Step 1: Find similar content for ALL questions at once (batch embeddings) + const searchStartTime = Date.now(); + const allSimilarContent = await findSimilarContentBatch(questions, organizationId); + const searchTime = Date.now() - searchStartTime; + + logger.info('Batch search completed', { + questionCount: questions.length, + searchTimeMs: searchTime, + }); + + // Step 2: Generate answers in parallel using pre-fetched content + const llmStartTime = Date.now(); + const answers = await Promise.all( + questions.map((question, index) => + generateAnswerFromContent(question, allSimilarContent[index] || []), + ), + ); + const llmTime = Date.now() - llmStartTime; + + const totalTime = Date.now() - startTime; + + logger.info('Batch RAG generation completed', { + questionCount: questions.length, + searchTimeMs: searchTime, + llmTimeMs: llmTime, + totalTimeMs: totalTime, + answeredCount: answers.filter((a) => a.answer !== null).length, + }); + + return answers; + } catch (error) { + logger.error('Failed batch RAG generation', { + questionCount: questions.length, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Return empty results for all questions on failure + return questions.map(() => ({ answer: null, sources: [] })); + } +} + +/** + * Helper function to generate an answer from pre-fetched similar content + * Used by both single and batch answer generation + * Exported for use in streaming endpoints that do batch search + parallel LLM + */ +export async function generateAnswerFromContent( + question: string, + similarContent: SimilarContentResult[], +): Promise { + try { + // If no relevant content found, return null + if (similarContent.length === 0) { + return { answer: null, sources: [] }; + } + + // Extract and deduplicate sources + const sources = extractAndDeduplicateSources(similarContent); + + // Build context and generate answer + const context = buildContextFromContent(similarContent); + const answer = await generateAnswerWithLLM(question, context); + + // Check if the answer indicates no evidence + if (isNoEvidenceAnswer(answer)) { + return { answer: null, sources: [] }; + } + + return { answer, sources }; + } catch (error) { + logger.error('Failed to generate answer from content', { + question: question.substring(0, 100), + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { answer: null, sources: [] }; + } +} diff --git a/apps/app/src/jobs/tasks/vendors/answer-question.ts b/apps/api/src/questionnaire/vendors/answer-question.ts similarity index 81% rename from apps/app/src/jobs/tasks/vendors/answer-question.ts rename to apps/api/src/questionnaire/vendors/answer-question.ts index c248ebf69..8c55926ca 100644 --- a/apps/app/src/jobs/tasks/vendors/answer-question.ts +++ b/apps/api/src/questionnaire/vendors/answer-question.ts @@ -1,4 +1,4 @@ -import { syncOrganizationEmbeddings } from '@/lib/vector'; +import { syncOrganizationEmbeddings } from '@/vector-store/lib'; import { logger, metadata, task } from '@trigger.dev/sdk'; import { generateAnswerWithRAG } from './answer-question-helpers'; @@ -28,6 +28,11 @@ export interface AnswerQuestionOptions { * Disable when running outside of a Trigger task (e.g. server actions). */ useMetadata?: boolean; + /** + * Whether to skip syncing organization embeddings. + * Set to true when sync has already been performed (e.g., in batch operations). + */ + skipSync?: boolean; } /** @@ -37,7 +42,7 @@ export async function answerQuestion( payload: AnswerQuestionPayload, options: AnswerQuestionOptions = {}, ): Promise { - const { useMetadata = true } = options; + const { useMetadata = true, skipSync = false } = options; const withMetadata = (fn: () => void) => { if (!useMetadata) { @@ -80,24 +85,34 @@ export async function answerQuestion( // Sync organization embeddings before generating answer // Uses incremental sync: only updates what changed (much faster than full sync) // Lock mechanism prevents concurrent syncs for the same organization - try { - await syncOrganizationEmbeddings(payload.organizationId); - logger.info('Organization embeddings synced successfully', { + // Skip sync if already performed (e.g., in batch operations) + if (!skipSync) { + try { + await syncOrganizationEmbeddings(payload.organizationId); + logger.info('Organization embeddings synced successfully', { + organizationId: payload.organizationId, + }); + } catch (error) { + logger.warn('Failed to sync organization embeddings', { + organizationId: payload.organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Continue with existing embeddings if sync fails + } + } else { + logger.info('Skipping sync (already performed)', { organizationId: payload.organizationId, }); - } catch (error) { - logger.warn('Failed to sync organization embeddings', { - organizationId: payload.organizationId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - // Continue with existing embeddings if sync fails } logger.info('🔍 Calling generateAnswerWithRAG', { questionIndex: payload.questionIndex, }); - const result = await generateAnswerWithRAG(payload.question, payload.organizationId); + const result = await generateAnswerWithRAG( + payload.question, + payload.organizationId, + ); // Update metadata with this answer immediately // This allows frontend to show answers as they complete individually diff --git a/apps/api/src/questionnaire/vendors/parse-questionnaire.ts b/apps/api/src/questionnaire/vendors/parse-questionnaire.ts new file mode 100644 index 000000000..82ab04808 --- /dev/null +++ b/apps/api/src/questionnaire/vendors/parse-questionnaire.ts @@ -0,0 +1,384 @@ +import { extractS3KeyFromUrl } from '@/app/s3'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { db } from '@db'; +import { logger, task } from '@trigger.dev/sdk'; + +// Import shared utilities +import { extractContentFromFile, type ContentExtractionLogger } from '@/questionnaire/utils/content-extractor'; +import { parseQuestionsAndAnswers, type QuestionAnswer } from '@/questionnaire/utils/question-parser'; + +// Adapter to convert Trigger.dev logger to ContentExtractionLogger interface +const triggerLogger: ContentExtractionLogger = { + info: (msg, meta) => logger.info(msg, meta), + warn: (msg, meta) => logger.warn(msg, meta), + error: (msg, meta) => logger.error(msg, meta), +}; + +/** + * Extracts content from a URL using Firecrawl + */ +async function extractContentFromUrl(url: string): Promise { + if (!process.env.FIRECRAWL_API_KEY) { + throw new Error('Firecrawl API key is not configured'); + } + + try { + const initialResponse = await fetch( + 'https://api.firecrawl.dev/v1/extract', + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.FIRECRAWL_API_KEY}`, + }, + body: JSON.stringify({ + urls: [url], + prompt: + 'Extract all text content from this page, including any questions and answers, forms, or questionnaire data.', + scrapeOptions: { + onlyMainContent: true, + removeBase64Images: true, + }, + }), + }, + ); + + const initialData = await initialResponse.json(); + + if (!initialData.success || !initialData.id) { + throw new Error('Failed to start Firecrawl extraction'); + } + + const jobId = initialData.id; + const maxWaitTime = 1000 * 60 * 5; // 5 minutes + const pollInterval = 5000; // 5 seconds + const startTime = Date.now(); + + while (Date.now() - startTime < maxWaitTime) { + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + + const statusResponse = await fetch( + `https://api.firecrawl.dev/v1/extract/${jobId}`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.FIRECRAWL_API_KEY}`, + }, + }, + ); + + const statusData = await statusResponse.json(); + + if (statusData.status === 'completed' && statusData.data) { + const extractedData = statusData.data; + if (typeof extractedData === 'string') { + return extractedData; + } + if (typeof extractedData === 'object' && extractedData.content) { + return typeof extractedData.content === 'string' + ? extractedData.content + : JSON.stringify(extractedData.content); + } + return JSON.stringify(extractedData); + } + + if (statusData.status === 'failed') { + throw new Error('Firecrawl extraction failed'); + } + + if (statusData.status === 'cancelled') { + throw new Error('Firecrawl extraction was cancelled'); + } + } + + throw new Error('Firecrawl extraction timed out'); + } catch (error) { + throw error instanceof Error + ? error + : new Error('Failed to extract content from URL'); + } +} + +/** + * Creates an S3 client instance for Trigger.dev tasks + */ +function createS3Client(): S3Client { + const region = process.env.APP_AWS_REGION || 'us-east-1'; + const accessKeyId = process.env.APP_AWS_ACCESS_KEY_ID; + const secretAccessKey = process.env.APP_AWS_SECRET_ACCESS_KEY; + + if (!accessKeyId || !secretAccessKey) { + throw new Error( + 'AWS S3 credentials are missing. Please set APP_AWS_ACCESS_KEY_ID and APP_AWS_SECRET_ACCESS_KEY environment variables in Trigger.dev.', + ); + } + + return new S3Client({ + region, + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +/** + * Extracts content from an attachment stored in S3 + */ +async function extractContentFromAttachment( + attachmentId: string, + organizationId: string, +): Promise<{ content: string; fileType: string }> { + const attachment = await db.attachment.findUnique({ + where: { + id: attachmentId, + organizationId, + }, + }); + + if (!attachment) { + throw new Error('Attachment not found'); + } + + const bucketName = process.env.APP_AWS_BUCKET_NAME; + if (!bucketName) { + throw new Error( + 'APP_AWS_BUCKET_NAME environment variable is not set in Trigger.dev.', + ); + } + + const key = extractS3KeyFromUrl(attachment.url); + const s3Client = createS3Client(); + const getCommand = new GetObjectCommand({ + Bucket: bucketName, + Key: key, + }); + + const response = await s3Client.send(getCommand); + + if (!response.Body) { + throw new Error('Failed to retrieve attachment from S3'); + } + + const chunks: Uint8Array[] = []; + for await (const chunk of response.Body as AsyncIterable) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + const base64Data = buffer.toString('base64'); + + const fileType = + response.ContentType || + (attachment.type === 'image' ? 'image/png' : 'application/pdf'); + + const content = await extractContentFromFile(base64Data, fileType, triggerLogger); + + return { content, fileType }; +} + +/** + * Extracts content from an S3 key (for temporary questionnaire files) + */ +async function extractContentFromS3Key( + s3Key: string, + fileType: string, +): Promise<{ content: string; fileType: string }> { + const questionnaireBucket = process.env.APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET; + + if (!questionnaireBucket) { + throw new Error( + 'Questionnaire upload bucket is not configured. Please set APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET environment variable in Trigger.dev.', + ); + } + + const s3Client = createS3Client(); + + const getCommand = new GetObjectCommand({ + Bucket: questionnaireBucket, + Key: s3Key, + }); + + const response = await s3Client.send(getCommand); + + if (!response.Body) { + throw new Error('Failed to retrieve file from S3'); + } + + const chunks: Uint8Array[] = []; + for await (const chunk of response.Body as AsyncIterable) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + const base64Data = buffer.toString('base64'); + + const detectedFileType = + response.ContentType || fileType || 'application/octet-stream'; + + const content = await extractContentFromFile(base64Data, detectedFileType, triggerLogger); + + return { content, fileType: detectedFileType }; +} + +export const parseQuestionnaireTask = task({ + id: 'parse-questionnaire', + retry: { + maxAttempts: 2, + }, + run: async (payload: { + inputType: 'file' | 'url' | 'attachment' | 's3'; + organizationId: string; + fileData?: string; + fileName?: string; + fileType?: string; + url?: string; + attachmentId?: string; + s3Key?: string; + }) => { + const taskStartTime = Date.now(); + + logger.info('Starting parse questionnaire task', { + inputType: payload.inputType, + organizationId: payload.organizationId, + }); + + try { + let extractedContent: string; + + // Extract content based on input type + switch (payload.inputType) { + case 'file': { + if (!payload.fileData || !payload.fileType) { + throw new Error('File data and file type are required for file input'); + } + extractedContent = await extractContentFromFile( + payload.fileData, + payload.fileType, + triggerLogger, + ); + break; + } + + case 'url': { + if (!payload.url) { + throw new Error('URL is required for URL input'); + } + extractedContent = await extractContentFromUrl(payload.url); + break; + } + + case 'attachment': { + if (!payload.attachmentId) { + throw new Error('Attachment ID is required for attachment input'); + } + const result = await extractContentFromAttachment( + payload.attachmentId, + payload.organizationId, + ); + extractedContent = result.content; + break; + } + + case 's3': { + if (!payload.s3Key || !payload.fileType) { + throw new Error('S3 key and file type are required for S3 input'); + } + const result = await extractContentFromS3Key( + payload.s3Key, + payload.fileType, + ); + extractedContent = result.content; + break; + } + + default: + throw new Error(`Unsupported input type: ${payload.inputType}`); + } + + logger.info('Content extracted successfully', { + inputType: payload.inputType, + contentLength: extractedContent.length, + }); + + // Parse questions and answers from extracted content + const parseStartTime = Date.now(); + const questionsAndAnswers = await parseQuestionsAndAnswers( + extractedContent, + triggerLogger, + ); + const parseTime = ((Date.now() - parseStartTime) / 1000).toFixed(2); + + const totalTime = ((Date.now() - taskStartTime) / 1000).toFixed(2); + + logger.info('Questions and answers parsed', { + questionCount: questionsAndAnswers.length, + parseTimeSeconds: parseTime, + totalTimeSeconds: totalTime, + }); + + // Create questionnaire record in database + let questionnaireId: string; + try { + const fileName = + payload.fileName || + payload.url || + payload.attachmentId || + 'questionnaire'; + const s3Key = payload.s3Key || ''; + const fileType = payload.fileType || 'application/octet-stream'; + const fileSize = payload.fileData + ? Buffer.from(payload.fileData, 'base64').length + : 0; + + const questionnaire = await db.questionnaire.create({ + data: { + filename: fileName, + s3Key: s3Key || '', + fileType, + fileSize, + organizationId: payload.organizationId, + status: 'completed', + parsedAt: new Date(), + totalQuestions: questionsAndAnswers.length, + answeredQuestions: 0, + questions: { + create: questionsAndAnswers.map((qa: QuestionAnswer, index: number) => ({ + question: qa.question, + answer: qa.answer || null, + questionIndex: index, + status: qa.answer ? 'generated' : 'untouched', + })), + }, + }, + }); + + questionnaireId = questionnaire.id; + + logger.info('Questionnaire record created', { + questionnaireId, + questionCount: questionsAndAnswers.length, + }); + } catch (error) { + logger.error('Failed to create questionnaire record', { + error: error instanceof Error ? error.message : 'Unknown error', + }); + questionnaireId = ''; + } + + return { + success: true, + questionnaireId, + questionsAndAnswers, + extractedContent: extractedContent.substring(0, 1000), + }; + } catch (error) { + logger.error('Failed to parse questionnaire', { + error: error instanceof Error ? error.message : 'Unknown error', + errorStack: error instanceof Error ? error.stack : undefined, + }); + throw error instanceof Error + ? error + : new Error('Failed to parse questionnaire'); + } + }, +}); diff --git a/apps/api/src/soa/dto/approve-soa-document.dto.ts b/apps/api/src/soa/dto/approve-soa-document.dto.ts new file mode 100644 index 000000000..073e9db43 --- /dev/null +++ b/apps/api/src/soa/dto/approve-soa-document.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class ApproveSOADocumentDto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; +} diff --git a/apps/api/src/soa/dto/auto-fill-soa.dto.ts b/apps/api/src/soa/dto/auto-fill-soa.dto.ts new file mode 100644 index 000000000..c52822d71 --- /dev/null +++ b/apps/api/src/soa/dto/auto-fill-soa.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class AutoFillSOADto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; +} diff --git a/apps/api/src/soa/dto/create-soa-document.dto.ts b/apps/api/src/soa/dto/create-soa-document.dto.ts new file mode 100644 index 000000000..29036b654 --- /dev/null +++ b/apps/api/src/soa/dto/create-soa-document.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class CreateSOADocumentDto { + @IsString() + organizationId!: string; + + @IsString() + frameworkId!: string; +} diff --git a/apps/api/src/soa/dto/decline-soa-document.dto.ts b/apps/api/src/soa/dto/decline-soa-document.dto.ts new file mode 100644 index 000000000..a754ad43c --- /dev/null +++ b/apps/api/src/soa/dto/decline-soa-document.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class DeclineSOADocumentDto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; +} diff --git a/apps/api/src/soa/dto/ensure-soa-setup.dto.ts b/apps/api/src/soa/dto/ensure-soa-setup.dto.ts new file mode 100644 index 000000000..8d36c4b9c --- /dev/null +++ b/apps/api/src/soa/dto/ensure-soa-setup.dto.ts @@ -0,0 +1,9 @@ +import { IsString } from 'class-validator'; + +export class EnsureSOASetupDto { + @IsString() + organizationId!: string; + + @IsString() + frameworkId!: string; +} diff --git a/apps/api/src/soa/dto/save-soa-answer.dto.ts b/apps/api/src/soa/dto/save-soa-answer.dto.ts new file mode 100644 index 000000000..32fc7f06e --- /dev/null +++ b/apps/api/src/soa/dto/save-soa-answer.dto.ts @@ -0,0 +1,24 @@ +import { IsBoolean, IsOptional, IsString } from 'class-validator'; + +export class SaveSOAAnswerDto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; + + @IsString() + questionId!: string; + + @IsOptional() + @IsString() + answer?: string | null; + + @IsOptional() + @IsBoolean() + isApplicable?: boolean | null; + + @IsOptional() + @IsString() + justification?: string | null; +} diff --git a/apps/api/src/soa/dto/submit-soa-for-approval.dto.ts b/apps/api/src/soa/dto/submit-soa-for-approval.dto.ts new file mode 100644 index 000000000..9b1acf4fd --- /dev/null +++ b/apps/api/src/soa/dto/submit-soa-for-approval.dto.ts @@ -0,0 +1,12 @@ +import { IsString } from 'class-validator'; + +export class SubmitSOAForApprovalDto { + @IsString() + organizationId!: string; + + @IsString() + documentId!: string; + + @IsString() + approverId!: string; +} diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/seedJson/ISO/config.json b/apps/api/src/soa/seedJson/ISO/config.json similarity index 100% rename from apps/app/src/app/(app)/[orgId]/questionnaire/soa/seedJson/ISO/config.json rename to apps/api/src/soa/seedJson/ISO/config.json diff --git a/apps/api/src/soa/soa.controller.ts b/apps/api/src/soa/soa.controller.ts new file mode 100644 index 000000000..e2089bc61 --- /dev/null +++ b/apps/api/src/soa/soa.controller.ts @@ -0,0 +1,395 @@ +import { + Controller, + Post, + Body, + Res, + HttpCode, + HttpStatus, + Logger, + BadRequestException, +} from '@nestjs/common'; +import type { Response } from 'express'; +import { + ApiTags, + ApiOperation, + ApiConsumes, + ApiProduces, + ApiOkResponse, + ApiBody, +} from '@nestjs/swagger'; +import { SOAService } from './soa.service'; +import { SaveSOAAnswerDto } from './dto/save-soa-answer.dto'; +import { AutoFillSOADto } from './dto/auto-fill-soa.dto'; +import { CreateSOADocumentDto } from './dto/create-soa-document.dto'; +import { EnsureSOASetupDto } from './dto/ensure-soa-setup.dto'; +import { ApproveSOADocumentDto } from './dto/approve-soa-document.dto'; +import { DeclineSOADocumentDto } from './dto/decline-soa-document.dto'; +import { SubmitSOAForApprovalDto } from './dto/submit-soa-for-approval.dto'; +import { syncOrganizationEmbeddings } from '@/vector-store/lib'; +import { OrganizationId } from '@/auth/auth-context.decorator'; +import { AuthContext } from '@/auth/auth-context.decorator'; +import type { AuthContext as AuthContextType } from '@/auth/types'; +import { UseGuards } from '@nestjs/common'; +import { HybridAuthGuard } from '@/auth/hybrid-auth.guard'; +import { ApiSecurity, ApiHeader } from '@nestjs/swagger'; +import { + createSafeSSESender, + setupSSEHeaders, + sanitizeErrorMessage, +} from '../utils/sse-utils'; + +@ApiTags('SOA') +@Controller({ + path: 'soa', + version: '1', +}) +@UseGuards(HybridAuthGuard) +@ApiSecurity('apikey') +@ApiHeader({ + name: 'X-Organization-Id', + description: + 'Organization ID (required for session auth, optional for API key auth)', + required: false, +}) +export class SOAController { + private readonly logger = new Logger(SOAController.name); + + constructor(private readonly soaService: SOAService) {} + + @Post('save-answer') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Save a SOA answer' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Answer saved successfully', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + }, + }, + }) + async saveAnswer( + @Body() dto: SaveSOAAnswerDto, + @OrganizationId() organizationId: string, + @AuthContext() authContext: AuthContextType, + ) { + if (!authContext.userId) { + throw new BadRequestException('User not authenticated'); + } + + return this.soaService.saveAnswer(dto, authContext.userId); + } + + @Post('auto-fill') + @ApiConsumes('application/json') + @ApiProduces('text/event-stream') + @ApiOperation({ + summary: 'Auto-fill SOA document', + description: 'Streams SOA answers via Server-Sent Events (SSE)', + }) + async autoFill( + @Body() dto: AutoFillSOADto, + @OrganizationId() organizationId: string, + @AuthContext() authContext: AuthContextType, + @Res() res: Response, + ): Promise { + if (!authContext.userId) { + throw new BadRequestException('User not authenticated'); + } + + const userId = authContext.userId; + setupSSEHeaders(res); + const send = createSafeSSESender(res); + + try { + this.logger.log('Starting auto-fill SOA via SSE', { + organizationId: dto.organizationId, + documentId: dto.documentId, + }); + + // Sync organization embeddings first to ensure vector database is up-to-date + // This ensures we have all policies, context, manual answers, and knowledge base documents + try { + await syncOrganizationEmbeddings(dto.organizationId); + } catch (error) { + this.logger.warn('Failed to sync organization embeddings', { + organizationId: dto.organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Continue anyway - might still have some data in vector DB + } + + // Get document with configuration first + const fullDocument = await this.soaService.getDocument( + dto.documentId, + dto.organizationId, + ); + if (!fullDocument) { + send({ + type: 'error', + error: 'SOA document not found', + }); + res.end(); + return; + } + + const configuration = fullDocument.configuration; + const questions = configuration.questions as Array<{ + id: string; + text: string; + columnMapping: { + closure: string; + title: string; + control_objective: string | null; + isApplicable: boolean | null; + justification: string | null; + }; + }>; + + // Send initial progress + send({ + type: 'progress', + total: questions.length, + completed: 0, + remaining: questions.length, + phase: 'searching', + }); + + // Check if organization is fully remote + const isFullyRemote = await this.soaService.checkIfFullyRemote( + dto.organizationId, + ); + + // Step 1: Batch search all questions (generates all embeddings in 1 API call) + const searchStartTime = Date.now(); + const similarContentMap = await this.soaService.batchSearchSOAQuestions( + questions, + dto.organizationId, + ); + const searchTime = Date.now() - searchStartTime; + + this.logger.log( + `Batch search completed in ${searchTime}ms for ${questions.length} SOA questions`, + ); + + send({ + type: 'progress', + total: questions.length, + completed: 0, + remaining: questions.length, + phase: 'generating', + searchTimeMs: searchTime, + }); + + // Send 'processing' status for all questions immediately for instant UI feedback + questions.forEach((question, index) => { + send({ + type: 'processing', + questionId: question.id, + questionIndex: index, + }); + }); + + // Process questions in parallel + const results: Array<{ + questionId: string; + isApplicable: boolean | null; + justification: string | null; + success: boolean; + error?: string; + insufficientData?: boolean; + }> = []; + + // Step 2: Process all questions in parallel using pre-fetched content + const promises = questions.map(async (question, index) => { + try { + const similarContent = similarContentMap.get(question.id) || []; + return await this.soaService.processSOAQuestionWithContent( + question, + index, + similarContent, + isFullyRemote, + send, + ); + } catch (error) { + this.logger.error('Failed to process SOA question', { + questionId: question.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + + send({ + type: 'answer', + questionId: question.id, + questionIndex: index, + isApplicable: null, + justification: null, + success: false, + error: 'Insufficient data', + insufficientData: true, + }); + + return { + questionId: question.id, + isApplicable: null, + justification: null, + success: false, + error: 'Insufficient data', + insufficientData: true, + }; + } + }); + + // Wait for all questions to complete + const settledResults = await Promise.allSettled(promises); + + // Collect all results + settledResults.forEach((result) => { + if (result.status === 'fulfilled') { + results.push(result.value); + } + }); + + // Save answers to database + const successfulResults = results.filter( + (r) => r.success && r.isApplicable !== null, + ); + + await this.soaService.saveAnswersToDatabase( + dto.documentId, + questions, + successfulResults, + userId, + ); + + // Update configuration with results + await this.soaService.updateConfigurationWithResults( + configuration.id, + questions, + successfulResults, + ); + + // Update document + const answeredCount = successfulResults.filter( + (r) => r.isApplicable !== null, + ).length; + await this.soaService.updateDocumentAfterAutoFill( + dto.documentId, + questions.length, + answeredCount, + ); + + // Send completion + send({ + type: 'complete', + total: questions.length, + answered: successfulResults.length, + results: successfulResults, + searchTimeMs: searchTime, + }); + + this.logger.log('Auto-fill SOA completed via SSE', { + organizationId: dto.organizationId, + documentId: dto.documentId, + totalQuestions: questions.length, + answered: successfulResults.length, + searchTimeMs: searchTime, + }); + + res.end(); + } catch (error) { + const safeErrorMessage = sanitizeErrorMessage(error); + this.logger.error('Error in auto-fill SOA SSE stream', { + organizationId: dto.organizationId, + error: safeErrorMessage, + }); + + send({ + type: 'error', + error: safeErrorMessage, + }); + + res.end(); + } + } + + @Post('create-document') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Create a new SOA document' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document created successfully', + }) + async createDocument( + @Body() dto: CreateSOADocumentDto, + @OrganizationId() organizationId: string, + ) { + return this.soaService.createDocument(dto); + } + + @Post('ensure-setup') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Ensure SOA configuration and document exist' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Setup ensured', + }) + async ensureSetup( + @Body() dto: EnsureSOASetupDto, + @OrganizationId() organizationId: string, + ) { + return this.soaService.ensureSetup(dto); + } + + @Post('approve') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Approve a SOA document' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document approved successfully', + }) + async approveDocument( + @Body() dto: ApproveSOADocumentDto, + @OrganizationId() organizationId: string, + @AuthContext() authContext: AuthContextType, + ) { + if (!authContext.userId) { + throw new BadRequestException('User not authenticated'); + } + + return this.soaService.approveDocument(dto, authContext.userId); + } + + @Post('decline') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Decline a SOA document' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document declined successfully', + }) + async declineDocument( + @Body() dto: DeclineSOADocumentDto, + @OrganizationId() organizationId: string, + @AuthContext() authContext: AuthContextType, + ) { + if (!authContext.userId) { + throw new BadRequestException('User not authenticated'); + } + + return this.soaService.declineDocument(dto, authContext.userId); + } + + @Post('submit-for-approval') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Submit SOA document for approval' }) + @ApiConsumes('application/json') + @ApiOkResponse({ + description: 'Document submitted for approval successfully', + }) + async submitForApproval( + @Body() dto: SubmitSOAForApprovalDto, + @OrganizationId() organizationId: string, + ) { + return this.soaService.submitForApproval(dto); + } +} diff --git a/apps/api/src/soa/soa.module.ts b/apps/api/src/soa/soa.module.ts new file mode 100644 index 000000000..c1dacfd3e --- /dev/null +++ b/apps/api/src/soa/soa.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { SOAController } from './soa.controller'; +import { SOAService } from './soa.service'; +import { AuthModule } from '../auth/auth.module'; + +@Module({ + imports: [AuthModule], + controllers: [SOAController], + providers: [SOAService], + exports: [SOAService], +}) +export class SOAModule {} diff --git a/apps/api/src/soa/soa.service.ts b/apps/api/src/soa/soa.service.ts new file mode 100644 index 000000000..c61f6f205 --- /dev/null +++ b/apps/api/src/soa/soa.service.ts @@ -0,0 +1,598 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { db } from '@db'; +import { SaveSOAAnswerDto } from './dto/save-soa-answer.dto'; +import { CreateSOADocumentDto } from './dto/create-soa-document.dto'; +import { EnsureSOASetupDto } from './dto/ensure-soa-setup.dto'; +import { ApproveSOADocumentDto } from './dto/approve-soa-document.dto'; +import { DeclineSOADocumentDto } from './dto/decline-soa-document.dto'; +import { SubmitSOAForApprovalDto } from './dto/submit-soa-for-approval.dto'; +import type { SimilarContentResult } from '@/vector-store/lib'; +import { loadISOConfig } from './utils/transform-iso-config'; +import { ISO27001_FRAMEWORK_NAMES } from './utils/constants'; +import { + batchSearchSOAQuestions, + generateSOAAnswerWithRAG, + generateSOAControlAnswer, +} from './utils/soa-answer-generator'; +import { + parseAndProcessSOAAnswer, + createDefaultYesResult, + createFullyRemoteResult, + isPhysicalSecurityControl, + type SOAQuestion, + type SOAQuestionResult, + type SOAStreamSender, +} from './utils/soa-answer-parser'; +import { + saveAnswersToDatabase, + updateConfigurationWithResults, + updateDocumentAfterAutoFill, + getAnsweredCountFromConfiguration, + updateDocumentAnsweredCount, + checkIfFullyRemote, + type SOAStorageLogger, +} from './utils/soa-storage'; + +@Injectable() +export class SOAService { + private readonly logger = new Logger(SOAService.name); + + private get storageLogger(): SOAStorageLogger { + return { + log: (msg, meta) => this.logger.log(msg, meta), + error: (msg, meta) => this.logger.error(msg, meta), + }; + } + + async saveAnswer(dto: SaveSOAAnswerDto, userId: string) { + // Verify document exists and belongs to organization + const document = await db.sOADocument.findFirst({ + where: { + id: dto.documentId, + organizationId: dto.organizationId, + }, + include: { + configuration: true, + }, + }); + + if (!document) { + throw new Error('SOA document not found'); + } + + // Get existing answer to determine version + const existingAnswer = await db.sOAAnswer.findFirst({ + where: { + documentId: dto.documentId, + questionId: dto.questionId, + isLatestAnswer: true, + }, + orderBy: { + answerVersion: 'desc', + }, + }); + + const nextVersion = existingAnswer ? existingAnswer.answerVersion + 1 : 1; + + // Mark existing answer as not latest if it exists + if (existingAnswer) { + await db.sOAAnswer.update({ + where: { id: existingAnswer.id }, + data: { isLatestAnswer: false }, + }); + } + + // Determine answer value + let finalAnswer: string | null = null; + if (dto.isApplicable !== undefined) { + finalAnswer = + dto.isApplicable === false + ? dto.justification || dto.answer || null + : null; + } else { + finalAnswer = dto.answer || null; + } + + // Create or update answer + await db.sOAAnswer.create({ + data: { + documentId: dto.documentId, + questionId: dto.questionId, + answer: finalAnswer, + status: + finalAnswer && finalAnswer.trim().length > 0 ? 'manual' : 'untouched', + answerVersion: nextVersion, + isLatestAnswer: true, + createdBy: existingAnswer ? undefined : userId, + updatedBy: userId, + }, + }); + + // Update configuration's question mapping if isApplicable or justification provided + if (dto.isApplicable !== undefined || dto.justification !== undefined) { + await this.updateQuestionMapping( + document.configuration.id, + dto.questionId, + dto.isApplicable ?? undefined, + dto.justification ?? undefined, + ); + } + + // Update document answered questions count + const answeredCount = await getAnsweredCountFromConfiguration( + document.configurationId, + ); + + await updateDocumentAnsweredCount( + dto.documentId, + document.totalQuestions, + answeredCount, + ); + + return { success: true }; + } + + private async updateQuestionMapping( + configurationId: string, + questionId: string, + isApplicable: boolean | undefined, + justification: string | undefined, + ) { + const configuration = await db.sOAFrameworkConfiguration.findUnique({ + where: { id: configurationId }, + }); + + if (!configuration) return; + + const questions = configuration.questions as unknown as SOAQuestion[]; + const updatedQuestions = questions.map((q) => { + if (q.id === questionId) { + return { + ...q, + columnMapping: { + ...q.columnMapping, + isApplicable: + isApplicable !== undefined + ? isApplicable + : q.columnMapping.isApplicable, + justification: + justification !== undefined + ? justification + : q.columnMapping.justification, + }, + }; + } + return q; + }); + + await db.sOAFrameworkConfiguration.update({ + where: { id: configurationId }, + data: { questions: JSON.parse(JSON.stringify(updatedQuestions)) }, + }); + } + + async createDocument(dto: CreateSOADocumentDto) { + const configuration = await db.sOAFrameworkConfiguration.findFirst({ + where: { + frameworkId: dto.frameworkId, + isLatest: true, + }, + }); + + if (!configuration) { + throw new Error('No SOA configuration found for this framework'); + } + + const existingLatestDocument = await db.sOADocument.findFirst({ + where: { + frameworkId: dto.frameworkId, + organizationId: dto.organizationId, + isLatest: true, + }, + }); + + let nextVersion = 1; + if (existingLatestDocument) { + await db.sOADocument.update({ + where: { id: existingLatestDocument.id }, + data: { isLatest: false }, + }); + nextVersion = existingLatestDocument.version + 1; + } + + const questions = configuration.questions as Array<{ id: string }>; + const totalQuestions = Array.isArray(questions) ? questions.length : 0; + + const document = await db.sOADocument.create({ + data: { + frameworkId: dto.frameworkId, + organizationId: dto.organizationId, + configurationId: configuration.id, + version: nextVersion, + isLatest: true, + status: 'draft', + totalQuestions, + answeredQuestions: 0, + }, + include: { + framework: true, + configuration: true, + }, + }); + + return { success: true, data: document }; + } + + async getDocument(documentId: string, organizationId: string) { + return db.sOADocument.findFirst({ + where: { + id: documentId, + organizationId, + }, + include: { + framework: true, + configuration: true, + answers: { + where: { isLatestAnswer: true }, + }, + }, + }); + } + + async ensureSetup(dto: EnsureSOASetupDto) { + const framework = await db.frameworkEditorFramework.findUnique({ + where: { id: dto.frameworkId }, + }); + + if (!framework) { + throw new Error('Framework not found'); + } + + const isISO27001 = ISO27001_FRAMEWORK_NAMES.includes(framework.name); + + if (!isISO27001) { + return { + success: false, + error: 'Only ISO 27001 framework is currently supported', + configuration: null, + document: null, + }; + } + + let configuration = await db.sOAFrameworkConfiguration.findFirst({ + where: { + frameworkId: dto.frameworkId, + isLatest: true, + }, + }); + + if (!configuration) { + try { + configuration = await this.seedISO27001SOAConfig(); + } catch (error) { + throw new Error( + `Failed to create SOA configuration: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + } + + let document = await db.sOADocument.findFirst({ + where: { + frameworkId: dto.frameworkId, + organizationId: dto.organizationId, + isLatest: true, + }, + include: { + answers: { where: { isLatestAnswer: true } }, + }, + }); + + if (!document && configuration) { + try { + document = await this.createSOADocumentDirect( + dto.frameworkId, + dto.organizationId, + configuration.id, + ); + } catch (error) { + throw new Error( + `Failed to create SOA document: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + } + + return { success: true, configuration, document }; + } + + async approveDocument(dto: ApproveSOADocumentDto, userId: string) { + const member = await this.validateOwnerOrAdmin(dto.organizationId, userId); + + const document = await db.sOADocument.findFirst({ + where: { + id: dto.documentId, + organizationId: dto.organizationId, + }, + }); + + if (!document) { + throw new Error('SOA document not found'); + } + + if (!document.approverId || document.approverId !== member.id) { + throw new Error('Document is not pending your approval'); + } + + if (document.status !== 'needs_review') { + throw new Error('Document is not in needs_review status'); + } + + const updatedDocument = await db.sOADocument.update({ + where: { id: dto.documentId }, + data: { + status: 'completed', + approvedAt: new Date(), + }, + }); + + return { success: true, data: updatedDocument }; + } + + async declineDocument(dto: DeclineSOADocumentDto, userId: string) { + const member = await this.validateOwnerOrAdmin(dto.organizationId, userId); + + const document = await db.sOADocument.findFirst({ + where: { + id: dto.documentId, + organizationId: dto.organizationId, + }, + }); + + if (!document) { + throw new Error('SOA document not found'); + } + + if (!document.approverId || document.approverId !== member.id) { + throw new Error('Document is not pending your approval'); + } + + if (document.status !== 'needs_review') { + throw new Error('Document is not in needs_review status'); + } + + const updatedDocument = await db.sOADocument.update({ + where: { id: dto.documentId }, + data: { + approverId: null, + approvedAt: null, + status: 'completed', + }, + }); + + return { success: true, data: updatedDocument }; + } + + async submitForApproval(dto: SubmitSOAForApprovalDto) { + const approverMember = await db.member.findFirst({ + where: { + id: dto.approverId, + organizationId: dto.organizationId, + deactivated: false, + }, + }); + + if (!approverMember) { + throw new Error('Approver not found in organization'); + } + + const isOwnerOrAdmin = + approverMember.role.includes('owner') || + approverMember.role.includes('admin'); + if (!isOwnerOrAdmin) { + throw new Error('Approver must be an owner or admin'); + } + + const document = await db.sOADocument.findFirst({ + where: { + id: dto.documentId, + organizationId: dto.organizationId, + }, + }); + + if (!document) { + throw new Error('SOA document not found'); + } + + if (document.status === 'needs_review') { + throw new Error('Document is already pending approval'); + } + + const updatedDocument = await db.sOADocument.update({ + where: { id: dto.documentId }, + data: { + approverId: dto.approverId, + status: 'needs_review', + }, + }); + + return { success: true, data: updatedDocument }; + } + + // Auto-fill related methods (delegating to utilities) + + async checkIfFullyRemote(organizationId: string): Promise { + return checkIfFullyRemote(organizationId, this.storageLogger); + } + + async batchSearchSOAQuestions( + questions: SOAQuestion[], + organizationId: string, + ): Promise> { + return batchSearchSOAQuestions(questions, organizationId); + } + + async processSOAQuestionWithContent( + question: SOAQuestion, + index: number, + similarContent: SimilarContentResult[], + isFullyRemote: boolean, + send: SOAStreamSender, + ): Promise { + const controlClosure = question.columnMapping.closure || ''; + + // If fully remote and control is physical security (section 7), return NO + if (isFullyRemote && isPhysicalSecurityControl(controlClosure)) { + return createFullyRemoteResult(question.id, index, send); + } + + // Generate answer from pre-fetched content + const soaResult = await generateSOAControlAnswer(question, similarContent); + + // If no answer, default to YES + if (!soaResult.answer) { + return createDefaultYesResult(question.id, index, send); + } + + return parseAndProcessSOAAnswer(question.id, index, soaResult.answer, send); + } + + async saveAnswersToDatabase( + documentId: string, + questions: SOAQuestion[], + results: SOAQuestionResult[], + userId: string, + ): Promise { + return saveAnswersToDatabase( + documentId, + questions, + results, + userId, + this.storageLogger, + ); + } + + async updateConfigurationWithResults( + configurationId: string, + questions: SOAQuestion[], + results: SOAQuestionResult[], + ): Promise { + return updateConfigurationWithResults(configurationId, questions, results); + } + + async updateDocumentAfterAutoFill( + documentId: string, + totalQuestions: number, + answeredCount: number, + ): Promise { + return updateDocumentAfterAutoFill(documentId, totalQuestions, answeredCount); + } + + // Private helper methods + + private async validateOwnerOrAdmin(organizationId: string, userId: string) { + const member = await db.member.findFirst({ + where: { + organizationId, + userId, + deactivated: false, + }, + }); + + if (!member) { + throw new Error('Member not found'); + } + + const isOwnerOrAdmin = + member.role.includes('owner') || member.role.includes('admin'); + + if (!isOwnerOrAdmin) { + throw new Error('Only owners and admins can perform this action'); + } + + return member; + } + + private async createSOADocumentDirect( + frameworkId: string, + organizationId: string, + configurationId: string, + ) { + const existingLatestDocument = await db.sOADocument.findFirst({ + where: { + frameworkId, + organizationId, + isLatest: true, + }, + }); + + let nextVersion = 1; + if (existingLatestDocument) { + await db.sOADocument.update({ + where: { id: existingLatestDocument.id }, + data: { isLatest: false }, + }); + nextVersion = existingLatestDocument.version + 1; + } + + const configuration = await db.sOAFrameworkConfiguration.findUnique({ + where: { id: configurationId }, + }); + + if (!configuration) { + throw new Error('Configuration not found'); + } + + const questions = configuration.questions as Array<{ id: string }>; + const totalQuestions = Array.isArray(questions) ? questions.length : 0; + + return db.sOADocument.create({ + data: { + frameworkId, + organizationId, + configurationId: configuration.id, + version: nextVersion, + isLatest: true, + status: 'draft', + totalQuestions, + answeredQuestions: 0, + }, + include: { + answers: { where: { isLatestAnswer: true } }, + }, + }); + } + + private async seedISO27001SOAConfig() { + const iso27001Framework = await db.frameworkEditorFramework.findFirst({ + where: { + OR: ISO27001_FRAMEWORK_NAMES.map((name) => ({ name })), + }, + }); + + if (!iso27001Framework) { + throw new Error('ISO 27001 framework not found'); + } + + const existingConfig = await db.sOAFrameworkConfiguration.findFirst({ + where: { + frameworkId: iso27001Framework.id, + isLatest: true, + }, + }); + + if (existingConfig) { + return existingConfig; + } + + const soaConfig = await loadISOConfig(); + + return db.sOAFrameworkConfiguration.create({ + data: { + frameworkId: iso27001Framework.id, + version: 1, + isLatest: true, + columns: soaConfig.columns, + questions: soaConfig.questions, + }, + }); + } +} diff --git a/apps/api/src/soa/utils/constants.ts b/apps/api/src/soa/utils/constants.ts new file mode 100644 index 000000000..91487b2e1 --- /dev/null +++ b/apps/api/src/soa/utils/constants.ts @@ -0,0 +1,127 @@ +/** + * SOA-specific constants and prompts + */ + +// LLM Model identifiers +export const SOA_RAG_MODEL = 'gpt-5-mini'; +export const SOA_BATCH_MODEL = 'gpt-4o-mini'; + +// Supported framework names for ISO 27001 +export const ISO27001_FRAMEWORK_NAMES = ['ISO 27001', 'iso27001', 'ISO27001']; + +// Remote work justification +export const FULLY_REMOTE_JUSTIFICATION = + 'This control is not applicable as our organization operates fully remotely.'; + +// System prompt for SOA RAG generation +export const SOA_RAG_SYSTEM_PROMPT = `You are an expert organizational analyst conducting a comprehensive assessment of a company for ISO 27001 compliance. + +Your task is to analyze the provided context entries and create a structured organizational profile. + +ANALYSIS FRAMEWORK: + +Extract and categorize information about the organization across these dimensions: +- Business type and industry +- Operational scope and scale +- Risk profile and risk management approach +- Regulatory requirements and compliance posture +- Technical infrastructure and security controls +- Organizational policies and procedures +- Governance structure + +CRITICAL RULES - YOU MUST FOLLOW THESE STRICTLY: +1. Answer based EXCLUSIVELY on the provided context from the organization's policies and documentation. +2. DO NOT use general knowledge, assumptions, or information not present in the context. +3. DO NOT hallucinate or invent facts that are not explicitly stated in the context. +4. If the context does not contain enough information to answer the question, respond with exactly: "INSUFFICIENT_DATA" +5. For applicability questions, respond with ONLY "YES" or "NO" - no additional explanation. +6. For justification questions, provide clear, professional explanations (2 sentences) based ONLY on the context provided. +7. Use enterprise-ready language appropriate for ISO 27001 compliance documentation. +8. Always write in first person plural (we, our, us) as if speaking on behalf of the organization. +9. Be precise and factual - base conclusions strictly on the provided evidence. +10. If you cannot find relevant information in the context to answer the question, you MUST respond with "INSUFFICIENT_DATA".`; + +// System prompt for batch SOA generation +export const SOA_BATCH_SYSTEM_PROMPT = `You are an expert organizational analyst conducting a comprehensive assessment of a company for ISO 27001 compliance. + +Your task is to analyze the provided context entries and create a structured organizational profile. + +ANALYSIS FRAMEWORK: + +Extract and categorize information about the organization across these dimensions: +- Business type and industry +- Operational scope and scale +- Risk profile and risk management approach +- Regulatory requirements and compliance posture +- Technical infrastructure and security controls +- Organizational policies and procedures +- Governance structure + +CRITICAL RULES - YOU MUST FOLLOW THESE STRICTLY: +1. Answer based EXCLUSIVELY on the provided context from the organization's policies and documentation. +2. If the context does not contain enough information to answer, respond with exactly: "N/A - no evidence found" +3. BE CONCISE. Give SHORT, direct answers. Do NOT provide detailed explanations or elaborate unnecessarily. +4. Use enterprise-ready language appropriate for SOA documents. +5. If multiple sources provide information, synthesize them into ONE concise answer. +6. Always write in first person plural (we, our, us) as if speaking on behalf of the organization. +7. Justifications should be 2-3 sentences maximum, directly referencing organizational capabilities or documentation.`; + +/** + * Builds the SOA question prompt for a given control + */ +export function buildSOAQuestionPrompt( + title: string, + text: string, +): string { + return `Analyze the control "${title}" (${text}) for our organization. + +Based EXCLUSIVELY on our organization's policies, documentation, business context, and operations, determine: + +1. Is this control applicable to our organization? Consider: + - Our business type and industry + - Our operational scope and scale + - Our risk profile + - Our regulatory requirements + - Our technical infrastructure + - Our existing policies and governance structure + +2. Provide a justification: + - If applicable: Explain how this control is currently implemented in our organization, including our policies, procedures, or technical measures that address this control. + - If not applicable: Explain why this control does not apply to our business context, our operational characteristics that make it irrelevant, and our risk profile considerations. + +Respond in the following JSON format: +{ + "isApplicable": "YES" or "NO", + "justification": "Your justification text here (2-3 sentences)" +} + +If you cannot find sufficient information in the provided context to answer either question, respond with: +{ + "isApplicable": "INSUFFICIENT_DATA", + "justification": null +} + +IMPORTANT: Base your answer ONLY on information found in our organization's documentation. Do NOT use general knowledge or make assumptions.`; +} + +// Indicators that an answer has insufficient data +export const INSUFFICIENT_DATA_INDICATORS = [ + 'INSUFFICIENT_DATA', + 'N/A', + 'NO EVIDENCE FOUND', + 'NOT ENOUGH INFORMATION', + 'INSUFFICIENT', + 'NOT FOUND IN THE CONTEXT', + 'NO INFORMATION AVAILABLE', +]; + +/** + * Checks if an answer indicates insufficient data + */ +export function isInsufficientDataAnswer(answer: string): boolean { + const upperAnswer = answer.toUpperCase(); + return INSUFFICIENT_DATA_INDICATORS.some((indicator) => + upperAnswer.includes(indicator), + ); +} + diff --git a/apps/api/src/soa/utils/soa-answer-generator.ts b/apps/api/src/soa/utils/soa-answer-generator.ts new file mode 100644 index 000000000..306bdfe36 --- /dev/null +++ b/apps/api/src/soa/utils/soa-answer-generator.ts @@ -0,0 +1,269 @@ +import { findSimilarContent, findSimilarContentBatch } from '@/vector-store/lib'; +import type { SimilarContentResult } from '@/vector-store/lib'; +import { openai } from '@ai-sdk/openai'; +import { generateText } from 'ai'; +import { deduplicateSources, type Source } from '@/questionnaire/utils/deduplicate-sources'; +import { + SOA_RAG_MODEL, + SOA_BATCH_MODEL, + SOA_RAG_SYSTEM_PROMPT, + SOA_BATCH_SYSTEM_PROMPT, + buildSOAQuestionPrompt, + isInsufficientDataAnswer, +} from './constants'; + +export interface SOAAnswerWithSources { + answer: string | null; + sources: Source[]; +} + +export interface SOAAnswerLogger { + log: (message: string, meta?: Record) => void; + warn: (message: string, meta?: Record) => void; + error: (message: string, meta?: Record) => void; +} + +const defaultLogger: SOAAnswerLogger = { + log: () => {}, + warn: () => {}, + error: () => {}, +}; + +/** + * Extracts source information from similar content results and deduplicates them + */ +function extractAndDeduplicateSources(similarContent: SimilarContentResult[]): Source[] { + const sourcesBeforeDedup = similarContent.map((result) => { + const r = result as SimilarContentResult; + let sourceName: string | undefined; + + if (r.policyName) { + sourceName = `Policy: ${r.policyName}`; + } else if (r.contextQuestion) { + sourceName = 'Context Q&A'; + } else if (r.sourceType === 'manual_answer') { + sourceName = undefined; + } + + return { + sourceType: r.sourceType, + sourceName, + sourceId: r.sourceId, + policyName: r.policyName, + documentName: r.documentName, + manualAnswerQuestion: r.manualAnswerQuestion, + score: r.score, + }; + }); + + return deduplicateSources(sourcesBeforeDedup); +} + +/** + * Builds context string from similar content for LLM prompt + */ +function buildContextFromContent(similarContent: SimilarContentResult[]): string { + const contextParts = similarContent.map((result, index) => { + const r = result as SimilarContentResult; + let sourceInfo = ''; + + if (r.policyName) { + sourceInfo = `Source: Policy "${r.policyName}"`; + } else if (r.contextQuestion) { + sourceInfo = `Source: Context Q&A`; + } else if (r.sourceType === 'knowledge_base_document') { + sourceInfo = r.documentName + ? `Source: Knowledge Base Document "${r.documentName}"` + : `Source: Knowledge Base Document`; + } else if (r.sourceType === 'manual_answer') { + sourceInfo = `Source: Manual Answer`; + } else { + sourceInfo = `Source: ${r.sourceType}`; + } + + return `[${index + 1}] ${sourceInfo}\n${r.content}`; + }); + + return contextParts.join('\n\n'); +} + +/** + * Generates a SOA answer using RAG (Retrieval-Augmented Generation) + * Performs vector search and LLM generation + */ +export async function generateSOAAnswerWithRAG( + question: string, + organizationId: string, + logger: SOAAnswerLogger = defaultLogger, +): Promise { + try { + // Find similar content from vector database + const similarContent = await findSimilarContent(question, organizationId); + + logger.log('Vector search results for SOA', { + question: question.substring(0, 100), + organizationId, + resultCount: similarContent.length, + }); + + // If no relevant content found, return null + if (similarContent.length === 0) { + logger.warn('No similar content found in vector database for SOA', { + question: question.substring(0, 100), + organizationId, + }); + return { answer: null, sources: [] }; + } + + return generateSOAAnswerFromContent(question, similarContent); + } catch (error) { + logger.error('Failed to generate SOA answer with RAG', { + question: question.substring(0, 100), + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + + return { answer: null, sources: [] }; + } +} + +/** + * Generates a SOA answer from pre-fetched similar content + * Used for batch processing - skips individual vector search + */ +export async function generateSOAAnswerFromContent( + question: string, + similarContent: SimilarContentResult[], +): Promise { + // If no relevant content found, return null + if (similarContent.length === 0) { + return { answer: null, sources: [] }; + } + + // Extract and deduplicate sources + const sources = extractAndDeduplicateSources(similarContent); + + // Build context from retrieved content + const context = buildContextFromContent(similarContent); + + // Generate answer using LLM with ISO 27001 compliance analysis prompt + const { text } = await generateText({ + model: openai(SOA_RAG_MODEL), + system: SOA_RAG_SYSTEM_PROMPT, + prompt: `Based EXCLUSIVELY on the following context from our organization's policies and documentation, answer this question: + +Question: ${question} + +Context: +${context} + +IMPORTANT: Answer the question based ONLY on the provided context above. DO NOT use any general knowledge or assumptions. If the context does not contain enough information to answer the question, respond with exactly "INSUFFICIENT_DATA". Use first person plural (we, our, us) when answering.`, + }); + + const trimmedAnswer = text.trim(); + + // Check if the answer indicates insufficient data + if (isInsufficientDataAnswer(trimmedAnswer)) { + // Try to parse as JSON to check isApplicable field + try { + const parsed = JSON.parse(trimmedAnswer); + if ( + parsed.isApplicable === 'INSUFFICIENT_DATA' || + parsed.isApplicable?.toUpperCase()?.includes('INSUFFICIENT') + ) { + return { answer: null, sources }; + } + } catch { + return { answer: null, sources }; + } + } + + return { answer: trimmedAnswer, sources }; +} + +/** + * Generates SOA answer from pre-fetched content for a specific control question + * Used for batch processing with pre-built question prompts + */ +export async function generateSOAControlAnswer( + question: { + id: string; + text: string; + columnMapping: { + closure: string; + title: string; + control_objective: string | null; + }; + }, + similarContent: SimilarContentResult[], +): Promise { + // If no relevant content found, return null + if (similarContent.length === 0) { + return { answer: null, sources: [] }; + } + + // Extract and deduplicate sources + const sources = extractAndDeduplicateSources(similarContent); + + // Build context from retrieved content + const context = buildContextFromContent(similarContent); + + // Build the SOA question prompt + const soaQuestion = buildSOAQuestionPrompt( + question.columnMapping.title, + question.text, + ); + + // Generate answer using LLM + const { text } = await generateText({ + model: openai(SOA_BATCH_MODEL), + system: SOA_BATCH_SYSTEM_PROMPT, + prompt: `Based on the following context from our organization's policies and documentation, analyze this SOA question: + +Question: ${soaQuestion} + +Context: +${context} + +Provide your analysis in the exact JSON format specified. If the context doesn't contain sufficient information, respond with "INSUFFICIENT_DATA".`, + }); + + return { answer: text.trim(), sources }; +} + +/** + * Batch fetch similar content for all SOA questions + * Uses batch embedding generation for significant speedup + */ +export async function batchSearchSOAQuestions( + questions: Array<{ + id: string; + text: string; + columnMapping: { + closure: string; + title: string; + control_objective: string | null; + }; + }>, + organizationId: string, +): Promise> { + // Build the SOA question texts + const questionTexts = questions.map((question) => + buildSOAQuestionPrompt(question.columnMapping.title, question.text), + ); + + // Batch search all questions + const allSimilarContent = await findSimilarContentBatch( + questionTexts, + organizationId, + ); + + // Create map by question ID + const contentMap = new Map(); + questions.forEach((question, index) => { + contentMap.set(question.id, allSimilarContent[index] || []); + }); + + return contentMap; +} + diff --git a/apps/api/src/soa/utils/soa-answer-parser.ts b/apps/api/src/soa/utils/soa-answer-parser.ts new file mode 100644 index 000000000..ab85e77e4 --- /dev/null +++ b/apps/api/src/soa/utils/soa-answer-parser.ts @@ -0,0 +1,189 @@ +import { isInsufficientDataAnswer, FULLY_REMOTE_JUSTIFICATION } from './constants'; + +export interface SOAQuestionResult { + questionId: string; + isApplicable: boolean | null; + justification: string | null; + success: boolean; + insufficientData?: boolean; + error?: string; +} + +export interface SOAQuestion { + id: string; + text: string; + columnMapping: { + closure: string; + title: string; + control_objective: string | null; + isApplicable: boolean | null; + justification: string | null; + }; +} + +export type SOAStreamSender = (data: { + type: string; + questionId?: string; + questionIndex?: number; + isApplicable?: boolean | null; + justification?: string | null; + success?: boolean; + insufficientData?: boolean; +}) => void; + +/** + * Creates a default YES result (used when insufficient data) + */ +export function createDefaultYesResult( + questionId: string, + index: number, + send: SOAStreamSender, +): SOAQuestionResult { + send({ + type: 'answer', + questionId, + questionIndex: index, + isApplicable: true, + justification: null, + success: true, + insufficientData: false, + }); + + return { + questionId, + isApplicable: true, + justification: null, + success: true, + insufficientData: false, + }; +} + +/** + * Creates a fully remote NO result (used for control 7.x when org is fully remote) + */ +export function createFullyRemoteResult( + questionId: string, + index: number, + send: SOAStreamSender, +): SOAQuestionResult { + send({ + type: 'answer', + questionId, + questionIndex: index, + isApplicable: false, + justification: FULLY_REMOTE_JUSTIFICATION, + success: true, + insufficientData: false, + }); + + return { + questionId, + isApplicable: false, + justification: FULLY_REMOTE_JUSTIFICATION, + success: true, + insufficientData: false, + }; +} + +/** + * Checks if a control is for physical security (section 7) + */ +export function isPhysicalSecurityControl(closure: string): boolean { + return closure.startsWith('7.'); +} + +/** + * Parses and processes a SOA answer response from LLM + * Returns structured result with isApplicable and justification + */ +export function parseAndProcessSOAAnswer( + questionId: string, + index: number, + answerText: string, + send: SOAStreamSender, +): SOAQuestionResult { + // Parse JSON response + let parsedAnswer: { + isApplicable?: string; + justification?: string | null; + } | null = null; + + try { + parsedAnswer = JSON.parse(answerText); + } catch { + // If JSON parsing fails, try to extract from text + const trimmedAnswer = answerText.trim(); + + // Check for insufficient data indicators - if insufficient, default to YES + if (isInsufficientDataAnswer(trimmedAnswer)) { + return createDefaultYesResult(questionId, index, send); + } + + // Try to extract YES/NO and justification from text + const isApplicableMatch = trimmedAnswer.match( + /(?:isApplicable|applicable)[:\s]*["']?(YES|NO|INSUFFICIENT_DATA)["']?/i, + ); + const justificationMatch = trimmedAnswer.match( + /(?:justification)[:\s]*["']?([^"']{20,})["']?/i, + ); + + parsedAnswer = { + isApplicable: isApplicableMatch + ? isApplicableMatch[1].toUpperCase() + : undefined, + justification: justificationMatch ? justificationMatch[1].trim() : null, + }; + } + + // Check for insufficient data - if insufficient, default to YES + if ( + !parsedAnswer || + !parsedAnswer.isApplicable || + parsedAnswer.isApplicable === 'INSUFFICIENT_DATA' || + parsedAnswer.isApplicable.toUpperCase().includes('INSUFFICIENT') + ) { + return createDefaultYesResult(questionId, index, send); + } + + // Parse isApplicable + const isApplicableText = parsedAnswer.isApplicable.toUpperCase(); + const isApplicable = + isApplicableText.includes('YES') || + isApplicableText.includes('APPLICABLE'); + const isNotApplicable = + isApplicableText.includes('NO') || + isApplicableText.includes('NOT APPLICABLE'); + + let finalIsApplicable: boolean | null = null; + if (isApplicable && !isNotApplicable) { + finalIsApplicable = true; + } else if (isNotApplicable && !isApplicable) { + finalIsApplicable = false; + } else { + // Can't determine YES/NO - default to YES + return createDefaultYesResult(questionId, index, send); + } + + // Get justification (only if NO) + const justification = + finalIsApplicable === false ? parsedAnswer.justification || null : null; + + send({ + type: 'answer', + questionId, + questionIndex: index, + isApplicable: finalIsApplicable, + justification, + success: true, + insufficientData: false, + }); + + return { + questionId, + isApplicable: finalIsApplicable, + justification, + success: true, + insufficientData: false, + }; +} + diff --git a/apps/api/src/soa/utils/soa-storage.ts b/apps/api/src/soa/utils/soa-storage.ts new file mode 100644 index 000000000..5fbcde573 --- /dev/null +++ b/apps/api/src/soa/utils/soa-storage.ts @@ -0,0 +1,222 @@ +import { db } from '@db'; +import type { SOAQuestion, SOAQuestionResult } from './soa-answer-parser'; + +export interface SOAStorageLogger { + log: (message: string, meta?: Record) => void; + error: (message: string, meta?: Record) => void; +} + +const defaultLogger: SOAStorageLogger = { + log: () => {}, + error: () => {}, +}; + +/** + * Saves auto-generated SOA answers to the database + */ +export async function saveAnswersToDatabase( + documentId: string, + questions: SOAQuestion[], + results: SOAQuestionResult[], + userId: string, + logger: SOAStorageLogger = defaultLogger, +): Promise { + const successfulResults = results.filter( + (r) => r.success && r.isApplicable !== null, + ); + + for (const result of successfulResults) { + const question = questions.find((q) => q.id === result.questionId); + if (!question) continue; + + try { + // Get existing answer to determine version + const existingAnswer = await db.sOAAnswer.findFirst({ + where: { + documentId, + questionId: question.id, + isLatestAnswer: true, + }, + orderBy: { + answerVersion: 'desc', + }, + }); + + const nextVersion = existingAnswer + ? existingAnswer.answerVersion + 1 + : 1; + + // Mark existing answer as not latest if it exists + if (existingAnswer) { + await db.sOAAnswer.update({ + where: { id: existingAnswer.id }, + data: { isLatestAnswer: false }, + }); + } + + // Store justification in answer field only if isApplicable is NO + const answerValue = + result.isApplicable === false ? result.justification : null; + + // Create new answer + await db.sOAAnswer.create({ + data: { + documentId, + questionId: question.id, + answer: answerValue, + status: 'generated', + generatedAt: new Date(), + answerVersion: nextVersion, + isLatestAnswer: true, + createdBy: userId, + }, + }); + } catch (error) { + logger.error('Failed to save SOA answer', { + questionId: question.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } +} + +/** + * Updates SOA configuration with auto-fill results + */ +export async function updateConfigurationWithResults( + configurationId: string, + configurationQuestions: SOAQuestion[], + results: SOAQuestionResult[], +): Promise { + const resultsMap = new Map( + results + .filter((r) => r.success && r.isApplicable !== null) + .map((r) => [r.questionId, r]), + ); + + const updatedQuestions = configurationQuestions.map((q) => { + const result = resultsMap.get(q.id); + if (result) { + return { + ...q, + columnMapping: { + ...q.columnMapping, + isApplicable: result.isApplicable, + justification: result.justification, + }, + }; + } + return q; + }); + + await db.sOAFrameworkConfiguration.update({ + where: { id: configurationId }, + data: { + questions: JSON.parse(JSON.stringify(updatedQuestions)), + }, + }); +} + +/** + * Updates SOA document status after auto-fill + */ +export async function updateDocumentAfterAutoFill( + documentId: string, + totalQuestions: number, + answeredCount: number, +): Promise { + await db.sOADocument.update({ + where: { id: documentId }, + data: { + answeredQuestions: answeredCount, + status: answeredCount === totalQuestions ? 'completed' : 'in_progress', + completedAt: answeredCount === totalQuestions ? new Date() : null, + approverId: null, + approvedAt: null, + }, + }); +} + +/** + * Gets the answered questions count from configuration + */ +export async function getAnsweredCountFromConfiguration( + configurationId: string, +): Promise { + const configuration = await db.sOAFrameworkConfiguration.findUnique({ + where: { id: configurationId }, + }); + + if (!configuration) return 0; + + const questions = configuration.questions as Array<{ + id: string; + columnMapping: { + isApplicable: boolean | null; + }; + }>; + + return questions.filter((q) => q.columnMapping.isApplicable !== null).length; +} + +/** + * Updates document answered count and status + */ +export async function updateDocumentAnsweredCount( + documentId: string, + totalQuestions: number, + answeredCount: number, +): Promise { + await db.sOADocument.update({ + where: { id: documentId }, + data: { + answeredQuestions: answeredCount, + status: answeredCount === totalQuestions ? 'completed' : 'in_progress', + completedAt: answeredCount === totalQuestions ? new Date() : null, + // Clear approval when answers are edited + approverId: null, + approvedAt: null, + }, + }); +} + +/** + * Checks if organization is fully remote based on context + */ +export async function checkIfFullyRemote( + organizationId: string, + logger: SOAStorageLogger = defaultLogger, +): Promise { + try { + const teamWorkContext = await db.context.findFirst({ + where: { + organizationId, + question: { + contains: 'How does your team work', + mode: 'insensitive', + }, + }, + }); + + logger.log('Team work context check for SOA auto-fill', { + organizationId, + found: !!teamWorkContext, + }); + + if (teamWorkContext?.answer) { + const answerLower = teamWorkContext.answer.toLowerCase(); + return ( + answerLower.includes('fully remote') || + answerLower.includes('fully-remote') + ); + } + return false; + } catch (error) { + logger.error('Failed to check team work mode for SOA', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return false; + } +} + diff --git a/apps/api/src/soa/utils/transform-iso-config.ts b/apps/api/src/soa/utils/transform-iso-config.ts new file mode 100644 index 000000000..512aa8f1f --- /dev/null +++ b/apps/api/src/soa/utils/transform-iso-config.ts @@ -0,0 +1,112 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; + +/** + * Transforms ISO control JSON into SOA configuration format + */ + +type ISOControl = { + title: string; + control_objective: string | null; + closure: string; + isApplicable: boolean | null; +}; + +type SOAColumn = { + name: string; + type: 'string' | 'boolean' | 'text'; +}; + +type SOAQuestion = { + id: string; + text: string; + columnMapping: { + title: string; + closure: string; + control_objective: string | null; + isApplicable: boolean | null; + justification: string | null; + }; +}; + +type SOAConfiguration = { + columns: SOAColumn[]; + questions: SOAQuestion[]; +}; + +export function transformISOConfigToSOA( + controls: ISOControl[], +): SOAConfiguration { + const columns: SOAColumn[] = [ + { name: 'closure', type: 'string' }, + { name: 'title', type: 'string' }, + { name: 'control_objective', type: 'string' }, + { name: 'isApplicable', type: 'boolean' }, + { name: 'justification', type: 'string' }, + ]; + + const questions: SOAQuestion[] = controls + .filter((control) => { + return ( + control.title && + control.control_objective !== null && + control.control_objective.trim() !== '' + ); + }) + .map((control, index) => { + const id = `iso-control-${index}-${control.title.toLowerCase().replace(/\s+/g, '-').slice(0, 30)}`; + + return { + id, + text: control.control_objective || control.title, + columnMapping: { + closure: control.closure, + title: control.title, + control_objective: control.control_objective, + isApplicable: control.isApplicable ?? null, + justification: null, + }, + }; + }); + + return { + columns, + questions, + }; +} + +/** + * Loads and transforms ISO config JSON file + */ +export async function loadISOConfig(): Promise { + // Use fs.readFileSync instead of import to avoid ESM import attribute issues + // Read from source directory since JSON files aren't copied to dist during compilation + // __dirname in compiled code is dist/src/soa/utils + // We need to reference the source file relative to the project root (apps/api) + // From dist/src/soa/utils, go up to dist/src, then replace 'dist' with 'src' + const sourceDir = __dirname.replace(/dist[\\/]src/, 'src'); + const configPath = join(sourceDir, '../seedJson/ISO/config.json'); + + try { + const configContent = readFileSync(configPath, 'utf-8'); + const isoControls: ISOControl[] = JSON.parse(configContent); + return transformISOConfigToSOA(isoControls); + } catch (error) { + // Fallback: try using process.cwd() (should be apps/api when running) + const fallbackPath = join( + process.cwd(), + 'src/soa/seedJson/ISO/config.json', + ); + try { + const configContent = readFileSync(fallbackPath, 'utf-8'); + const isoControls: ISOControl[] = JSON.parse(configContent); + return transformISOConfigToSOA(isoControls); + } catch { + throw new Error( + `Failed to load ISO config: ${error instanceof Error ? error.message : 'Unknown error'}. ` + + `Tried paths: ${configPath}, ${fallbackPath}. ` + + `__dirname: ${__dirname}, process.cwd(): ${process.cwd()}`, + ); + } + } +} diff --git a/apps/api/src/trust-portal/dto/compliance-resource.dto.ts b/apps/api/src/trust-portal/dto/compliance-resource.dto.ts new file mode 100644 index 000000000..e31c4a0ee --- /dev/null +++ b/apps/api/src/trust-portal/dto/compliance-resource.dto.ts @@ -0,0 +1,71 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsEnum, IsString } from 'class-validator'; +import { TrustFramework } from '@prisma/client'; + +export class ComplianceResourceBaseDto { + @ApiProperty({ + description: 'Organization ID that owns the compliance resource', + example: 'org_6914cd0e16e4c7dccbb54426', + }) + @IsString() + organizationId!: string; + + @ApiProperty({ + description: 'Compliance framework identifier', + enum: TrustFramework, + example: TrustFramework.iso_27001, + }) + @IsEnum(TrustFramework) + framework!: TrustFramework; +} + +export class UploadComplianceResourceDto extends ComplianceResourceBaseDto { + @ApiProperty({ + description: 'Original file name (PDF only)', + example: 'iso-27001-certificate.pdf', + }) + @IsString() + fileName!: string; + + @ApiProperty({ + description: 'MIME type of the file', + example: 'application/pdf', + }) + @IsString() + fileType!: string; + + @ApiProperty({ + description: 'Base64 encoded PDF content', + }) + @IsString() + fileData!: string; +} + +export class ComplianceResourceSignedUrlDto extends ComplianceResourceBaseDto {} + +export class ComplianceResourceResponseDto { + @ApiProperty({ enum: TrustFramework }) + framework!: TrustFramework; + + @ApiProperty() + fileName!: string; + + @ApiProperty({ description: 'File size in bytes' }) + fileSize!: number; + + @ApiProperty({ + description: 'ISO timestamp when the certificate was last updated', + }) + updatedAt!: string; +} + +export class ComplianceResourceUrlResponseDto { + @ApiProperty() + signedUrl!: string; + + @ApiProperty() + fileName!: string; + + @ApiProperty({ description: 'File size in bytes' }) + fileSize!: number; +} diff --git a/apps/api/src/trust-portal/nda-pdf.service.ts b/apps/api/src/trust-portal/nda-pdf.service.ts index 4b5de879c..18a97852e 100644 --- a/apps/api/src/trust-portal/nda-pdf.service.ts +++ b/apps/api/src/trust-portal/nda-pdf.service.ts @@ -137,12 +137,15 @@ By signing below, the Receiving Party agrees to be bound by the terms of this Ag name: string, email: string, agreementId: string, + customWatermarkText?: string, ) { const font = await pdfDoc.embedFont(StandardFonts.HelveticaBold); const pages = pdfDoc.getPages(); const timestamp = new Date().toISOString(); - const watermarkText = `For: ${name} <${email}> | ${timestamp} | ID: ${agreementId}`; + const watermarkText = + customWatermarkText || + `For: ${name} <${email}> | ${timestamp} | ID: ${agreementId}`; for (const page of pages) { const { width, height } = page.getSize(); @@ -235,11 +238,12 @@ By signing below, the Receiving Party agrees to be bound by the terms of this Ag name: string; email: string; docId: string; + watermarkText?: string; }, ): Promise { - const { name, email, docId } = params; + const { name, email, docId, watermarkText } = params; const pdfDoc = await PDFDocument.load(pdfBuffer); - await this.addWatermark(pdfDoc, name, email, docId); + await this.addWatermark(pdfDoc, name, email, docId, watermarkText); const pdfBytes = await pdfDoc.save(); return Buffer.from(pdfBytes); } diff --git a/apps/api/src/trust-portal/trust-access.controller.ts b/apps/api/src/trust-portal/trust-access.controller.ts index 895e3f076..7e4afec7a 100644 --- a/apps/api/src/trust-portal/trust-access.controller.ts +++ b/apps/api/src/trust-portal/trust-access.controller.ts @@ -15,6 +15,7 @@ import { ApiHeader, ApiOperation, ApiParam, + ApiQuery, ApiResponse, ApiSecurity, ApiTags, @@ -30,6 +31,7 @@ import { ReclaimAccessDto, RevokeGrantDto, } from './dto/trust-access.dto'; +import { TrustFramework } from '@prisma/client'; import { SignNdaDto } from './dto/nda.dto'; import { TrustAccessService } from './trust-access.service'; @@ -375,6 +377,12 @@ export class TrustAccessController { name: 'friendlyUrl', description: 'Trust Portal friendly URL or Organization ID', }) + @ApiQuery({ + name: 'query', + required: false, + description: 'Query parameter to append to the access link (e.g., security-questionnaire)', + example: 'security-questionnaire', + }) @ApiResponse({ status: HttpStatus.OK, description: 'Access link sent to email', @@ -383,8 +391,9 @@ export class TrustAccessController { // Note: friendlyUrl can be either the custom friendly URL or the organization ID @Param('friendlyUrl') friendlyUrl: string, @Body() dto: ReclaimAccessDto, + @Query('query') query?: string, ) { - return this.trustAccessService.reclaimAccess(friendlyUrl, dto.email); + return this.trustAccessService.reclaimAccess(friendlyUrl, dto.email, query); } @Get('access/:token') @@ -429,4 +438,51 @@ export class TrustAccessController { async downloadAllPolicies(@Param('token') token: string) { return this.trustAccessService.downloadAllPoliciesByAccessToken(token); } + + @Get('access/:token/compliance-resources') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'List compliance resources by access token', + description: 'Get list of uploaded compliance certificates for the organization', + }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Compliance resources list returned', + }) + async getComplianceResourcesByAccessToken(@Param('token') token: string) { + return this.trustAccessService.getComplianceResourcesByAccessToken(token); + } + + @Get('access/:token/compliance-resources/:framework') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Download compliance resource by access token', + description: 'Get signed URL to download a specific compliance certificate file', + }) + @ApiParam({ + name: 'framework', + enum: Object.values(TrustFramework), + description: 'Compliance framework identifier', + }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Signed URL for compliance resource returned', + }) + @ApiResponse({ + status: HttpStatus.NOT_FOUND, + description: 'Compliance resource not found', + }) + @ApiResponse({ + status: HttpStatus.BAD_REQUEST, + description: 'Invalid framework or access token', + }) + async getComplianceResourceUrlByAccessToken( + @Param('token') token: string, + @Param('framework') framework: string, + ) { + return this.trustAccessService.getComplianceResourceUrlByAccessToken( + token, + framework as any, + ); + } } diff --git a/apps/api/src/trust-portal/trust-access.service.ts b/apps/api/src/trust-portal/trust-access.service.ts index 930bede9b..8fe183e48 100644 --- a/apps/api/src/trust-portal/trust-access.service.ts +++ b/apps/api/src/trust-portal/trust-access.service.ts @@ -1,9 +1,10 @@ import { BadRequestException, Injectable, + InternalServerErrorException, NotFoundException, } from '@nestjs/common'; -import { db } from '@trycompai/db'; +import { db } from '@db'; import { randomBytes } from 'crypto'; import { ApproveAccessRequestDto, @@ -16,6 +17,10 @@ import { TrustEmailService } from './email.service'; import { NdaPdfService } from './nda-pdf.service'; import { AttachmentsService } from '../attachments/attachments.service'; import { PolicyPdfRendererService } from './policy-pdf-renderer.service'; +import { GetObjectCommand } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { APP_AWS_ORG_ASSETS_BUCKET, s3Client } from '../app/s3'; +import { TrustFramework } from '@prisma/client'; @Injectable() export class TrustAccessService { @@ -840,7 +845,7 @@ export class TrustAccessService { }; } - async reclaimAccess(id: string, email: string) { + async reclaimAccess(id: string, email: string, query?: string) { const trust = await this.findPublishedTrustByRouteId(id); const grant = await db.trustAccessGrant.findFirst({ @@ -892,7 +897,13 @@ export class TrustAccessService { } const urlId = trust.friendlyUrl || trust.organizationId; - const accessLink = `${this.TRUST_APP_URL}/${urlId}/access/${accessToken}`; + let accessLink = `${this.TRUST_APP_URL}/${urlId}/access/${accessToken}`; + + // Append query parameter if provided + if (query) { + const separator = accessLink.includes('?') ? '&' : '?'; + accessLink = `${accessLink}${separator}query=${encodeURIComponent(query)}`; + } await this.emailService.sendAccessReclaimEmail({ toEmail: email, @@ -953,6 +964,11 @@ export class TrustAccessService { }; } + async validateAccessTokenAndGetOrganizationId(token: string): Promise { + const grant = await this.validateAccessToken(token); + return grant.accessRequest.organizationId; + } + private async validateAccessToken(token: string) { const grant = await db.trustAccessGrant.findUnique({ where: { accessToken: token }, @@ -1037,6 +1053,144 @@ export class TrustAccessService { return policies; } + async getComplianceResourcesByAccessToken(token: string) { + const grant = await db.trustAccessGrant.findUnique({ + where: { accessToken: token }, + include: { + accessRequest: { + include: { + organization: true, + }, + }, + }, + }); + + if (!grant) { + throw new NotFoundException('Invalid access token'); + } + + if (grant.status !== 'active') { + throw new BadRequestException('Access grant is not active'); + } + + if (grant.expiresAt < new Date()) { + throw new BadRequestException('Access grant has expired'); + } + + if ( + !grant.accessTokenExpiresAt || + grant.accessTokenExpiresAt < new Date() + ) { + throw new BadRequestException('Access token has expired'); + } + + const complianceResources = await db.trustResource.findMany({ + where: { + organizationId: grant.accessRequest.organizationId, + }, + select: { + framework: true, + fileName: true, + fileSize: true, + updatedAt: true, + }, + orderBy: { + updatedAt: 'desc', + }, + }); + + return complianceResources.map((resource) => ({ + framework: resource.framework, + fileName: resource.fileName, + fileSize: resource.fileSize, + updatedAt: resource.updatedAt.toISOString(), + })); + } + + async getComplianceResourceUrlByAccessToken( + token: string, + framework: TrustFramework, + ) { + const grant = await this.validateAccessToken(token); + + // Validate framework enum + if (!Object.values(TrustFramework).includes(framework)) { + throw new BadRequestException(`Invalid framework: ${framework}`); + } + + if (!s3Client || !APP_AWS_ORG_ASSETS_BUCKET) { + throw new InternalServerErrorException( + 'Organization assets bucket is not configured', + ); + } + + const record = await db.trustResource.findUnique({ + where: { + organizationId_framework: { + organizationId: grant.accessRequest.organizationId, + framework, + }, + }, + }); + + if (!record) { + throw new NotFoundException( + `No certificate uploaded for framework ${framework}`, + ); + } + + // Download the original PDF from S3 + const getCommand = new GetObjectCommand({ + Bucket: APP_AWS_ORG_ASSETS_BUCKET, + Key: record.s3Key, + }); + + const response = await s3Client.send(getCommand); + const chunks: Uint8Array[] = []; + + if (!response.Body) { + throw new InternalServerErrorException('No file data received from S3'); + } + + for await (const chunk of response.Body as any) { + chunks.push(chunk); + } + + const originalPdfBuffer = Buffer.concat(chunks); + + // Watermark the PDF + const docId = `compliance-${grant.id}-${framework}-${Date.now()}`; + const watermarked = await this.ndaPdfService.watermarkExistingPdf( + originalPdfBuffer, + { + name: grant.accessRequest.name, + email: grant.subjectEmail, + docId, + watermarkText: 'Comp AI', + }, + ); + + // Upload watermarked PDF to S3 + const key = await this.attachmentsService.uploadToS3( + watermarked, + `compliance-${framework}-grant-${grant.id}-${Date.now()}.pdf`, + 'application/pdf', + grant.accessRequest.organizationId, + 'trust_compliance_downloads', + `${grant.id}`, + ); + + // Generate signed URL for the watermarked PDF + const downloadUrl = + await this.attachmentsService.getPresignedDownloadUrl(key); + + return { + signedUrl: downloadUrl, + fileName: record.fileName, + fileSize: watermarked.length, + }; + } + async downloadAllPoliciesByAccessToken(token: string) { const grant = await this.validateAccessToken(token); diff --git a/apps/api/src/trust-portal/trust-portal.controller.ts b/apps/api/src/trust-portal/trust-portal.controller.ts index 9a102882f..e1cbb5869 100644 --- a/apps/api/src/trust-portal/trust-portal.controller.ts +++ b/apps/api/src/trust-portal/trust-portal.controller.ts @@ -1,26 +1,49 @@ import { + BadRequestException, + Body, Controller, Get, HttpCode, HttpStatus, + Post, Query, UseGuards, } from '@nestjs/common'; import { + ApiBody, ApiHeader, ApiOperation, + ApiProperty, ApiQuery, ApiResponse, ApiSecurity, ApiTags, } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; import { HybridAuthGuard } from '../auth/hybrid-auth.guard'; +import { AuthContext } from '../auth/auth-context.decorator'; +import type { AuthContext as AuthContextType } from '../auth/types'; import { DomainStatusResponseDto, GetDomainStatusDto, } from './dto/domain-status.dto'; +import { + ComplianceResourceResponseDto, + ComplianceResourceSignedUrlDto, + ComplianceResourceUrlResponseDto, + UploadComplianceResourceDto, +} from './dto/compliance-resource.dto'; import { TrustPortalService } from './trust-portal.service'; +class ListComplianceResourcesDto { + @ApiProperty({ + description: 'Organization ID that owns the compliance resources', + example: 'org_6914cd0e16e4c7dccbb54426', + }) + @IsString() + organizationId!: string; +} + @ApiTags('Trust Portal') @Controller({ path: 'trust-portal', version: '1' }) @UseGuards(HybridAuthGuard) @@ -65,4 +88,79 @@ export class TrustPortalController { ): Promise { return this.trustPortalService.getDomainStatus(dto); } + + @Post('compliance-resources/upload') + @HttpCode(HttpStatus.CREATED) + @ApiOperation({ + summary: 'Upload or replace a compliance certificate (PDF only)', + description: + 'Stores the compliance certificate in the organization assets bucket and replaces any previous file for the same framework.', + }) + @ApiBody({ type: UploadComplianceResourceDto }) + @ApiResponse({ + status: HttpStatus.CREATED, + description: 'Compliance certificate uploaded successfully', + type: ComplianceResourceResponseDto, + }) + @ApiResponse({ + status: HttpStatus.BAD_REQUEST, + description: + 'Framework not compliant, PDF validation failed, or organization mismatch', + }) + async uploadComplianceResource( + @Body() dto: UploadComplianceResourceDto, + @AuthContext() authContext: AuthContextType, + ): Promise { + this.assertOrganizationAccess(dto.organizationId, authContext); + return this.trustPortalService.uploadComplianceResource(dto); + } + + @Post('compliance-resources/signed-url') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Generate a temporary signed URL for a compliance certificate', + }) + @ApiBody({ type: ComplianceResourceSignedUrlDto }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Signed URL generated successfully', + type: ComplianceResourceUrlResponseDto, + }) + async getComplianceResourceUrl( + @Body() dto: ComplianceResourceSignedUrlDto, + @AuthContext() authContext: AuthContextType, + ): Promise { + this.assertOrganizationAccess(dto.organizationId, authContext); + return this.trustPortalService.getComplianceResourceUrl(dto); + } + + @Post('compliance-resources/list') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'List uploaded compliance certificates for the organization', + }) + @ApiBody({ type: ListComplianceResourcesDto }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Compliance certificates retrieved successfully', + type: [ComplianceResourceResponseDto], + }) + async listComplianceResources( + @Body() dto: ListComplianceResourcesDto, + @AuthContext() authContext: AuthContextType, + ): Promise { + this.assertOrganizationAccess(dto.organizationId, authContext); + return this.trustPortalService.listComplianceResources(dto.organizationId); + } + + private assertOrganizationAccess( + organizationId: string, + authContext: AuthContextType, + ): void { + if (organizationId !== authContext.organizationId) { + throw new BadRequestException( + 'Organization mismatch. You can only manage resources for your own organization.', + ); + } + } } diff --git a/apps/api/src/trust-portal/trust-portal.service.ts b/apps/api/src/trust-portal/trust-portal.service.ts index 185d7d0f0..243d6a1da 100644 --- a/apps/api/src/trust-portal/trust-portal.service.ts +++ b/apps/api/src/trust-portal/trust-portal.service.ts @@ -1,15 +1,31 @@ +import { Prisma, TrustFramework } from '@prisma/client'; import { BadRequestException, Injectable, InternalServerErrorException, Logger, + NotFoundException, } from '@nestjs/common'; import axios, { AxiosInstance } from 'axios'; +import { + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { db } from '@db'; import { DomainStatusResponseDto, DomainVerificationDto, GetDomainStatusDto, } from './dto/domain-status.dto'; +import { + ComplianceResourceResponseDto, + ComplianceResourceSignedUrlDto, + ComplianceResourceUrlResponseDto, + UploadComplianceResourceDto, +} from './dto/compliance-resource.dto'; +import { APP_AWS_ORG_ASSETS_BUCKET, s3Client } from '../app/s3'; interface VercelDomainVerification { type: string; @@ -28,6 +44,8 @@ interface VercelDomainResponse { export class TrustPortalService { private readonly logger = new Logger(TrustPortalService.name); private readonly vercelApi: AxiosInstance; + private readonly MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; + private readonly SIGNED_URL_EXPIRY_SECONDS = 900; constructor() { const bearerToken = process.env.VERCEL_ACCESS_TOKEN; @@ -46,6 +64,79 @@ export class TrustPortalService { }); } + private static readonly FRAMEWORK_CONFIG: Record< + TrustFramework, + { + statusField: + | 'iso27001_status' + | 'iso42001_status' + | 'gdpr_status' + | 'hipaa_status' + | 'soc2type1_status' + | 'soc2type2_status' + | 'pci_dss_status' + | 'nen7510_status' + | 'iso9001_status'; + enabledField: + | 'iso27001' + | 'iso42001' + | 'gdpr' + | 'hipaa' + | 'soc2type1' + | 'soc2type2' + | 'pci_dss' + | 'nen7510' + | 'iso9001'; + slug: string; + } + > = { + [TrustFramework.iso_27001]: { + statusField: 'iso27001_status', + enabledField: 'iso27001', + slug: 'iso_27001', + }, + [TrustFramework.iso_42001]: { + statusField: 'iso42001_status', + enabledField: 'iso42001', + slug: 'iso_42001', + }, + [TrustFramework.gdpr]: { + statusField: 'gdpr_status', + enabledField: 'gdpr', + slug: 'gdpr', + }, + [TrustFramework.hipaa]: { + statusField: 'hipaa_status', + enabledField: 'hipaa', + slug: 'hipaa', + }, + [TrustFramework.soc2_type1]: { + statusField: 'soc2type1_status', + enabledField: 'soc2type1', + slug: 'soc2_type1', + }, + [TrustFramework.soc2_type2]: { + statusField: 'soc2type2_status', + enabledField: 'soc2type2', + slug: 'soc2_type2', + }, + [TrustFramework.pci_dss]: { + statusField: 'pci_dss_status', + enabledField: 'pci_dss', + slug: 'pci_dss', + }, + [TrustFramework.nen_7510]: { + statusField: 'nen7510_status', + enabledField: 'nen7510', + slug: 'nen_7510', + }, + [TrustFramework.iso_9001]: { + statusField: 'iso9001_status', + enabledField: 'iso9001', + slug: 'iso_9001', + }, + }; + async getDomainStatus( dto: GetDomainStatusDto, ): Promise { @@ -114,4 +205,211 @@ export class TrustPortalService { ); } } + + async uploadComplianceResource( + dto: UploadComplianceResourceDto, + ): Promise { + this.ensureS3Availability(); + await this.assertFrameworkIsCompliant(dto.organizationId, dto.framework); + + const { fileBuffer, sanitizedFileName } = this.preparePdfPayload(dto); + const slug = TrustPortalService.FRAMEWORK_CONFIG[dto.framework].slug; + const timestamp = Date.now(); + const s3Prefix = `${dto.organizationId}/resources/${slug}`; + const s3Key = `${s3Prefix}/${timestamp}-${sanitizedFileName}`; + + const existingResource = await db.trustResource.findUnique({ + where: { + organizationId_framework: { + organizationId: dto.organizationId, + framework: dto.framework, + }, + }, + }); + + if (existingResource) { + await this.safeDeleteObject(existingResource.s3Key); + } + + const putCommand = new PutObjectCommand({ + Bucket: APP_AWS_ORG_ASSETS_BUCKET, + Key: s3Key, + Body: fileBuffer, + ContentType: 'application/pdf', + Metadata: { + organizationId: dto.organizationId, + framework: slug, + originalFileName: dto.fileName, + }, + }); + + await s3Client!.send(putCommand); + + const record = await db.trustResource.upsert({ + where: { + organizationId_framework: { + organizationId: dto.organizationId, + framework: dto.framework, + }, + }, + update: { + s3Key, + fileName: dto.fileName, + fileSize: fileBuffer.length, + }, + create: { + organizationId: dto.organizationId, + framework: dto.framework, + s3Key, + fileName: dto.fileName, + fileSize: fileBuffer.length, + }, + }); + + return { + framework: record.framework, + fileName: record.fileName, + fileSize: record.fileSize, + updatedAt: record.updatedAt.toISOString(), + }; + } + + async listComplianceResources( + organizationId: string, + ): Promise { + const records = await db.trustResource.findMany({ + where: { + organizationId, + }, + orderBy: { + updatedAt: 'desc', + }, + }); + + return records.map((record) => ({ + framework: record.framework, + fileName: record.fileName, + fileSize: record.fileSize, + updatedAt: record.updatedAt.toISOString(), + })); + } + + async getComplianceResourceUrl( + dto: ComplianceResourceSignedUrlDto, + ): Promise { + this.ensureS3Availability(); + + const record = await db.trustResource.findUnique({ + where: { + organizationId_framework: { + organizationId: dto.organizationId, + framework: dto.framework, + }, + }, + }); + + if (!record) { + throw new NotFoundException( + `No certificate uploaded for framework ${dto.framework}`, + ); + } + + const getCommand = new GetObjectCommand({ + Bucket: APP_AWS_ORG_ASSETS_BUCKET, + Key: record.s3Key, + }); + + const signedUrl = await getSignedUrl(s3Client!, getCommand, { + expiresIn: this.SIGNED_URL_EXPIRY_SECONDS, + }); + + return { + signedUrl, + fileName: record.fileName, + fileSize: record.fileSize, + }; + } + + private async assertFrameworkIsCompliant( + organizationId: string, + framework: TrustFramework, + ): Promise { + const config = TrustPortalService.FRAMEWORK_CONFIG[framework]; + const trustRecord = await db.trust.findUnique({ + where: { organizationId }, + }); + + if (!trustRecord) { + throw new BadRequestException( + 'Trust portal configuration not found for organization', + ); + } + + if (!trustRecord[config.enabledField]) { + throw new BadRequestException( + `Framework ${framework} is not enabled for this organization`, + ); + } + + if (trustRecord[config.statusField] !== 'compliant') { + throw new BadRequestException( + `Framework ${framework} must be marked as compliant before uploading a certificate`, + ); + } + } + + private preparePdfPayload(dto: UploadComplianceResourceDto) { + if ( + dto.fileType.toLowerCase() !== 'application/pdf' && + !dto.fileName.toLowerCase().endsWith('.pdf') + ) { + throw new BadRequestException('Only PDF files are supported'); + } + + let fileBuffer: Buffer; + try { + fileBuffer = Buffer.from(dto.fileData, 'base64'); + } catch { + throw new BadRequestException( + 'Invalid file data. Expected base64 string.', + ); + } + + if (!fileBuffer.length) { + throw new BadRequestException('File cannot be empty'); + } + + if (fileBuffer.length > this.MAX_FILE_SIZE_BYTES) { + throw new BadRequestException( + `File exceeds the ${this.MAX_FILE_SIZE_BYTES / (1024 * 1024)}MB limit`, + ); + } + + const sanitizedFileName = dto.fileName.replace(/[^a-zA-Z0-9.-]/g, '_'); + + return { fileBuffer, sanitizedFileName }; + } + + private ensureS3Availability(): void { + if (!s3Client || !APP_AWS_ORG_ASSETS_BUCKET) { + throw new InternalServerErrorException( + 'Organization assets bucket is not configured', + ); + } + } + + private async safeDeleteObject(key: string): Promise { + try { + const deleteCommand = new DeleteObjectCommand({ + Bucket: APP_AWS_ORG_ASSETS_BUCKET, + Key: key, + }); + await s3Client!.send(deleteCommand); + } catch (error) { + this.logger.warn( + `Failed to delete previous compliance resource with key ${key}`, + error instanceof Error ? error.message : error, + ); + } + } } diff --git a/apps/api/src/utils/sse-utils.ts b/apps/api/src/utils/sse-utils.ts new file mode 100644 index 000000000..42c11fe22 --- /dev/null +++ b/apps/api/src/utils/sse-utils.ts @@ -0,0 +1,61 @@ +import type { Response } from 'express'; + +/** + * Escapes special characters in JSON strings using Unicode escapes. + * This prevents potential XSS if JSON is ever interpreted as HTML, + * while keeping the JSON valid. + * + * JSON.stringify handles standard JSON escaping, but doesn't escape + * <, >, & which could be problematic if the response is misinterpreted. + */ +function escapeJsonString(jsonStr: string): string { + return jsonStr + .replace(//g, '\\u003e') + .replace(/&/g, '\\u0026'); +} + +/** + * Creates a safe SSE sender function. + * + * Security measures: + * 1. JSON.stringify handles escaping for JSON context + * 2. Unicode escapes for <, >, & prevent HTML interpretation + * 3. Content-Type: text/event-stream prevents browser HTML rendering + * 4. X-Content-Type-Options: nosniff prevents MIME sniffing + */ +export function createSafeSSESender(res: Response) { + return (data: object) => { + // JSON.stringify provides safe JSON encoding + // Additional unicode escapes for <, >, & as defense-in-depth + const jsonData = escapeJsonString(JSON.stringify(data)); + res.write(`data: ${jsonData}\n\n`); + }; +} + +/** + * Sanitizes an error message for safe inclusion in responses. + * Uses Unicode escapes instead of HTML entities to keep the message + * valid for JSON contexts while preventing XSS. + */ +export function sanitizeErrorMessage(error: unknown): string { + const message = + error instanceof Error ? error.message : 'An unexpected error occurred'; + // Use unicode escapes for safety (same as escapeJsonString but for plain strings) + return message + .replace(//g, '\\u003e') + .replace(/&/g, '\\u0026'); +} + +/** + * Sets up SSE headers on a response object + */ +export function setupSSEHeaders(res: Response): void { + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + (res as Response & { flushHeaders?: () => void }).flushHeaders?.(); +} + diff --git a/apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts b/apps/api/src/vector-store/jobs/delete-all-manual-answers-orchestrator.ts similarity index 92% rename from apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts rename to apps/api/src/vector-store/jobs/delete-all-manual-answers-orchestrator.ts index 40d9d1c94..04a71464a 100644 --- a/apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts +++ b/apps/api/src/vector-store/jobs/delete-all-manual-answers-orchestrator.ts @@ -26,7 +26,7 @@ export const deleteAllManualAnswersOrchestratorTask = task({ try { // Use provided IDs if available, otherwise fetch from DB let manualAnswers: Array<{ id: string }>; - + if (payload.manualAnswerIds && payload.manualAnswerIds.length > 0) { // Use IDs passed directly (avoids race condition with DB deletion) manualAnswers = payload.manualAnswerIds.map((id) => ({ id })); @@ -45,7 +45,7 @@ export const deleteAllManualAnswersOrchestratorTask = task({ id: true, }, }); - + logger.info('Fetched manual answers from DB', { organizationId: payload.organizationId, count: manualAnswers.length, @@ -67,7 +67,10 @@ export const deleteAllManualAnswersOrchestratorTask = task({ metadata.set('deletedCount', 0); metadata.set('failedCount', 0); metadata.set('currentBatch', 0); - metadata.set('totalBatches', Math.ceil(manualAnswers.length / BATCH_SIZE)); + metadata.set( + 'totalBatches', + Math.ceil(manualAnswers.length / BATCH_SIZE), + ); let deletedCount = 0; let failedCount = 0; @@ -78,10 +81,13 @@ export const deleteAllManualAnswersOrchestratorTask = task({ const batchNumber = Math.floor(i / BATCH_SIZE) + 1; const totalBatches = Math.ceil(manualAnswers.length / BATCH_SIZE); - logger.info(`Processing deletion batch ${batchNumber}/${totalBatches}`, { - batchSize: batch.length, - manualAnswerIds: batch.map((ma) => ma.id), - }); + logger.info( + `Processing deletion batch ${batchNumber}/${totalBatches}`, + { + batchSize: batch.length, + manualAnswerIds: batch.map((ma) => ma.id), + }, + ); // Update metadata metadata.set('currentBatch', batchNumber); @@ -94,7 +100,8 @@ export const deleteAllManualAnswersOrchestratorTask = task({ }, })); - const batchHandle = await deleteManualAnswerTask.batchTriggerAndWait(batchItems); + const batchHandle = + await deleteManualAnswerTask.batchTriggerAndWait(batchItems); // Process batch results batchHandle.runs.forEach((run, batchIdx) => { @@ -164,4 +171,3 @@ export const deleteAllManualAnswersOrchestratorTask = task({ } }, }); - diff --git a/apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts b/apps/api/src/vector-store/jobs/delete-knowledge-base-document.ts similarity index 77% rename from apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts rename to apps/api/src/vector-store/jobs/delete-knowledge-base-document.ts index d752af191..53043e5e7 100644 --- a/apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts +++ b/apps/api/src/vector-store/jobs/delete-knowledge-base-document.ts @@ -1,6 +1,6 @@ import { logger, task } from '@trigger.dev/sdk'; -import { findEmbeddingsForSource } from '@/lib/vector/core/find-existing-embeddings'; -import { vectorIndex } from '@/lib/vector/core/client'; +import { findEmbeddingsForSource } from '@/vector-store/lib/core/find-existing-embeddings'; +import { vectorIndex } from '@/vector-store/lib/core/client'; import { db } from '@db'; /** @@ -11,10 +11,7 @@ export const deleteKnowledgeBaseDocumentTask = task({ retry: { maxAttempts: 3, }, - run: async (payload: { - documentId: string; - organizationId: string; - }) => { + run: async (payload: { documentId: string; organizationId: string }) => { logger.info('Deleting Knowledge Base document from vector DB', { documentId: payload.documentId, organizationId: payload.organizationId, @@ -72,7 +69,7 @@ export const deleteKnowledgeBaseDocumentTask = task({ } const idsToDelete = existingEmbeddings.map((e) => e.id); - + if (idsToDelete.length === 0) { logger.info('No embeddings to delete for document', { documentId: payload.documentId, @@ -87,7 +84,7 @@ export const deleteKnowledgeBaseDocumentTask = task({ // Delete all embeddings in batches (Upstash Vector supports batch delete) const batchSize = 100; let deletedCount = 0; - + for (let i = 0; i < idsToDelete.length; i += batchSize) { const batch = idsToDelete.slice(i, i + batchSize); try { @@ -103,7 +100,10 @@ export const deleteKnowledgeBaseDocumentTask = task({ logger.error('Error deleting batch of embeddings', { documentId: payload.documentId, batchSize: batch.length, - error: batchError instanceof Error ? batchError.message : 'Unknown error', + error: + batchError instanceof Error + ? batchError.message + : 'Unknown error', }); // Continue with next batch even if one fails } @@ -128,20 +128,25 @@ export const deleteKnowledgeBaseDocumentTask = task({ // Retry deletion up to 3 times if chunks remain let retryAttempt = 0; const maxRetries = 3; - + while (remainingEmbeddings.length > 0 && retryAttempt < maxRetries) { retryAttempt++; - logger.warn('Some embeddings were not deleted, attempting retry deletion', { - documentId: payload.documentId, - remainingCount: remainingEmbeddings.length, - remainingIds: remainingEmbeddings.map((e) => e.id), - retryAttempt, - maxRetries, - }); - + logger.warn( + 'Some embeddings were not deleted, attempting retry deletion', + { + documentId: payload.documentId, + remainingCount: remainingEmbeddings.length, + remainingIds: remainingEmbeddings.map((e) => e.id), + retryAttempt, + maxRetries, + }, + ); + // Wait before retry to allow propagation - await new Promise((resolve) => setTimeout(resolve, 2000 * retryAttempt)); // Increasing delay - + await new Promise((resolve) => + setTimeout(resolve, 2000 * retryAttempt), + ); // Increasing delay + // Try deleting remaining chunks const remainingIds = remainingEmbeddings.map((e) => e.id); try { @@ -152,7 +157,7 @@ export const deleteKnowledgeBaseDocumentTask = task({ await vectorIndex.delete(batch); deletedCount += batch.length; } - + logger.info('Deleted remaining embeddings in retry attempt', { documentId: payload.documentId, deletedCount: remainingIds.length, @@ -162,10 +167,13 @@ export const deleteKnowledgeBaseDocumentTask = task({ logger.error('Error deleting remaining embeddings in retry attempt', { documentId: payload.documentId, retryAttempt, - error: retryError instanceof Error ? retryError.message : 'Unknown error', + error: + retryError instanceof Error + ? retryError.message + : 'Unknown error', }); } - + // Query again to check if deletion was successful await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for propagation remainingEmbeddings = await findEmbeddingsForSource( @@ -178,11 +186,14 @@ export const deleteKnowledgeBaseDocumentTask = task({ // Final verification - if chunks still remain, try one more aggressive search if (remainingEmbeddings.length > 0) { - logger.warn('Chunks still remain after retries, attempting final aggressive search', { - documentId: payload.documentId, - remainingCount: remainingEmbeddings.length, - remainingIds: remainingEmbeddings.map((e) => e.id), - }); + logger.warn( + 'Chunks still remain after retries, attempting final aggressive search', + { + documentId: payload.documentId, + remainingCount: remainingEmbeddings.length, + remainingIds: remainingEmbeddings.map((e) => e.id), + }, + ); // Wait a bit longer for final attempt await new Promise((resolve) => setTimeout(resolve, 3000)); @@ -208,7 +219,10 @@ export const deleteKnowledgeBaseDocumentTask = task({ } catch (finalError) { logger.error('Error in final deletion attempt', { documentId: payload.documentId, - error: finalError instanceof Error ? finalError.message : 'Unknown error', + error: + finalError instanceof Error + ? finalError.message + : 'Unknown error', }); } @@ -222,26 +236,32 @@ export const deleteKnowledgeBaseDocumentTask = task({ ); if (trulyRemaining.length > 0) { - logger.error('CRITICAL: Some embeddings still remain after all deletion attempts', { - documentId: payload.documentId, - remainingCount: trulyRemaining.length, - remainingIds: trulyRemaining.map((e) => e.id), - remainingChunks: trulyRemaining.map((e) => ({ - id: e.id, - sourceId: e.sourceId, - updatedAt: e.updatedAt, - })), - note: 'These chunks may need manual deletion or there may be a synchronization issue with Upstash Vector', - }); + logger.error( + 'CRITICAL: Some embeddings still remain after all deletion attempts', + { + documentId: payload.documentId, + remainingCount: trulyRemaining.length, + remainingIds: trulyRemaining.map((e) => e.id), + remainingChunks: trulyRemaining.map((e) => ({ + id: e.id, + sourceId: e.sourceId, + updatedAt: e.updatedAt, + })), + note: 'These chunks may need manual deletion or there may be a synchronization issue with Upstash Vector', + }, + ); } } } - logger.info('Successfully deleted Knowledge Base document embeddings from vector DB', { - documentId: payload.documentId, - deletedCount, - totalFound: idsToDelete.length, - }); + logger.info( + 'Successfully deleted Knowledge Base document embeddings from vector DB', + { + documentId: payload.documentId, + deletedCount, + totalFound: idsToDelete.length, + }, + ); return { success: true, @@ -261,4 +281,3 @@ export const deleteKnowledgeBaseDocumentTask = task({ } }, }); - diff --git a/apps/app/src/jobs/tasks/vector/delete-manual-answer.ts b/apps/api/src/vector-store/jobs/delete-manual-answer.ts similarity index 89% rename from apps/app/src/jobs/tasks/vector/delete-manual-answer.ts rename to apps/api/src/vector-store/jobs/delete-manual-answer.ts index 85cc52380..576568c56 100644 --- a/apps/app/src/jobs/tasks/vector/delete-manual-answer.ts +++ b/apps/api/src/vector-store/jobs/delete-manual-answer.ts @@ -1,5 +1,5 @@ import { logger, task } from '@trigger.dev/sdk'; -import { deleteManualAnswerFromVector } from '@/lib/vector/sync/sync-manual-answer'; +import { deleteManualAnswerFromVector } from '@/vector-store/lib/sync/sync-manual-answer'; /** * Task to delete a single manual answer from vector database @@ -10,10 +10,7 @@ export const deleteManualAnswerTask = task({ retry: { maxAttempts: 3, }, - run: async (payload: { - manualAnswerId: string; - organizationId: string; - }) => { + run: async (payload: { manualAnswerId: string; organizationId: string }) => { logger.info('Deleting manual answer from vector DB', { manualAnswerId: payload.manualAnswerId, organizationId: payload.organizationId, @@ -58,4 +55,3 @@ export const deleteManualAnswerTask = task({ } }, }); - diff --git a/apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts b/apps/api/src/vector-store/jobs/helpers/extract-content-from-file.ts similarity index 80% rename from apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts rename to apps/api/src/vector-store/jobs/helpers/extract-content-from-file.ts index 371e0f716..9565b5ebd 100644 --- a/apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts +++ b/apps/api/src/vector-store/jobs/helpers/extract-content-from-file.ts @@ -1,4 +1,4 @@ -import { logger } from '@/utils/logger'; +import { logger } from '../../logger'; import { openai } from '@ai-sdk/openai'; import { generateText } from 'ai'; import * as XLSX from 'xlsx'; @@ -19,7 +19,10 @@ const decodeBasicHtmlEntities = (input: string) => { do { previousValue = decoded; - decoded = decoded.replace(entityPattern, (entity) => htmlEntityMap[entity as keyof typeof htmlEntityMap] ?? entity); + decoded = decoded.replace( + entityPattern, + (entity) => htmlEntityMap[entity as keyof typeof htmlEntityMap] ?? entity, + ); } while (decoded !== previousValue); return decoded; @@ -34,47 +37,55 @@ export async function extractContentFromFile( fileType: string, ): Promise { const fileBuffer = Buffer.from(fileData, 'base64'); - + // Handle Excel files (.xlsx, .xls) if ( fileType === 'application/vnd.ms-excel' || - fileType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || + fileType === + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || fileType === 'application/vnd.ms-excel.sheet.macroEnabled.12' ) { try { const excelStartTime = Date.now(); const fileSizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2); - + logger.info('Processing Excel file', { fileType, fileSizeMB, }); - + const workbook = XLSX.read(fileBuffer, { type: 'buffer' }); - + // Process sheets sequentially const sheets: string[] = []; - + for (const sheetName of workbook.SheetNames) { const worksheet = workbook.Sheets[sheetName]; - const jsonData = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: '' }); - + const jsonData = XLSX.utils.sheet_to_json(worksheet, { + header: 1, + defval: '', + }); + // Convert to readable text format const sheetText = jsonData .map((row: any) => { if (Array.isArray(row)) { - return row.filter((cell) => cell !== null && cell !== undefined && cell !== '').join(' | '); + return row + .filter( + (cell) => cell !== null && cell !== undefined && cell !== '', + ) + .join(' | '); } return String(row); }) .filter((line: string) => line.trim() !== '') .join('\n'); - + if (sheetText.trim()) { sheets.push(`Sheet: ${sheetName}\n${sheetText}`); } } - + const extractionTime = ((Date.now() - excelStartTime) / 1000).toFixed(2); logger.info('Excel file processed', { fileSizeMB, @@ -82,13 +93,15 @@ export async function extractContentFromFile( extractedLength: sheets.join('\n\n').length, extractionTimeSeconds: extractionTime, }); - + return sheets.join('\n\n'); } catch (error) { - throw new Error(`Failed to parse Excel file: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to parse Excel file: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); } } - + // Handle CSV files if (fileType === 'text/csv' || fileType === 'text/comma-separated-values') { try { @@ -97,47 +110,56 @@ export async function extractContentFromFile( const lines = text.split('\n').filter((line) => line.trim() !== ''); return lines.join('\n'); } catch (error) { - throw new Error(`Failed to parse CSV file: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to parse CSV file: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); } } - + // Handle plain text files if (fileType === 'text/plain' || fileType.startsWith('text/')) { try { return fileBuffer.toString('utf-8'); } catch (error) { - throw new Error(`Failed to read text file: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to read text file: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); } } - + // Handle Word documents (.docx) - extract text using mammoth library - if (fileType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { + if ( + fileType === + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) { try { const docxStartTime = Date.now(); const fileSizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2); - + logger.info('Processing DOCX file', { fileType, fileSizeMB, }); - + // Extract text from DOCX using mammoth const result = await mammoth.extractRawText({ buffer: fileBuffer }); const text = result.value; - + // Also extract formatted text (includes formatting information) - const formattedResult = await mammoth.convertToHtml({ buffer: fileBuffer }); - + const formattedResult = await mammoth.convertToHtml({ + buffer: fileBuffer, + }); + // Use formatted HTML if available, otherwise use plain text const extractedText = formattedResult.value || text; - + const extractionTime = ((Date.now() - docxStartTime) / 1000).toFixed(2); logger.info('DOCX file processed', { fileSizeMB, extractedLength: extractedText.length, extractionTimeSeconds: extractionTime, }); - + // Convert HTML to plain text if needed (remove HTML tags) if (formattedResult.value) { // Simple HTML tag removal - keep text content and decode entities safely @@ -146,43 +168,48 @@ export async function extractContentFromFile( ) .replace(/\s+/g, ' ') // Replace multiple spaces with single space .trim(); - + return plainText || text; } - + return text; } catch (error) { logger.error('Failed to parse DOCX file', { fileType, error: error instanceof Error ? error.message : 'Unknown error', }); - throw new Error(`Failed to parse DOCX file: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to parse DOCX file: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); } } - + // Handle legacy Word documents (.doc) - not supported, suggest conversion if (fileType === 'application/msword') { throw new Error( 'Legacy Word documents (.doc) are not supported. Please convert to .docx or PDF format before uploading.', ); } - + // For images and PDFs, use OpenAI vision API const isImage = fileType.startsWith('image/'); const isPdf = fileType === 'application/pdf'; - + if (isImage || isPdf) { const base64Data = fileData; const mimeType = fileType; - const fileSizeMB = (Buffer.from(fileData, 'base64').length / (1024 * 1024)).toFixed(2); - + const fileSizeMB = ( + Buffer.from(fileData, 'base64').length / + (1024 * 1024) + ).toFixed(2); + logger.info('Extracting content from PDF/image using vision API', { fileType: mimeType, fileSizeMB, }); - + const startTime = Date.now(); - + try { const { text } = await generateText({ model: openai('gpt-4o-mini'), // Using gpt-4o-mini for better text extraction @@ -202,14 +229,14 @@ export async function extractContentFromFile( }, ], }); - + const extractionTime = ((Date.now() - startTime) / 1000).toFixed(2); logger.info('Content extracted from PDF/image', { fileType: mimeType, extractedLength: text.length, extractionTimeSeconds: extractionTime, }); - + return text; } catch (error) { const extractionTime = ((Date.now() - startTime) / 1000).toFixed(2); @@ -219,13 +246,14 @@ export async function extractContentFromFile( extractionTimeSeconds: extractionTime, error: error instanceof Error ? error.message : 'Unknown error', }); - throw new Error(`Failed to extract content: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error( + `Failed to extract content: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); } } - + // For other file types that might be binary formats, provide helpful error message throw new Error( `Unsupported file type: ${fileType}. Supported formats: PDF, images (PNG, JPG, etc.), Excel (.xlsx, .xls), CSV, text files (.txt, .md), Word documents (.docx). Legacy Word documents (.doc) should be converted to .docx or PDF.`, ); } - diff --git a/apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts b/apps/api/src/vector-store/jobs/process-knowledge-base-document.ts similarity index 91% rename from apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts rename to apps/api/src/vector-store/jobs/process-knowledge-base-document.ts index 571516fee..2862d7769 100644 --- a/apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts +++ b/apps/api/src/vector-store/jobs/process-knowledge-base-document.ts @@ -1,8 +1,10 @@ import { logger, task } from '@trigger.dev/sdk'; import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { db } from '@db'; -import { batchUpsertEmbeddings } from '@/lib/vector/core/upsert-embedding'; -import { chunkText } from '@/lib/vector/utils/chunk-text'; +import { batchUpsertEmbeddings } from '@/vector-store/lib/core/upsert-embedding'; +import { chunkText } from '@/vector-store/lib/utils/chunk-text'; +import { findEmbeddingsForSource } from '@/vector-store/lib/core/find-existing-embeddings'; +import { vectorIndex } from '@/vector-store/lib/core/client'; import { extractContentFromFile } from './helpers/extract-content-from-file'; /** @@ -36,9 +38,11 @@ async function extractContentFromKnowledgeBaseDocument( fileType: string, ): Promise { const knowledgeBaseBucket = process.env.APP_AWS_KNOWLEDGE_BASE_BUCKET; - + if (!knowledgeBaseBucket) { - throw new Error('Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable in Trigger.dev.'); + throw new Error( + 'Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable in Trigger.dev.', + ); } const s3Client = createS3Client(); @@ -47,13 +51,13 @@ async function extractContentFromKnowledgeBaseDocument( Bucket: knowledgeBaseBucket, Key: s3Key, }); - + const response = await s3Client.send(getCommand); - + if (!response.Body) { throw new Error('Failed to retrieve file from S3'); } - + // Convert stream to buffer const chunks: Uint8Array[] = []; for await (const chunk of response.Body as any) { @@ -61,12 +65,13 @@ async function extractContentFromKnowledgeBaseDocument( } const buffer = Buffer.concat(chunks); const base64Data = buffer.toString('base64'); - + // Use provided fileType or determine from content type - const detectedFileType = response.ContentType || fileType || 'application/octet-stream'; - + const detectedFileType = + response.ContentType || fileType || 'application/octet-stream'; + const content = await extractContentFromFile(base64Data, detectedFileType); - + return content; } @@ -80,10 +85,7 @@ export const processKnowledgeBaseDocumentTask = task({ maxAttempts: 3, }, maxDuration: 1000 * 60 * 30, // 30 minutes for large files - run: async (payload: { - documentId: string; - organizationId: string; - }) => { + run: async (payload: { documentId: string; organizationId: string }) => { logger.info('Processing Knowledge Base document', { documentId: payload.documentId, organizationId: payload.organizationId, @@ -152,7 +154,6 @@ export const processKnowledgeBaseDocumentTask = task({ }); // Delete existing embeddings for this document (if any) - const { findEmbeddingsForSource } = await import('@/lib/vector/core/find-existing-embeddings'); const existingEmbeddings = await findEmbeddingsForSource( document.id, 'knowledge_base_document', @@ -160,7 +161,6 @@ export const processKnowledgeBaseDocumentTask = task({ ); if (existingEmbeddings.length > 0) { - const { vectorIndex } = await import('@/lib/vector/core/client'); if (vectorIndex) { const idsToDelete = existingEmbeddings.map((e) => e.id); try { @@ -268,7 +268,10 @@ export const processKnowledgeBaseDocumentTask = task({ } catch (updateError) { logger.error('Failed to update document status to failed', { documentId: payload.documentId, - error: updateError instanceof Error ? updateError.message : 'Unknown error', + error: + updateError instanceof Error + ? updateError.message + : 'Unknown error', }); } @@ -280,4 +283,3 @@ export const processKnowledgeBaseDocumentTask = task({ } }, }); - diff --git a/apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts b/apps/api/src/vector-store/jobs/process-knowledge-base-documents-orchestrator.ts similarity index 90% rename from apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts rename to apps/api/src/vector-store/jobs/process-knowledge-base-documents-orchestrator.ts index f84395ccf..2e74a47d5 100644 --- a/apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts +++ b/apps/api/src/vector-store/jobs/process-knowledge-base-documents-orchestrator.ts @@ -12,10 +12,7 @@ export const processKnowledgeBaseDocumentsOrchestratorTask = task({ retry: { maxAttempts: 3, }, - run: async (payload: { - documentIds: string[]; - organizationId: string; - }) => { + run: async (payload: { documentIds: string[]; organizationId: string }) => { logger.info('Starting Knowledge Base documents processing orchestrator', { organizationId: payload.organizationId, documentCount: payload.documentIds.length, @@ -36,7 +33,10 @@ export const processKnowledgeBaseDocumentsOrchestratorTask = task({ metadata.set('documentsFailed', 0); metadata.set('documentsRemaining', payload.documentIds.length); metadata.set('currentBatch', 0); - metadata.set('totalBatches', Math.ceil(payload.documentIds.length / BATCH_SIZE)); + metadata.set( + 'totalBatches', + Math.ceil(payload.documentIds.length / BATCH_SIZE), + ); // Initialize individual document statuses - all start as 'pending' payload.documentIds.forEach((documentId, index) => { @@ -77,7 +77,8 @@ export const processKnowledgeBaseDocumentsOrchestratorTask = task({ }, })); - const batchHandle = await processKnowledgeBaseDocumentTask.batchTriggerAndWait(batchItems); + const batchHandle = + await processKnowledgeBaseDocumentTask.batchTriggerAndWait(batchItems); // Process batch results batchHandle.runs.forEach((run, batchIdx) => { @@ -126,8 +127,13 @@ export const processKnowledgeBaseDocumentsOrchestratorTask = task({ }); // Update remaining count - const completed = results.filter((r) => r.success).length + results.filter((r) => !r.success).length; - metadata.set('documentsRemaining', payload.documentIds.length - completed); + const completed = + results.filter((r) => r.success).length + + results.filter((r) => !r.success).length; + metadata.set( + 'documentsRemaining', + payload.documentIds.length - completed, + ); logger.info(`Batch ${batchNumber}/${totalBatches} completed`, { batchSize: batch.length, @@ -157,4 +163,3 @@ export const processKnowledgeBaseDocumentsOrchestratorTask = task({ }; }, }); - diff --git a/apps/api/src/vector-store/lib/README-MANUAL-ANSWERS.md b/apps/api/src/vector-store/lib/README-MANUAL-ANSWERS.md new file mode 100644 index 000000000..c787b5903 --- /dev/null +++ b/apps/api/src/vector-store/lib/README-MANUAL-ANSWERS.md @@ -0,0 +1,110 @@ +# Manual Answers Vector Database Integration + +## Overview + +Manual answers are automatically synced to the Upstash Vector database to improve AI answer generation quality. This document explains how to verify embeddings and troubleshoot sync issues. + +## Embedding ID Format + +When a manual answer is saved, it gets an embedding ID in the format: +``` +manual_answer_{manualAnswerId} +``` + +For example: +- Manual Answer ID: `sqma_abc123xyz` +- Embedding ID: `manual_answer_sqma_abc123xyz` + +## Verifying Embeddings + +### Method 1: Check Embedding ID in Response + +When you save a manual answer, the response includes the `embeddingId`: + +```typescript +const result = await saveManualAnswer.execute({ + question: "What is your data retention policy?", + answer: "We retain data for 7 years as per GDPR requirements.", +}); + +if (result.data?.success) { + console.log('Embedding ID:', result.data.embeddingId); + // Output: "manual_answer_sqma_abc123xyz" +} +``` + +### Method 2: Search in Upstash Vector Dashboard + +1. Go to your Upstash Vector dashboard +2. Use the search/filter functionality +3. Search for the embedding ID: `manual_answer_sqma_abc123xyz` +4. Or filter by metadata: + - `sourceType`: `manual_answer` + - `sourceId`: `sqma_abc123xyz` (the manual answer ID) + - `organizationId`: `org_123` + +## Sync Behavior + +### Synchronous Sync (Single Manual Answer) + +When a user saves a manual answer: +1. Manual answer is saved to the database +2. **Immediately** synced to vector DB (~1-2 seconds) +3. Embedding ID is returned in the response +4. Manual answer is **immediately available** for answer generation + +### Automatic Sync (Before Answer Generation) + +Before generating answers for questionnaires: +1. `syncOrganizationEmbeddings()` is called automatically +2. This ensures all manual answers are up-to-date +3. Manual answers are included in the RAG search + +### Background Sync (Delete All) + +When deleting all manual answers: +1. Orchestrator task is triggered in the background +2. Deletions happen in parallel batches (50 at a time) +3. Progress can be tracked via Trigger.dev dashboard + +## Troubleshooting + +### Embedding Not Found + +If an embedding is not found: + +1. **Check if sync succeeded**: Look at the `embeddingId` field in the save response - if present, sync was successful +2. **Check logs**: Look for errors in the server logs +3. **Manual sync**: The embedding will be synced automatically on the next `syncOrganizationEmbeddings()` call +4. **Check Upstash Vector Dashboard**: Use the dashboard to search for the embedding ID or filter by metadata +5. **Check Upstash Vector**: Verify the vector database is configured correctly + +### Sync Failed + +If sync fails: +- The manual answer is still saved in the database +- It will be synced automatically on the next organization sync +- Check server logs for detailed error messages + +## Testing + +To verify that an embedding was created: + +```typescript +// After saving a manual answer +const saveResult = await saveManualAnswer.execute({...}); + +if (saveResult.data?.embeddingId) { + console.log('Embedding ID:', saveResult.data.embeddingId); + // The embedding ID confirms that sync was successful + // You can verify it exists in the Upstash Vector Dashboard +} +``` + +## Related Files + +- `apps/app/src/lib/vector/sync/sync-manual-answer.ts` - Sync functions +- `apps/app/src/lib/vector/core/find-existing-embeddings.ts` - Functions to find embeddings by source +- `apps/app/src/jobs/tasks/vector/delete-manual-answer.ts` - Single deletion task +- `apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts` - Batch deletion orchestrator + diff --git a/apps/api/src/vector-store/lib/README.md b/apps/api/src/vector-store/lib/README.md new file mode 100644 index 000000000..bddced32a --- /dev/null +++ b/apps/api/src/vector-store/lib/README.md @@ -0,0 +1,115 @@ +# Vector Search Utilities + +This directory contains utilities for semantic search using Upstash Vector and OpenAI embeddings. + +## Structure + +``` +lib/vector/ +├── core/ # Core functionality +│ ├── client.ts # Upstash Vector client initialization +│ ├── generate-embedding.ts # OpenAI embedding generation +│ ├── find-similar.ts # Semantic search function +│ └── upsert-embedding.ts # Embedding storage +├── utils/ # Utility functions +│ ├── chunk-text.ts # Text chunking utility +│ └── extract-policy-text.ts # TipTap JSON to text conversion +├── index.ts # Main exports +└── README.md # This file +``` + +## Setup + +1. **Create Upstash Vector Database** + - Go to [Upstash Console](https://console.upstash.com) + - Create a new Vector Database + - Copy the REST URL and Token + +2. **Add Environment Variables** + Add to your `.env` file: + ``` + UPSTASH_VECTOR_REST_URL=your_vector_rest_url + UPSTASH_VECTOR_REST_TOKEN=your_vector_rest_token + OPENAI_API_KEY=your_openai_api_key + ``` + +3. **Automatic Embedding Creation** + Embeddings are automatically created when parsing vendor questionnaires. + The system checks if embeddings exist for your organization and creates them + automatically if needed (first 10 policies and 10 context entries). + +## Usage + +### Find Similar Content + +```typescript +import { findSimilarContent } from '@/vector-store/lib'; + +const results = await findSimilarContent( + "How do we handle encryption?", + organizationId, +); + +// Returns ALL results above similarity threshold (0.2) +// No artificial limit - all relevant data reaches the LLM +// +// Results contain: +// - id: embedding ID +// - score: similarity score (0-1) +// - content: text content +// - sourceType: 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document' +// - sourceId: ID of the source document +// - policyName: (if sourceType is 'policy') +// - contextQuestion: (if sourceType is 'context') +// - manualAnswerQuestion: (if sourceType is 'manual_answer') +// - documentName: (if sourceType is 'knowledge_base_document') +``` + +### Upsert Embedding + +```typescript +import { upsertEmbedding } from '@/vector-store/lib'; + +await upsertEmbedding( + 'policy_pol123_chunk0', + 'Text content to embed...', + { + organizationId: 'org_123', + sourceType: 'policy', + sourceId: 'pol_123', + content: 'Text content...', + policyName: 'Security Policy', + } +); +``` + +### Utilities + +```typescript +import { chunkText, extractTextFromPolicy } from '@/vector-store/lib'; + +// Chunk text into smaller pieces +const chunks = chunkText(longText, 500, 50); // 500 tokens, 50 overlap + +// Extract text from TipTap JSON policy +const text = extractTextFromPolicy(policy); +``` + +## Files + +### Core (`core/`) +- `client.ts` - Upstash Vector client initialization +- `generate-embedding.ts` - OpenAI embedding generation +- `find-similar.ts` - Semantic search function +- `upsert-embedding.ts` - Embedding storage + +### Utils (`utils/`) +- `chunk-text.ts` - Text chunking utility +- `extract-policy-text.ts` - TipTap JSON to text conversion + +## Next Steps + +After setting up vector search, you can: +1. Use `findSimilarContent()` in your auto-answer functionality +2. Create scheduled jobs to keep embeddings up-to-date +3. Add document hub support for additional context sources diff --git a/apps/api/src/vector-store/lib/core/client.ts b/apps/api/src/vector-store/lib/core/client.ts new file mode 100644 index 000000000..ad76e1044 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/client.ts @@ -0,0 +1,19 @@ +import { Index } from '@upstash/vector'; +import { logger } from '../../logger'; + +const upstashUrl = process.env.UPSTASH_VECTOR_REST_URL; +const upstashToken = process.env.UPSTASH_VECTOR_REST_TOKEN; + +if (!upstashUrl || !upstashToken) { + logger.warn( + 'Upstash Vector credentials not configured. Vector search functionality will be disabled.', + ); +} + +export const vectorIndex: Index | null = + upstashUrl && upstashToken + ? new Index({ + url: upstashUrl, + token: upstashToken, + }) + : null; diff --git a/apps/api/src/vector-store/lib/core/count-embeddings.ts b/apps/api/src/vector-store/lib/core/count-embeddings.ts new file mode 100644 index 000000000..84359bf51 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/count-embeddings.ts @@ -0,0 +1,141 @@ +import { vectorIndex } from './client'; +import { generateEmbedding } from './generate-embedding'; +import { logger } from '../../logger'; + +/** + * Counts embeddings for a specific organization and source type + * Useful for debugging and verification + */ +export async function countEmbeddings( + organizationId: string, + sourceType?: 'policy' | 'context' | 'manual_answer', +): Promise<{ + total: number; + bySourceType: Record; + error?: string; +}> { + if (!vectorIndex) { + return { + total: 0, + bySourceType: {}, + error: 'Vector DB not configured', + }; + } + + try { + // Use organizationId as query to find all embeddings + const queryEmbedding = await generateEmbedding(organizationId); + + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 100, // Max allowed by Upstash Vector + includeMetadata: true, + }); + + // Filter by organizationId + const orgResults = results.filter((result) => { + const metadata = result.metadata as any; + return metadata?.organizationId === organizationId; + }); + + // Count by sourceType + const bySourceType: Record = {}; + let total = 0; + + for (const result of orgResults) { + const metadata = result.metadata as any; + const st = metadata?.sourceType || 'unknown'; + + if (!sourceType || st === sourceType) { + bySourceType[st] = (bySourceType[st] || 0) + 1; + total++; + } + } + + logger.info('Counted embeddings', { + organizationId, + sourceType: sourceType || 'all', + total, + bySourceType, + }); + + return { + total, + bySourceType, + }; + } catch (error) { + logger.error('Failed to count embeddings', { + organizationId, + sourceType, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + total: 0, + bySourceType: {}, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} + +/** + * Lists all manual answer embeddings for an organization + * Useful for debugging + */ +export async function listManualAnswerEmbeddings( + organizationId: string, +): Promise< + Array<{ + id: string; + sourceId: string; + content: string; + updatedAt?: string; + }> +> { + if (!vectorIndex) { + return []; + } + + try { + // Use organizationId as query + const queryEmbedding = await generateEmbedding(organizationId); + + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 100, + includeMetadata: true, + }); + + // Filter for manual_answer type + const manualAnswerEmbeddings = results + .filter((result) => { + const metadata = result.metadata as any; + return ( + metadata?.organizationId === organizationId && + metadata?.sourceType === 'manual_answer' + ); + }) + .map((result) => { + const metadata = result.metadata as any; + return { + id: String(result.id), + sourceId: metadata?.sourceId || '', + content: metadata?.content || '', + updatedAt: metadata?.updatedAt, + }; + }); + + logger.info('Listed manual answer embeddings', { + organizationId, + count: manualAnswerEmbeddings.length, + ids: manualAnswerEmbeddings.map((e) => e.id), + }); + + return manualAnswerEmbeddings; + } catch (error) { + logger.error('Failed to list manual answer embeddings', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return []; + } +} diff --git a/apps/api/src/vector-store/lib/core/delete-embeddings.ts b/apps/api/src/vector-store/lib/core/delete-embeddings.ts new file mode 100644 index 000000000..f8d9f89c8 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/delete-embeddings.ts @@ -0,0 +1,118 @@ +import { vectorIndex } from './client'; +import { generateEmbedding } from './generate-embedding'; +import { logger } from '../../logger'; + +/** + * Deletes all embeddings for an organization from the vector database + * Uses search to find all embeddings, filters by organizationId in metadata, then deletes them + */ +export async function deleteOrganizationEmbeddings( + organizationId: string, +): Promise { + if (!vectorIndex) { + logger.warn('Upstash Vector is not configured, skipping deletion'); + return; + } + + if (!organizationId || organizationId.trim().length === 0) { + logger.warn('Invalid organizationId provided for deletion'); + return; + } + + try { + const allIds: string[] = []; + + // Use multiple search queries to find all types of embeddings + // Since Upstash Vector doesn't support metadata filtering in query, + // we use broad searches and filter results + const searchQueries = [ + 'policy security compliance', + 'context question answer', + organizationId, + ]; + + logger.info('Searching for embeddings to delete', { organizationId }); + + for (const query of searchQueries) { + try { + const queryEmbedding = await generateEmbedding(query); + + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 1000, // Max allowed by Upstash Vector + includeMetadata: true, + }); + + // Filter by organizationId in metadata + const orgResults = results + .filter((result) => { + const metadata = result.metadata as any; + return metadata?.organizationId === organizationId; + }) + .map((result) => String(result.id)); + + allIds.push(...orgResults); + + logger.info('Found embeddings in search query', { + query, + found: orgResults.length, + }); + } catch (error) { + logger.warn('Failed to search embeddings', { + query, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + + // Remove duplicates + const uniqueIds = [...new Set(allIds)]; + + logger.info('Found embeddings to delete', { + organizationId, + count: uniqueIds.length, + }); + + if (uniqueIds.length === 0) { + logger.info('No embeddings found to delete', { organizationId }); + return; + } + + // Delete in batches (Upstash Vector supports batch delete) + const batchSize = 100; + let deletedCount = 0; + + for (let i = 0; i < uniqueIds.length; i += batchSize) { + const batch = uniqueIds.slice(i, i + batchSize); + + try { + await vectorIndex.delete(batch); + deletedCount += batch.length; + + logger.info('Deleted batch of embeddings', { + batchSize: batch.length, + totalDeleted: deletedCount, + remaining: uniqueIds.length - deletedCount, + }); + } catch (error) { + logger.warn('Failed to delete batch', { + batchSize: batch.length, + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Continue with other batches + } + } + + logger.info('Successfully deleted organization embeddings', { + organizationId, + deletedCount, + totalFound: uniqueIds.length, + }); + } catch (error) { + logger.error('Failed to delete organization embeddings', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw error; + } +} diff --git a/apps/api/src/vector-store/lib/core/find-existing-embeddings.ts b/apps/api/src/vector-store/lib/core/find-existing-embeddings.ts new file mode 100644 index 000000000..705c689bc --- /dev/null +++ b/apps/api/src/vector-store/lib/core/find-existing-embeddings.ts @@ -0,0 +1,203 @@ +import { vectorIndex } from './client'; +import { generateEmbedding } from './generate-embedding'; +import { logger } from '../../logger'; +import { + type ExistingEmbedding, + type SourceType, + type QueryFilter, + executeVectorQuery, + addToResultsMap, + fetchChunkContent, + GENERIC_DOCUMENT_QUERIES, + filterAndMapResults, +} from './query-helpers'; + +// Re-export types for backward compatibility +export type { ExistingEmbedding, SourceType }; + +/** + * Finds existing embeddings for a specific policy, context, manual answer, or knowledge base document + * Uses multiple query strategies to ensure we find ALL chunks + */ +export async function findEmbeddingsForSource( + sourceId: string, + sourceType: SourceType, + organizationId: string, + documentName?: string, +): Promise { + if (!vectorIndex || !sourceId || !organizationId) { + return []; + } + + const filter: QueryFilter = { organizationId, sourceType, sourceId }; + const allResults = new Map(); + + try { + // Strategy 1-3: Basic queries (orgId, sourceId, combined) + await runBasicQueryStrategies(filter, allResults); + + // Strategy 4: Query with documentName (for knowledge_base_document only) + if (sourceType === 'knowledge_base_document' && documentName) { + const results = await executeVectorQuery(documentName, filter, 'documentName'); + addToResultsMap(allResults, results); + } + + // Strategy 5: Query with content from already-found chunks + if (sourceType === 'knowledge_base_document' && allResults.size > 0) { + await runChunkContentQueryStrategy(filter, allResults); + } + + // Strategy 6: Query with generic terms (for knowledge_base_document) + if (sourceType === 'knowledge_base_document') { + await runGenericQueryStrategy(filter, allResults); + } + + const matchingEmbeddings = Array.from(allResults.values()); + + logger.info('Found embeddings for source', { + sourceId, + sourceType, + organizationId, + count: matchingEmbeddings.length, + }); + + return matchingEmbeddings; + } catch (error) { + logger.warn('Failed to find embeddings for source', { + sourceId, + sourceType, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return []; + } +} + +/** + * Run basic query strategies (organizationId, sourceId, combined) + */ +async function runBasicQueryStrategies( + filter: QueryFilter, + resultsMap: Map, +): Promise { + const queries = [ + { text: filter.organizationId, strategyName: 'organizationId' }, + { text: filter.sourceId, strategyName: 'sourceId' }, + { text: `${filter.organizationId} ${filter.sourceId}`, strategyName: 'combined' }, + ]; + + for (const { text, strategyName } of queries) { + const results = await executeVectorQuery(text, filter, strategyName); + addToResultsMap(resultsMap, results); + } +} + +/** + * Query with content from already-found chunks to find more related chunks + */ +async function runChunkContentQueryStrategy( + filter: QueryFilter, + resultsMap: Map, +): Promise { + const foundChunkIds = Array.from(resultsMap.keys()).slice(0, 3); + + for (const chunkId of foundChunkIds) { + const chunkData = await fetchChunkContent(chunkId); + if (!chunkData) continue; + + // Query with chunk content + if (chunkData.content && chunkData.content.length > 50) { + const contentQuery = chunkData.content.substring(0, 200); + const results = await executeVectorQuery(contentQuery, filter, 'chunkContent'); + addToResultsMap(resultsMap, results); + } + + // Query with filename from chunk metadata + if (chunkData.documentName && chunkData.documentName.length > 0) { + const results = await executeVectorQuery(chunkData.documentName, filter, 'chunkFilename'); + addToResultsMap(resultsMap, results); + } + } +} + +/** + * Query with generic terms that are likely to match document content + */ +async function runGenericQueryStrategy( + filter: QueryFilter, + resultsMap: Map, +): Promise { + for (const genericQuery of GENERIC_DOCUMENT_QUERIES) { + const results = await executeVectorQuery(genericQuery, filter, 'generic'); + addToResultsMap(resultsMap, results); + } +} + +/** + * Finds all existing embeddings for an organization (for orphaned detection) + * Uses pagination approach to respect Upstash Vector 1000 limit + */ +export async function findAllOrganizationEmbeddings( + organizationId: string, +): Promise> { + if (!vectorIndex) { + logger.warn('Upstash Vector is not configured, returning empty map'); + return new Map(); + } + + if (!organizationId || organizationId.trim().length === 0) { + return new Map(); + } + + try { + const queryEmbedding = await generateEmbedding(organizationId); + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 100, + includeMetadata: true, + }); + + // Filter by organizationId and valid source types + const validSourceTypes = ['policy', 'context', 'manual_answer', 'knowledge_base_document']; + const orgResults = results + .filter((result) => { + const metadata = result.metadata as Record | undefined; + return ( + metadata?.organizationId === organizationId && + validSourceTypes.includes(metadata?.sourceType as string) + ); + }) + .map((result) => { + const metadata = result.metadata as Record; + return { + id: String(result.id), + sourceId: (metadata?.sourceId as string) || '', + sourceType: metadata?.sourceType as SourceType, + updatedAt: metadata?.updatedAt as string | undefined, + }; + }); + + // Group by sourceId + const groupedBySourceId = new Map(); + + for (const embedding of orgResults) { + const existing = groupedBySourceId.get(embedding.sourceId) || []; + existing.push(embedding); + groupedBySourceId.set(embedding.sourceId, existing); + } + + logger.info('Found existing embeddings for organization', { + organizationId, + totalEmbeddings: orgResults.length, + uniqueSources: groupedBySourceId.size, + }); + + return groupedBySourceId; + } catch (error) { + logger.error('Failed to find existing embeddings', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return new Map(); + } +} diff --git a/apps/api/src/vector-store/lib/core/find-similar.ts b/apps/api/src/vector-store/lib/core/find-similar.ts new file mode 100644 index 000000000..6d8c889a2 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/find-similar.ts @@ -0,0 +1,228 @@ +import { vectorIndex } from './client'; +import { generateEmbedding, batchGenerateEmbeddings } from './generate-embedding'; +import { logger } from '../../logger'; + +export interface SimilarContentResult { + id: string; + score: number; + content: string; + sourceType: + | 'policy' + | 'context' + | 'document_hub' + | 'attachment' + | 'manual_answer' + | 'knowledge_base_document'; + sourceId: string; + policyName?: string; + contextQuestion?: string; + documentName?: string; + manualAnswerQuestion?: string; +} + +// Minimum similarity threshold - results below this are considered noise +// Set to 0.2 for maximum recall while filtering obvious noise +const MIN_SIMILARITY_SCORE = 0.2; + +// Maximum results to fetch from Upstash Vector (their limit is 1000, but 100 is practical) +const MAX_TOP_K = 100; + +/** + * Finds similar content using semantic search in Upstash Vector + * Optimized for RAG (Retrieval-Augmented Generation) answer generation + * + * Returns ALL results above the similarity threshold - no artificial limit. + * This ensures the LLM receives all potentially relevant organizational data. + * + * @param question - The question or query text to search for + * @param organizationId - Filter results to this organization only + * @returns Array of similar content results sorted by relevance score (highest first) + */ +export async function findSimilarContent( + question: string, + organizationId: string, +): Promise { + if (!vectorIndex) { + logger.warn('Upstash Vector is not configured, returning empty results'); + return []; + } + + if (!question || question.trim().length === 0) { + return []; + } + + try { + // Generate embedding for the question + const queryEmbedding = await generateEmbedding(question); + + // Search in Upstash Vector with server-side organization filtering + // Fetch maximum results, then filter by score threshold + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: MAX_TOP_K, + includeMetadata: true, + filter: `organizationId = "${organizationId}"`, + }); + + // Filter by minimum similarity score only - no artificial limit + // All relevant organizational data should reach the LLM + const filteredResults: SimilarContentResult[] = results + .filter((result) => result.score >= MIN_SIMILARITY_SCORE) + .map((result): SimilarContentResult => { + const metadata = result.metadata as any; + return { + id: String(result.id), + score: result.score, + content: metadata?.content || '', + sourceType: (metadata?.sourceType || + 'policy') as SimilarContentResult['sourceType'], + sourceId: metadata?.sourceId || '', + policyName: metadata?.policyName, + contextQuestion: metadata?.contextQuestion, + documentName: metadata?.documentName, + manualAnswerQuestion: metadata?.manualAnswerQuestion, + }; + }); + + logger.info('Vector search completed', { + question: question.substring(0, 100), + organizationId, + topK: MAX_TOP_K, + totalResults: results.length, + filteredResults: filteredResults.length, + scoreRange: filteredResults.length > 0 + ? { min: filteredResults[filteredResults.length - 1]?.score, max: filteredResults[0]?.score } + : null, + }); + + return filteredResults; + } catch (error) { + logger.error('Failed to find similar content', { + question: question.substring(0, 100), + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw error; + } +} + +/** + * Batch version of findSimilarContent - processes multiple questions efficiently + * Uses batch embedding generation (1 API call instead of N) for significant speedup + * + * For 116 questions: + * - Regular: 116 embedding API calls (~8-12 seconds) + * - Batch: 1 embedding API call (~1-2 seconds) + * + * @param questions - Array of questions to search for + * @param organizationId - Filter results to this organization only + * @returns Array of results arrays, one per question (same order as input) + */ +export async function findSimilarContentBatch( + questions: string[], + organizationId: string, +): Promise { + if (!vectorIndex) { + logger.warn('Upstash Vector is not configured, returning empty results'); + return questions.map(() => []); + } + + if (questions.length === 0) { + return []; + } + + const startTime = Date.now(); + + try { + // Step 1: Generate ALL embeddings in one batch API call (major time savings) + logger.info('Generating batch embeddings', { + questionCount: questions.length, + organizationId, + }); + + const embeddingStartTime = Date.now(); + const embeddings = await batchGenerateEmbeddings(questions); + const embeddingTime = Date.now() - embeddingStartTime; + + logger.info('Batch embeddings generated', { + questionCount: questions.length, + embeddingTimeMs: embeddingTime, + }); + + // Step 2: Query Upstash Vector in parallel for all questions + const queryStartTime = Date.now(); + const queryResults = await Promise.all( + embeddings.map(async (embedding, index) => { + // Skip empty embeddings (from empty questions) + if (!embedding || embedding.length === 0) { + return { index, results: [] }; + } + + try { + const results = await vectorIndex!.query({ + vector: embedding, + topK: MAX_TOP_K, + includeMetadata: true, + filter: `organizationId = "${organizationId}"`, + }); + return { index, results }; + } catch (error) { + logger.warn('Query failed for question', { + questionIndex: index, + question: questions[index]?.substring(0, 50), + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { index, results: [] }; + } + }), + ); + const queryTime = Date.now() - queryStartTime; + + // Step 3: Filter and map results for each question + const allFilteredResults: SimilarContentResult[][] = questions.map(() => []); + + for (const { index, results } of queryResults) { + const filtered = results + .filter((result) => result.score >= MIN_SIMILARITY_SCORE) + .map((result): SimilarContentResult => { + const metadata = result.metadata as any; + return { + id: String(result.id), + score: result.score, + content: metadata?.content || '', + sourceType: (metadata?.sourceType || + 'policy') as SimilarContentResult['sourceType'], + sourceId: metadata?.sourceId || '', + policyName: metadata?.policyName, + contextQuestion: metadata?.contextQuestion, + documentName: metadata?.documentName, + manualAnswerQuestion: metadata?.manualAnswerQuestion, + }; + }); + + allFilteredResults[index] = filtered; + } + + const totalTime = Date.now() - startTime; + + logger.info('Batch vector search completed', { + questionCount: questions.length, + organizationId, + embeddingTimeMs: embeddingTime, + queryTimeMs: queryTime, + totalTimeMs: totalTime, + avgResultsPerQuestion: + allFilteredResults.reduce((sum, r) => sum + r.length, 0) / + questions.length, + }); + + return allFilteredResults; + } catch (error) { + logger.error('Failed to find similar content batch', { + questionCount: questions.length, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw error; + } +} diff --git a/apps/api/src/vector-store/lib/core/generate-embedding.ts b/apps/api/src/vector-store/lib/core/generate-embedding.ts new file mode 100644 index 000000000..4186571f3 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/generate-embedding.ts @@ -0,0 +1,76 @@ +import { openai } from '@ai-sdk/openai'; +import { embed, embedMany } from 'ai'; + +/** + * Generates an embedding vector for the given text using OpenAI's embedding model + * @param text - The text to generate an embedding for + * @returns An array of numbers representing the embedding vector + */ +export async function generateEmbedding(text: string): Promise { + if (!process.env.OPENAI_API_KEY) { + throw new Error('OPENAI_API_KEY is not configured'); + } + + try { + const { embedding } = await embed({ + model: openai.embedding('text-embedding-3-small'), + value: text, + }); + + return embedding; + } catch (error) { + throw new Error( + `Failed to generate embedding: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } +} + +/** + * Generates embedding vectors for multiple texts in a single batch API call + * Much faster than calling generateEmbedding() multiple times + * + * @param texts - Array of texts to generate embeddings for + * @returns Array of embedding vectors in the same order as input texts + */ +export async function batchGenerateEmbeddings( + texts: string[], +): Promise { + if (!process.env.OPENAI_API_KEY) { + throw new Error('OPENAI_API_KEY is not configured'); + } + + if (texts.length === 0) { + return []; + } + + // Filter out empty texts and track their indices + const validTexts: { text: string; originalIndex: number }[] = []; + texts.forEach((text, index) => { + if (text && text.trim().length > 0) { + validTexts.push({ text: text.trim(), originalIndex: index }); + } + }); + + if (validTexts.length === 0) { + return texts.map(() => []); + } + + try { + const { embeddings } = await embedMany({ + model: openai.embedding('text-embedding-3-small'), + values: validTexts.map((v) => v.text), + }); + + // Map embeddings back to original indices, filling empty arrays for skipped texts + const result: number[][] = texts.map(() => []); + validTexts.forEach((item, idx) => { + result[item.originalIndex] = embeddings[idx]; + }); + + return result; + } catch (error) { + throw new Error( + `Failed to generate batch embeddings: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } +} diff --git a/apps/api/src/vector-store/lib/core/query-helpers.ts b/apps/api/src/vector-store/lib/core/query-helpers.ts new file mode 100644 index 000000000..afdc1a9a9 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/query-helpers.ts @@ -0,0 +1,152 @@ +import { vectorIndex } from './client'; +import { generateEmbedding } from './generate-embedding'; +import { logger } from '../../logger'; + +export type SourceType = + | 'policy' + | 'context' + | 'manual_answer' + | 'knowledge_base_document'; + +export interface ExistingEmbedding { + id: string; + sourceId: string; + sourceType: SourceType; + updatedAt?: string; +} + +export interface QueryFilter { + organizationId: string; + sourceType: SourceType; + sourceId: string; +} + +/** + * Execute a vector query and filter results by metadata + */ +export async function executeVectorQuery( + queryText: string, + filter: QueryFilter, + strategyName: string, +): Promise { + if (!vectorIndex) { + return []; + } + + try { + const queryEmbedding = await generateEmbedding(queryText); + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 100, + includeMetadata: true, + }); + + return filterAndMapResults(results, filter); + } catch (error) { + logger.warn(`Error in ${strategyName} query strategy`, { + sourceId: filter.sourceId, + sourceType: filter.sourceType, + organizationId: filter.organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return []; + } +} + +/** + * Filter vector query results by metadata and map to ExistingEmbedding + */ +export function filterAndMapResults( + results: Array<{ id: string | number; metadata?: unknown }>, + filter: QueryFilter, +): ExistingEmbedding[] { + const filtered: ExistingEmbedding[] = []; + + for (const result of results) { + const metadata = result.metadata as Record | undefined; + if ( + metadata?.organizationId === filter.organizationId && + metadata?.sourceType === filter.sourceType && + metadata?.sourceId === filter.sourceId + ) { + filtered.push({ + id: String(result.id), + sourceId: (metadata?.sourceId as string) || '', + sourceType: metadata?.sourceType as SourceType, + updatedAt: metadata?.updatedAt as string | undefined, + }); + } + } + + return filtered; +} + +/** + * Add embeddings to a Map, avoiding duplicates + */ +export function addToResultsMap( + resultsMap: Map, + embeddings: ExistingEmbedding[], +): void { + for (const embedding of embeddings) { + if (!resultsMap.has(embedding.id)) { + resultsMap.set(embedding.id, embedding); + } + } +} + +/** + * Execute multiple vector queries in sequence and collect unique results + */ +export async function executeMultipleQueries( + queries: Array<{ text: string; strategyName: string }>, + filter: QueryFilter, +): Promise> { + const allResults = new Map(); + + for (const { text, strategyName } of queries) { + const results = await executeVectorQuery(text, filter, strategyName); + addToResultsMap(allResults, results); + } + + return allResults; +} + +/** + * Fetch chunk content by ID from vector index + */ +export async function fetchChunkContent( + chunkId: string, +): Promise<{ content?: string; documentName?: string } | null> { + if (!vectorIndex) { + return null; + } + + try { + const chunkResult = await vectorIndex.fetch([chunkId]); + if (chunkResult && chunkResult.length > 0 && chunkResult[0]) { + const metadata = chunkResult[0].metadata as Record; + return { + content: metadata?.content as string | undefined, + documentName: metadata?.documentName as string | undefined, + }; + } + return null; + } catch (error) { + logger.warn('Error fetching chunk content', { + chunkId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return null; + } +} + +/** + * Generic queries for knowledge base documents + */ +export const GENERIC_DOCUMENT_QUERIES = [ + 'document information content', + 'knowledge base document', + 'file content text', +]; + diff --git a/apps/api/src/vector-store/lib/core/upsert-embedding.ts b/apps/api/src/vector-store/lib/core/upsert-embedding.ts new file mode 100644 index 000000000..ab32fe871 --- /dev/null +++ b/apps/api/src/vector-store/lib/core/upsert-embedding.ts @@ -0,0 +1,203 @@ +import { vectorIndex } from './client'; +import { generateEmbedding } from './generate-embedding'; +import { logger } from '../../logger'; + +export type SourceType = + | 'policy' + | 'context' + | 'document_hub' + | 'attachment' + | 'manual_answer' + | 'knowledge_base_document'; + +export interface EmbeddingMetadata { + organizationId: string; + sourceType: SourceType; + sourceId: string; + content: string; + policyName?: string; + contextQuestion?: string; + documentName?: string; + manualAnswerQuestion?: string; + updatedAt?: string; // ISO timestamp for incremental sync comparison +} + +/** + * Upserts an embedding into Upstash Vector + * @param id - Unique identifier for this embedding (e.g., "policy_pol123_chunk0") + * @param text - The text content to embed + * @param metadata - Metadata associated with this embedding + */ +export async function upsertEmbedding( + id: string, + text: string, + metadata: EmbeddingMetadata, +): Promise { + if (!vectorIndex) { + const errorMsg = + 'Upstash Vector is not configured - check UPSTASH_VECTOR_REST_URL and UPSTASH_VECTOR_REST_TOKEN'; + logger.error(errorMsg, { + id, + sourceType: metadata.sourceType, + hasUrl: !!process.env.UPSTASH_VECTOR_REST_URL, + hasToken: !!process.env.UPSTASH_VECTOR_REST_TOKEN, + }); + throw new Error(errorMsg); + } + + if (!text || text.trim().length === 0) { + logger.warn('Skipping empty text for embedding', { + id, + sourceType: metadata.sourceType, + }); + return; + } + + try { + // Generate embedding + const embedding = await generateEmbedding(text); + + // Prepare metadata + const vectorMetadata = { + organizationId: metadata.organizationId, + sourceType: metadata.sourceType, + sourceId: metadata.sourceId, + content: text.substring(0, 1000), // Store first 1000 chars for reference + ...(metadata.policyName && { policyName: metadata.policyName }), + ...(metadata.contextQuestion && { + contextQuestion: metadata.contextQuestion, + }), + ...(metadata.manualAnswerQuestion && { + manualAnswerQuestion: metadata.manualAnswerQuestion, + }), + ...(metadata.documentName && { documentName: metadata.documentName }), + ...(metadata.updatedAt && { updatedAt: metadata.updatedAt }), + }; + + // Log detailed info for manual_answer type (for debugging) + if (metadata.sourceType === 'manual_answer') { + logger.info('Upserting manual answer embedding', { + id, + embeddingId: id, + vectorLength: embedding.length, + vectorPreview: embedding.slice(0, 5).map((v) => v.toFixed(6)), // First 5 dimensions + vectorStats: { + min: Math.min(...embedding), + max: Math.max(...embedding), + mean: embedding.reduce((a, b) => a + b, 0) / embedding.length, + }, + metadata: vectorMetadata, + textPreview: text.substring(0, 200), + }); + } + + // Upsert into Upstash Vector + const upsertResult = await vectorIndex.upsert({ + id, + vector: embedding, + metadata: vectorMetadata, + }); + + // Log success for manual_answer type with upsert result + if (metadata.sourceType === 'manual_answer') { + logger.info('✅ Successfully upserted manual answer embedding', { + id, + embeddingId: id, + organizationId: metadata.organizationId, + sourceId: metadata.sourceId, + upsertResult: upsertResult ? 'success' : 'unknown', + vectorIndexConfigured: !!vectorIndex, + }); + } + } catch (error) { + logger.error('Failed to upsert embedding', { + id, + sourceType: metadata.sourceType, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw error; + } +} + +/** + * Batch upsert embeddings: generates embeddings in parallel, then upserts in parallel + * Much faster than sequential upsertEmbedding calls + * @param items - Array of items to upsert + */ +export async function batchUpsertEmbeddings( + items: Array<{ + id: string; + text: string; + metadata: EmbeddingMetadata; + }>, +): Promise { + if (!vectorIndex) { + throw new Error('Upstash Vector is not configured'); + } + + if (items.length === 0) { + return; + } + + // Filter out empty texts + const validItems = items.filter( + (item) => item.text && item.text.trim().length > 0, + ); + + if (validItems.length === 0) { + return; + } + + try { + // Step 1: Generate all embeddings in parallel (much faster) + const embeddings = await Promise.all( + validItems.map((item) => generateEmbedding(item.text)), + ); + + // Step 2: Upsert all embeddings in parallel + // Check vectorIndex before using it (TypeScript safety) + if (!vectorIndex) { + throw new Error('Upstash Vector is not configured'); + } + + // Store reference to avoid null check issues in map + const index = vectorIndex; + + await Promise.all( + validItems.map((item, idx) => { + const embedding = embeddings[idx]; + return index.upsert({ + id: item.id, + vector: embedding, + metadata: { + organizationId: item.metadata.organizationId, + sourceType: item.metadata.sourceType, + sourceId: item.metadata.sourceId, + content: item.text.substring(0, 1000), // Store first 1000 chars for reference + ...(item.metadata.policyName && { + policyName: item.metadata.policyName, + }), + ...(item.metadata.contextQuestion && { + contextQuestion: item.metadata.contextQuestion, + }), + ...(item.metadata.manualAnswerQuestion && { + manualAnswerQuestion: item.metadata.manualAnswerQuestion, + }), + ...(item.metadata.documentName && { + documentName: item.metadata.documentName, + }), + ...(item.metadata.updatedAt && { + updatedAt: item.metadata.updatedAt, + }), + }, + }); + }), + ); + } catch (error) { + logger.error('Failed to batch upsert embeddings', { + itemCount: validItems.length, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw error; + } +} diff --git a/apps/api/src/vector-store/lib/index.ts b/apps/api/src/vector-store/lib/index.ts new file mode 100644 index 000000000..1f0243d20 --- /dev/null +++ b/apps/api/src/vector-store/lib/index.ts @@ -0,0 +1,35 @@ +// Core functionality +export { vectorIndex } from './core/client'; +export { generateEmbedding, batchGenerateEmbeddings } from './core/generate-embedding'; +export { + findSimilarContent, + findSimilarContentBatch, + type SimilarContentResult, +} from './core/find-similar'; +export { + upsertEmbedding, + batchUpsertEmbeddings, + type EmbeddingMetadata, + type SourceType, +} from './core/upsert-embedding'; +export { deleteOrganizationEmbeddings } from './core/delete-embeddings'; +export { + findEmbeddingsForSource, + findAllOrganizationEmbeddings, +} from './core/find-existing-embeddings'; +export type { ExistingEmbedding } from './core/find-existing-embeddings'; + +// Sync functionality +export { syncOrganizationEmbeddings } from './sync/sync-organization'; +export { + syncManualAnswerToVector, + deleteManualAnswerFromVector, +} from './sync/sync-manual-answer'; + +// Utilities +export { + countEmbeddings, + listManualAnswerEmbeddings, +} from './core/count-embeddings'; +export { chunkText } from './utils/chunk-text'; +export { extractTextFromPolicy } from './utils/extract-policy-text'; diff --git a/apps/api/src/vector-store/lib/sync/sync-context.ts b/apps/api/src/vector-store/lib/sync/sync-context.ts new file mode 100644 index 000000000..7adcc462f --- /dev/null +++ b/apps/api/src/vector-store/lib/sync/sync-context.ts @@ -0,0 +1,133 @@ +import { db } from '@db'; +import { logger } from '../../logger'; +import type { ExistingEmbedding } from '../core/find-existing-embeddings'; +import { + needsUpdate, + deleteOldEmbeddings, + createChunkItems, + upsertChunks, + initSyncStats, + type SyncStats, +} from './sync-utils'; + +const CONTEXT_BATCH_SIZE = 100; + +interface ContextData { + id: string; + question: string; + answer: string; + organizationId: string; + updatedAt: Date; +} + +/** + * Fetch all context entries for an organization + */ +export async function fetchContextEntries(organizationId: string): Promise { + return db.context.findMany({ + where: { organizationId }, + select: { + id: true, + question: true, + answer: true, + organizationId: true, + updatedAt: true, + }, + }); +} + +/** + * Sync a single context entry's embeddings + */ +async function syncSingleContext( + context: ContextData, + existingEmbeddings: ExistingEmbedding[], + organizationId: string, +): Promise<'created' | 'updated' | 'skipped'> { + const contextUpdatedAt = context.updatedAt.toISOString(); + + // Check if context needs update + if (!needsUpdate(existingEmbeddings, contextUpdatedAt)) { + return 'skipped'; + } + + // Delete old embeddings if they exist + await deleteOldEmbeddings(existingEmbeddings, { contextId: context.id }); + + // Create new embeddings + const contextText = `Question: ${context.question}\n\nAnswer: ${context.answer}`; + + if (!contextText || contextText.trim().length === 0) { + return 'skipped'; + } + + // Use larger chunk size for context entries + const chunkItems = createChunkItems( + contextText, + context.id, + 'context', + organizationId, + contextUpdatedAt, + 'context', + { contextQuestion: context.question }, + 8000, // Larger chunk size for context + 50, + ); + + if (chunkItems.length === 0) { + return 'skipped'; + } + + await upsertChunks(chunkItems); + + return existingEmbeddings.length === 0 ? 'created' : 'updated'; +} + +/** + * Sync all context entries for an organization + */ +export async function syncContextEntries( + organizationId: string, + existingEmbeddingsMap: Map, +): Promise { + const contextEntries = await fetchContextEntries(organizationId); + + logger.info('Found context entries to sync', { + organizationId, + count: contextEntries.length, + }); + + const stats = initSyncStats(contextEntries.length); + + // Process context entries in parallel batches + for (let i = 0; i < contextEntries.length; i += CONTEXT_BATCH_SIZE) { + const batch = contextEntries.slice(i, i + CONTEXT_BATCH_SIZE); + + await Promise.all( + batch.map(async (context) => { + try { + const contextEmbeddings = existingEmbeddingsMap.get(context.id) || []; + const result = await syncSingleContext(context, contextEmbeddings, organizationId); + + if (result === 'created') stats.created++; + else if (result === 'updated') stats.updated++; + else stats.skipped++; + } catch (error) { + logger.error('Failed to sync context', { + contextId: context.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + stats.failed++; + } + }), + ); + } + + logger.info('Context sync completed', { + organizationId, + ...stats, + }); + + return stats; +} + diff --git a/apps/api/src/vector-store/lib/sync/sync-knowledge-base.ts b/apps/api/src/vector-store/lib/sync/sync-knowledge-base.ts new file mode 100644 index 000000000..b70471e3a --- /dev/null +++ b/apps/api/src/vector-store/lib/sync/sync-knowledge-base.ts @@ -0,0 +1,226 @@ +import { db } from '@db'; +import { vectorIndex } from '../core/client'; +import { findEmbeddingsForSource, type ExistingEmbedding } from '../core/find-existing-embeddings'; +import { logger } from '../../logger'; +import { + extractContentFromS3Document, + needsUpdate, + createChunkItems, + upsertChunks, + initSyncStats, + type SyncStats, +} from './sync-utils'; + +const DOCUMENT_BATCH_SIZE = 20; + +interface KnowledgeBaseDocumentData { + id: string; + name: string; + s3Key: string; + fileType: string; + processingStatus: string; + updatedAt: Date; +} + +/** + * Fetch all knowledge base documents for an organization + */ +export async function fetchKnowledgeBaseDocuments( + organizationId: string, +): Promise { + return db.knowledgeBaseDocument.findMany({ + where: { organizationId }, + select: { + id: true, + name: true, + s3Key: true, + fileType: true, + processingStatus: true, + updatedAt: true, + }, + }); +} + +/** + * Update document processing status + */ +async function updateDocumentStatus( + documentId: string, + status: 'pending' | 'processing' | 'completed' | 'failed', +): Promise { + try { + await db.knowledgeBaseDocument.update({ + where: { id: documentId }, + data: { + processingStatus: status, + ...(status === 'completed' || status === 'failed' ? { processedAt: new Date() } : {}), + }, + }); + } catch (error) { + logger.error('Failed to update document status', { + documentId, + status, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } +} + +/** + * Delete existing embeddings for a document + */ +async function deleteExistingDocumentEmbeddings( + documentId: string, + organizationId: string, +): Promise { + const existingDocEmbeddings = await findEmbeddingsForSource( + documentId, + 'knowledge_base_document', + organizationId, + ); + + if (existingDocEmbeddings.length > 0 && vectorIndex) { + const idsToDelete = existingDocEmbeddings.map((e) => e.id); + try { + await vectorIndex.delete(idsToDelete); + logger.info('Deleted existing embeddings', { + documentId, + deletedCount: idsToDelete.length, + }); + } catch (error) { + logger.warn('Failed to delete existing embeddings', { + documentId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } +} + +/** + * Process a single knowledge base document + */ +async function processSingleDocument( + document: KnowledgeBaseDocumentData, + organizationId: string, +): Promise<'processed' | 'failed'> { + const documentUpdatedAt = document.updatedAt.toISOString(); + + logger.info('Processing Knowledge Base document', { + documentId: document.id, + organizationId, + s3Key: document.s3Key, + }); + + await updateDocumentStatus(document.id, 'processing'); + + // Extract content from S3 + const content = await extractContentFromS3Document(document.s3Key, document.fileType); + + if (!content || content.trim().length === 0) { + logger.warn('No content extracted from document', { documentId: document.id }); + await updateDocumentStatus(document.id, 'failed'); + return 'failed'; + } + + // Delete existing embeddings + await deleteExistingDocumentEmbeddings(document.id, organizationId); + + // Create new embeddings + const chunkItems = createChunkItems( + content, + document.id, + 'knowledge_base_document', + organizationId, + documentUpdatedAt, + 'knowledge_base_document', + { documentName: document.name }, + ); + + if (chunkItems.length === 0) { + logger.warn('No chunks created from content', { documentId: document.id }); + await updateDocumentStatus(document.id, 'failed'); + return 'failed'; + } + + await upsertChunks(chunkItems); + logger.info('Successfully created embeddings', { + documentId: document.id, + embeddingCount: chunkItems.length, + }); + + await updateDocumentStatus(document.id, 'completed'); + return 'processed'; +} + +/** + * Filter documents that need processing + */ +function filterDocumentsToProcess( + documents: KnowledgeBaseDocumentData[], + existingEmbeddingsMap: Map, +): KnowledgeBaseDocumentData[] { + return documents.filter((document) => { + const documentEmbeddings = existingEmbeddingsMap.get(document.id) || []; + const documentUpdatedAt = document.updatedAt.toISOString(); + + return ( + document.processingStatus === 'pending' || + document.processingStatus === 'failed' || + needsUpdate(documentEmbeddings, documentUpdatedAt) + ); + }); +} + +/** + * Sync all knowledge base documents for an organization + */ +export async function syncKnowledgeBaseDocuments( + organizationId: string, + existingEmbeddingsMap: Map, +): Promise { + const allDocuments = await fetchKnowledgeBaseDocuments(organizationId); + + logger.info('Found Knowledge Base documents to sync', { + organizationId, + count: allDocuments.length, + }); + + const documentsToProcess = filterDocumentsToProcess(allDocuments, existingEmbeddingsMap); + + const stats = initSyncStats(allDocuments.length); + stats.skipped = allDocuments.length - documentsToProcess.length; + + // Process documents in parallel batches + for (let i = 0; i < documentsToProcess.length; i += DOCUMENT_BATCH_SIZE) { + const batch = documentsToProcess.slice(i, i + DOCUMENT_BATCH_SIZE); + + await Promise.all( + batch.map(async (document) => { + try { + const result = await processSingleDocument(document, organizationId); + + if (result === 'processed') stats.created++; + else stats.failed++; + } catch (error) { + logger.error('Failed to process Knowledge Base document', { + documentId: document.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + + await updateDocumentStatus(document.id, 'failed'); + stats.failed++; + } + }), + ); + } + + logger.info('Knowledge Base documents sync completed', { + organizationId, + processed: stats.created, + skipped: stats.skipped, + failed: stats.failed, + total: stats.total, + }); + + return stats; +} + diff --git a/apps/api/src/vector-store/lib/sync/sync-manual-answer.ts b/apps/api/src/vector-store/lib/sync/sync-manual-answer.ts new file mode 100644 index 000000000..f29bb6c1d --- /dev/null +++ b/apps/api/src/vector-store/lib/sync/sync-manual-answer.ts @@ -0,0 +1,205 @@ +import { upsertEmbedding } from '../core/upsert-embedding'; +import { vectorIndex } from '../core/client'; +import { db } from '@db'; +import { logger } from '../../logger'; + +/** + * Syncs a single manual answer to vector database SYNCHRONOUSLY + * Fast operation (~1-2 seconds) - acceptable for UX + * This ensures manual answers are immediately available for answer generation + */ +export async function syncManualAnswerToVector( + manualAnswerId: string, + organizationId: string, +): Promise<{ success: boolean; error?: string; embeddingId?: string }> { + // Check if vectorIndex is configured + if (!vectorIndex) { + logger.error( + '❌ Upstash Vector not configured - check UPSTASH_VECTOR_REST_URL and UPSTASH_VECTOR_REST_TOKEN', + { + manualAnswerId, + organizationId, + hasUrl: !!process.env.UPSTASH_VECTOR_REST_URL, + hasToken: !!process.env.UPSTASH_VECTOR_REST_TOKEN, + }, + ); + return { success: false, error: 'Vector DB not configured' }; + } + + logger.info('🔍 Vector Index configuration check', { + vectorIndexExists: !!vectorIndex, + manualAnswerId, + organizationId, + }); + + try { + const manualAnswer = await db.securityQuestionnaireManualAnswer.findUnique({ + where: { id: manualAnswerId, organizationId }, + }); + + if (!manualAnswer) { + logger.warn('Manual answer not found for sync', { + manualAnswerId, + organizationId, + }); + return { success: false, error: 'Manual answer not found' }; + } + + // Create embedding ID: manual_answer_{id} + const embeddingId = `manual_answer_${manualAnswerId}`; + + // Combine question and answer for better semantic search + const text = `${manualAnswer.question}\n\n${manualAnswer.answer}`; + + logger.info('🔄 Starting sync manual answer to vector DB', { + manualAnswerId, + organizationId, + embeddingId, + question: manualAnswer.question.substring(0, 100), + answer: manualAnswer.answer.substring(0, 100), + textLength: text.length, + }); + + await upsertEmbedding(embeddingId, text, { + organizationId, + sourceType: 'manual_answer', + sourceId: manualAnswerId, + content: text, + manualAnswerQuestion: manualAnswer.question, // Store question for source identification + updatedAt: manualAnswer.updatedAt.toISOString(), + }); + + // Verify the embedding was actually added by fetching it directly by ID + // Using direct fetch with retry to handle eventual consistency in Upstash Vector + // Even direct fetch can have slight delays, so we retry a few times with exponential backoff + let wasFound = false; + const maxRetries = 3; + const initialDelay = 100; // 100ms + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + const fetchedEmbeddings = await vectorIndex.fetch([embeddingId]); + wasFound = + fetchedEmbeddings && + fetchedEmbeddings.length > 0 && + fetchedEmbeddings[0] !== null; + + if (wasFound) { + break; // Found it, exit retry loop + } + + // If not found and not the last attempt, wait before retrying + if (attempt < maxRetries - 1) { + const delay = initialDelay * Math.pow(2, attempt); // Exponential backoff: 100ms, 200ms, 400ms + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } catch (verifyError) { + // If it's the last attempt, log the error + if (attempt === maxRetries - 1) { + logger.warn( + 'Failed to verify embedding after upsert (final attempt)', + { + embeddingId, + manualAnswerId, + attempt: attempt + 1, + error: + verifyError instanceof Error + ? verifyError.message + : 'Unknown error', + }, + ); + } + // Continue to next retry + } + } + + logger.info('✅ Successfully synced manual answer to vector DB', { + manualAnswerId, + organizationId, + embeddingId, + question: manualAnswer.question.substring(0, 100), + answer: manualAnswer.answer.substring(0, 100), + verified: wasFound, + verificationAttempts: wasFound ? 'success' : `${maxRetries} attempts`, + metadata: { + organizationId, + sourceType: 'manual_answer', + sourceId: manualAnswerId, + updatedAt: manualAnswer.updatedAt.toISOString(), + }, + }); + + // Only log info if verification failed after all retries (non-critical) + // This is non-critical - upsert succeeded, so embedding will be available soon + // Upstash Vector has eventual consistency, so immediate fetch might not find it + // We don't log this as a warning since it's expected behavior + if (!wasFound) { + // Silently continue - upsert succeeded, embedding will be available soon + // No need to log as this is normal eventual consistency behavior + } + return { + success: true, + embeddingId, // Return embedding ID for verification + }; + } catch (error) { + logger.error('Failed to sync manual answer to vector DB', { + manualAnswerId, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} + +/** + * Deletes manual answer from vector database + * Called when manual answer is deleted + */ +export async function deleteManualAnswerFromVector( + manualAnswerId: string, + organizationId: string, +): Promise<{ success: boolean; error?: string }> { + if (!vectorIndex) { + return { success: false, error: 'Vector DB not configured' }; + } + + try { + // Find existing embeddings for this manual answer + // We need to search for embeddings with this sourceId + const embeddingId = `manual_answer_${manualAnswerId}`; + + // Try to delete directly by ID (most efficient) + try { + await vectorIndex.delete([embeddingId]); + logger.info('Deleted manual answer from vector DB', { + manualAnswerId, + organizationId, + embeddingId, + }); + return { success: true }; + } catch (deleteError) { + // If direct delete fails (embedding might not exist), log and continue + logger.warn('Failed to delete manual answer embedding (may not exist)', { + manualAnswerId, + embeddingId, + error: + deleteError instanceof Error ? deleteError.message : 'Unknown error', + }); + // Still return success - embedding might not exist + return { success: true }; + } + } catch (error) { + logger.error('Failed to delete manual answer from vector DB', { + manualAnswerId, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} diff --git a/apps/api/src/vector-store/lib/sync/sync-organization.ts b/apps/api/src/vector-store/lib/sync/sync-organization.ts new file mode 100644 index 000000000..0e42a669b --- /dev/null +++ b/apps/api/src/vector-store/lib/sync/sync-organization.ts @@ -0,0 +1,230 @@ +import { db } from '@db'; +import { vectorIndex } from '../core/client'; +import { + findAllOrganizationEmbeddings, + type ExistingEmbedding, +} from '../core/find-existing-embeddings'; +import { batchUpsertEmbeddings } from '../core/upsert-embedding'; +import { logger } from '../../logger'; +import { syncPolicies, fetchPolicies } from './sync-policies'; +import { syncContextEntries, fetchContextEntries } from './sync-context'; +import { syncKnowledgeBaseDocuments, fetchKnowledgeBaseDocuments } from './sync-knowledge-base'; + +/** + * Lock map to prevent concurrent syncs for the same organization + */ +const syncLocks = new Map>(); + +/** + * Full resync of organization embeddings + * Uses a lock mechanism to prevent concurrent syncs for the same organization. + */ +export async function syncOrganizationEmbeddings(organizationId: string): Promise { + if (!organizationId || organizationId.trim().length === 0) { + logger.warn('Invalid organizationId provided for sync'); + return; + } + + // Check if sync is already in progress + const existingSync = syncLocks.get(organizationId); + if (existingSync) { + logger.info('Sync already in progress, waiting for completion', { organizationId }); + return existingSync; + } + + // Create and store new sync promise + const syncPromise = performSync(organizationId); + syncLocks.set(organizationId, syncPromise); + + // Clean up lock when sync completes + syncPromise + .finally(() => { + syncLocks.delete(organizationId); + logger.info('Sync lock released', { organizationId }); + }) + .catch(() => { + // Error already logged in performSync + }); + + return syncPromise; +} + +/** + * Internal function that performs the actual sync operation + */ +async function performSync(organizationId: string): Promise { + logger.info('Starting incremental organization embeddings sync', { organizationId }); + + try { + // Step 1: Fetch all existing embeddings once + const existingEmbeddings = await findAllOrganizationEmbeddings(organizationId); + logger.info('Fetched existing embeddings', { + organizationId, + totalSources: existingEmbeddings.size, + }); + + // Step 2: Sync policies + const policyStats = await syncPolicies(organizationId, existingEmbeddings); + + // Step 3: Sync context entries + const contextStats = await syncContextEntries(organizationId, existingEmbeddings); + + // Step 4: Sync manual answers + const manualAnswerStats = await syncManualAnswers(organizationId, existingEmbeddings); + + // Step 5: Sync Knowledge Base documents + const kbDocStats = await syncKnowledgeBaseDocuments(organizationId, existingEmbeddings); + + // Step 6: Delete orphaned embeddings + const orphanedDeleted = await deleteOrphanedEmbeddings(organizationId, existingEmbeddings); + + logger.info('Incremental organization embeddings sync completed', { + organizationId, + policies: policyStats, + context: contextStats, + manualAnswers: manualAnswerStats, + knowledgeBaseDocuments: kbDocStats, + orphanedDeleted, + }); + } catch (error) { + logger.error('Failed to sync organization embeddings', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + errorStack: error instanceof Error ? error.stack : undefined, + }); + throw error; + } + } + +/** + * Sync manual answers for an organization + */ +async function syncManualAnswers( + organizationId: string, + existingEmbeddings: Map, +): Promise<{ created: number; updated: number; skipped: number; total: number }> { + const manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ + where: { organizationId }, + select: { + id: true, + question: true, + answer: true, + updatedAt: true, + }, + }); + + logger.info('Syncing manual answers', { + organizationId, + count: manualAnswers.length, + }); + + let created = 0; + let updated = 0; + let skipped = 0; + + if (manualAnswers.length > 0) { + const itemsToUpsert = manualAnswers + .map((ma) => { + const embeddingId = `manual_answer_${ma.id}`; + const text = `${ma.question}\n\n${ma.answer}`; + const updatedAt = ma.updatedAt.toISOString(); + + const existing = existingEmbeddings.get(ma.id) || []; + const needsUpdate = + existing.length === 0 || existing[0]?.updatedAt !== updatedAt; + + if (!needsUpdate) { + skipped++; + return null; + } + + if (existing.length === 0) created++; + else updated++; + + return { + id: embeddingId, + text, + metadata: { + organizationId, + sourceType: 'manual_answer' as const, + sourceId: ma.id, + content: text, + manualAnswerQuestion: ma.question, + updatedAt, + }, + }; + }) + .filter((item): item is NonNullable => item !== null); + + if (itemsToUpsert.length > 0) { + await batchUpsertEmbeddings(itemsToUpsert); + } + } + + logger.info('Manual answers sync completed', { + organizationId, + created, + updated, + skipped, + total: manualAnswers.length, + }); + + return { created, updated, skipped, total: manualAnswers.length }; +} + +/** + * Delete orphaned embeddings (sources that no longer exist in DB) + */ +async function deleteOrphanedEmbeddings( + organizationId: string, + existingEmbeddings: Map, +): Promise { + // Fetch current DB IDs + const [policies, contextEntries, manualAnswers, kbDocuments] = await Promise.all([ + fetchPolicies(organizationId), + fetchContextEntries(organizationId), + db.securityQuestionnaireManualAnswer.findMany({ + where: { organizationId }, + select: { id: true }, + }), + fetchKnowledgeBaseDocuments(organizationId), + ]); + + const dbPolicyIds = new Set(policies.map((p) => p.id)); + const dbContextIds = new Set(contextEntries.map((c) => c.id)); + const dbManualAnswerIds = new Set(manualAnswers.map((ma) => ma.id)); + const dbKbDocIds = new Set(kbDocuments.map((d) => d.id)); + + let orphanedDeleted = 0; + + for (const [sourceId, embeddings] of existingEmbeddings.entries()) { + const sourceType = embeddings[0]?.sourceType; + if (!sourceType) continue; + + const shouldExist = + (sourceType === 'policy' && dbPolicyIds.has(sourceId)) || + (sourceType === 'context' && dbContextIds.has(sourceId)) || + (sourceType === 'manual_answer' && dbManualAnswerIds.has(sourceId)) || + (sourceType === 'knowledge_base_document' && dbKbDocIds.has(sourceId)); + + if (!shouldExist && vectorIndex) { + const idsToDelete = embeddings.map((e) => e.id); + try { + await vectorIndex.delete(idsToDelete); + orphanedDeleted += idsToDelete.length; + logger.info('Deleted orphaned embeddings', { + sourceId, + sourceType, + deletedCount: idsToDelete.length, + }); + } catch (error) { + logger.warn('Failed to delete orphaned embeddings', { + sourceId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + } + + return orphanedDeleted; +} diff --git a/apps/api/src/vector-store/lib/sync/sync-policies.ts b/apps/api/src/vector-store/lib/sync/sync-policies.ts new file mode 100644 index 000000000..2b51805d3 --- /dev/null +++ b/apps/api/src/vector-store/lib/sync/sync-policies.ts @@ -0,0 +1,136 @@ +import { db } from '@db'; +import { extractTextFromPolicy } from '../utils/extract-policy-text'; +import { logger } from '../../logger'; +import type { ExistingEmbedding } from '../core/find-existing-embeddings'; +import { + needsUpdate, + deleteOldEmbeddings, + createChunkItems, + upsertChunks, + initSyncStats, + type SyncStats, +} from './sync-utils'; + +const POLICY_BATCH_SIZE = 100; + +interface PolicyData { + id: string; + name: string; + description: string | null; + content: unknown; + organizationId: string; + updatedAt: Date; +} + +/** + * Fetch all published policies for an organization + */ +export async function fetchPolicies(organizationId: string): Promise { + return db.policy.findMany({ + where: { + organizationId, + status: 'published', + }, + select: { + id: true, + name: true, + description: true, + content: true, + organizationId: true, + updatedAt: true, + }, + }); +} + +/** + * Sync a single policy's embeddings + */ +async function syncSinglePolicy( + policy: PolicyData, + existingEmbeddings: ExistingEmbedding[], + organizationId: string, +): Promise<'created' | 'updated' | 'skipped'> { + const policyUpdatedAt = policy.updatedAt.toISOString(); + + // Check if policy needs update + if (!needsUpdate(existingEmbeddings, policyUpdatedAt)) { + return 'skipped'; + } + + // Delete old embeddings if they exist + await deleteOldEmbeddings(existingEmbeddings, { policyId: policy.id }); + + // Create new embeddings + const policyText = extractTextFromPolicy(policy as Parameters[0]); + + if (!policyText || policyText.trim().length === 0) { + return 'skipped'; + } + + const chunkItems = createChunkItems( + policyText, + policy.id, + 'policy', + organizationId, + policyUpdatedAt, + 'policy', + { policyName: policy.name }, + ); + + if (chunkItems.length === 0) { + return 'skipped'; + } + + await upsertChunks(chunkItems); + + return existingEmbeddings.length === 0 ? 'created' : 'updated'; +} + +/** + * Sync all policies for an organization + */ +export async function syncPolicies( + organizationId: string, + existingEmbeddingsMap: Map, +): Promise { + const policies = await fetchPolicies(organizationId); + + logger.info('Found policies to sync', { + organizationId, + count: policies.length, + }); + + const stats = initSyncStats(policies.length); + + // Process policies in parallel batches + for (let i = 0; i < policies.length; i += POLICY_BATCH_SIZE) { + const batch = policies.slice(i, i + POLICY_BATCH_SIZE); + + await Promise.all( + batch.map(async (policy) => { + try { + const policyEmbeddings = existingEmbeddingsMap.get(policy.id) || []; + const result = await syncSinglePolicy(policy, policyEmbeddings, organizationId); + + if (result === 'created') stats.created++; + else if (result === 'updated') stats.updated++; + else stats.skipped++; + } catch (error) { + logger.error('Failed to sync policy', { + policyId: policy.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + stats.failed++; + } + }), + ); + } + + logger.info('Policies sync completed', { + organizationId, + ...stats, + }); + + return stats; +} + diff --git a/apps/api/src/vector-store/lib/sync/sync-utils.ts b/apps/api/src/vector-store/lib/sync/sync-utils.ts new file mode 100644 index 000000000..75e16864f --- /dev/null +++ b/apps/api/src/vector-store/lib/sync/sync-utils.ts @@ -0,0 +1,183 @@ +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { extractContentFromFile } from '@/vector-store/jobs/helpers/extract-content-from-file'; +import { vectorIndex } from '../core/client'; +import { batchUpsertEmbeddings } from '../core/upsert-embedding'; +import { chunkText } from '../utils/chunk-text'; +import { logger } from '../../logger'; +import type { ExistingEmbedding } from '../core/find-existing-embeddings'; + +export type SourceType = 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document'; + +export interface SyncStats { + created: number; + updated: number; + skipped: number; + failed: number; + total: number; +} + +export interface ChunkItem { + id: string; + text: string; + metadata: { + organizationId: string; + sourceType: SourceType; + sourceId: string; + content: string; + updatedAt: string; + [key: string]: string; + }; +} + +/** + * Creates an S3 client instance for Knowledge Base document processing + */ +export function createKnowledgeBaseS3Client(): S3Client { + const region = process.env.APP_AWS_REGION || 'us-east-1'; + const accessKeyId = process.env.APP_AWS_ACCESS_KEY_ID; + const secretAccessKey = process.env.APP_AWS_SECRET_ACCESS_KEY; + + if (!accessKeyId || !secretAccessKey) { + throw new Error( + 'AWS S3 credentials are missing. Please set APP_AWS_ACCESS_KEY_ID and APP_AWS_SECRET_ACCESS_KEY environment variables.', + ); + } + + return new S3Client({ + region, + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +/** + * Extracts content from a Knowledge Base document stored in S3 + */ +export async function extractContentFromS3Document( + s3Key: string, + fileType: string, +): Promise { + const knowledgeBaseBucket = process.env.APP_AWS_KNOWLEDGE_BASE_BUCKET; + + if (!knowledgeBaseBucket) { + throw new Error( + 'Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable.', + ); + } + + const s3Client = createKnowledgeBaseS3Client(); + + const getCommand = new GetObjectCommand({ + Bucket: knowledgeBaseBucket, + Key: s3Key, + }); + + const response = await s3Client.send(getCommand); + + if (!response.Body) { + throw new Error('Failed to retrieve file from S3'); + } + + // Convert stream to buffer + const chunks: Uint8Array[] = []; + for await (const chunk of response.Body as AsyncIterable) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + const base64Data = buffer.toString('base64'); + + const detectedFileType = response.ContentType || fileType || 'application/octet-stream'; + return extractContentFromFile(base64Data, detectedFileType); +} + +/** + * Check if embeddings need to be updated based on updatedAt timestamp + */ +export function needsUpdate( + existingEmbeddings: ExistingEmbedding[], + updatedAt: string, +): boolean { + return ( + existingEmbeddings.length === 0 || + existingEmbeddings.some((e) => !e.updatedAt || e.updatedAt < updatedAt) + ); +} + +/** + * Delete old embeddings by IDs + */ +export async function deleteOldEmbeddings( + embeddings: ExistingEmbedding[], + logContext: Record, +): Promise { + if (embeddings.length === 0 || !vectorIndex) { + return; + } + + const idsToDelete = embeddings.map((e) => e.id); + try { + await vectorIndex.delete(idsToDelete); + } catch (error) { + logger.warn('Failed to delete old embeddings', { + ...logContext, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } +} + +/** + * Create chunk items from text for embedding + */ +export function createChunkItems( + text: string, + sourceId: string, + sourceType: SourceType, + organizationId: string, + updatedAt: string, + idPrefix: string, + extraMetadata: Record = {}, + chunkSize = 500, + overlap = 50, +): ChunkItem[] { + const chunks = chunkText(text, chunkSize, overlap); + + return chunks + .map((chunk, chunkIndex) => ({ + id: `${idPrefix}_${sourceId}_chunk${chunkIndex}`, + text: chunk, + metadata: { + organizationId, + sourceType, + sourceId, + content: chunk, + updatedAt, + ...extraMetadata, + }, + })) + .filter((item) => item.text && item.text.trim().length > 0); +} + +/** + * Upsert chunk items to vector store + */ +export async function upsertChunks(chunkItems: ChunkItem[]): Promise { + if (chunkItems.length > 0) { + await batchUpsertEmbeddings(chunkItems); + } +} + +/** + * Initialize sync stats + */ +export function initSyncStats(total: number): SyncStats { + return { + created: 0, + updated: 0, + skipped: 0, + failed: 0, + total, + }; +} + diff --git a/apps/api/src/vector-store/lib/utils/chunk-text.ts b/apps/api/src/vector-store/lib/utils/chunk-text.ts new file mode 100644 index 000000000..2f17d1a8c --- /dev/null +++ b/apps/api/src/vector-store/lib/utils/chunk-text.ts @@ -0,0 +1,85 @@ +/** + * Splits text into chunks of approximately the specified token size + * Uses a simple approximation: 1 token ≈ 4 characters + * @param text - The text to chunk + * @param chunkSizeTokens - Target size in tokens (default: 500) + * @param overlapTokens - Number of tokens to overlap between chunks (default: 50) + * @returns Array of text chunks + */ +export function chunkText( + text: string, + chunkSizeTokens: number = 500, + overlapTokens: number = 50, +): string[] { + // Validate inputs + if (!text || typeof text !== 'string' || text.trim().length === 0) { + return []; + } + + if (chunkSizeTokens <= 0 || overlapTokens < 0) { + throw new Error( + 'Invalid chunk parameters: chunkSizeTokens must be > 0, overlapTokens must be >= 0', + ); + } + + if (overlapTokens >= chunkSizeTokens) { + throw new Error( + 'Invalid chunk parameters: overlapTokens must be less than chunkSizeTokens', + ); + } + + // Simple approximation: 1 token ≈ 4 characters + const chunkSizeChars = Math.max(1, Math.floor(chunkSizeTokens * 4)); + const overlapChars = Math.max(0, Math.floor(overlapTokens * 4)); + + if (text.length <= chunkSizeChars) { + return [text.trim()].filter((chunk) => chunk.length > 0); + } + + const chunks: string[] = []; + let start = 0; + let iterations = 0; + const maxIterations = + Math.ceil(text.length / (chunkSizeChars - overlapChars)) + 10; // Safety limit + + while (start < text.length && iterations < maxIterations) { + iterations++; + const end = Math.min(start + chunkSizeChars, text.length); + + if (end <= start) { + break; // Prevent infinite loop + } + + let chunk = text.slice(start, end); + + // Try to break at sentence boundaries if possible + if (end < text.length && chunk.length > chunkSizeChars * 0.7) { + const lastPeriod = chunk.lastIndexOf('.'); + const lastNewline = chunk.lastIndexOf('\n'); + const breakPoint = Math.max(lastPeriod, lastNewline); + + if (breakPoint > chunkSizeChars * 0.7) { + // Only break at sentence if we're at least 70% through the chunk + chunk = chunk.slice(0, breakPoint + 1); + } + } + + const trimmedChunk = chunk.trim(); + if (trimmedChunk.length > 0) { + chunks.push(trimmedChunk); + } + + // Move start position forward, accounting for overlap + const nextStart = end - overlapChars; + if (nextStart <= start) { + // Prevent infinite loop - move forward at least 1 character + start = start + 1; + } else { + start = nextStart; + } + + if (start >= text.length) break; + } + + return chunks.filter((chunk) => chunk.length > 0); +} diff --git a/apps/api/src/vector-store/lib/utils/extract-policy-text.ts b/apps/api/src/vector-store/lib/utils/extract-policy-text.ts new file mode 100644 index 000000000..eb4d8e606 --- /dev/null +++ b/apps/api/src/vector-store/lib/utils/extract-policy-text.ts @@ -0,0 +1,107 @@ +import type { Policy } from '@db'; + +/** + * Extracts plain text from a TipTap JSON policy content + * Handles various TipTap node types: paragraphs, headings, lists, etc. + * @param policy - The policy object with TipTap JSON content + * @returns Plain text representation of the policy + */ +export function extractTextFromPolicy(policy: Policy): string { + if (!policy.content || !Array.isArray(policy.content)) { + return ''; + } + + const textParts: string[] = []; + + // Add policy name and description if available + if (policy.name) { + textParts.push(`Policy: ${policy.name}`); + } + if (policy.description) { + textParts.push(`Description: ${policy.description}`); + } + + // Process TipTap JSON content + const processNode = (node: any): string => { + if (!node || typeof node !== 'object') { + return ''; + } + + const parts: string[] = []; + + // Handle text nodes + if (node.type === 'text' && node.text) { + return node.text; + } + + // Handle headings + if (node.type === 'heading' && node.content) { + const headingText = node.content + .map((child: any) => processNode(child)) + .join(''); + parts.push(headingText); + } + + // Handle paragraphs + if (node.type === 'paragraph' && node.content) { + const paraText = node.content + .map((child: any) => processNode(child)) + .join(''); + if (paraText.trim()) { + parts.push(paraText); + } + } + + // Handle bullet lists + if (node.type === 'bulletList' && node.content) { + node.content.forEach((listItem: any) => { + if (listItem.type === 'listItem' && listItem.content) { + listItem.content.forEach((itemContent: any) => { + const itemText = processNode(itemContent); + if (itemText.trim()) { + parts.push(`• ${itemText}`); + } + }); + } + }); + } + + // Handle ordered lists + if (node.type === 'orderedList' && node.content) { + let index = 1; + node.content.forEach((listItem: any) => { + if (listItem.type === 'listItem' && listItem.content) { + listItem.content.forEach((itemContent: any) => { + const itemText = processNode(itemContent); + if (itemText.trim()) { + parts.push(`${index}. ${itemText}`); + index++; + } + }); + } + }); + } + + // Handle other node types recursively + if (node.content && Array.isArray(node.content)) { + node.content.forEach((child: any) => { + const childText = processNode(child); + if (childText.trim()) { + parts.push(childText); + } + }); + } + + return parts.join('\n'); + }; + + // Process all content nodes + policy.content.forEach((node: any) => { + const nodeText = processNode(node); + if (nodeText.trim()) { + textParts.push(nodeText); + } + }); + + return textParts.join('\n\n'); +} diff --git a/apps/api/src/vector-store/logger.ts b/apps/api/src/vector-store/logger.ts new file mode 100644 index 000000000..371800220 --- /dev/null +++ b/apps/api/src/vector-store/logger.ts @@ -0,0 +1,90 @@ +/** + * Universal logger that works in both NestJS and Trigger.dev runtime environments + * + * In Trigger.dev tasks, we use console.log with structured output + * In NestJS services, we use the NestJS Logger + * + * This allows shared lib files to be imported by both Trigger.dev tasks and NestJS services + */ + +type LogPayload = Record | undefined; + +const formatMessage = (message: string, payload?: LogPayload): string => { + if (!payload) { + return message; + } + try { + return `${message} ${JSON.stringify(payload)}`; + } catch { + return message; + } +}; + +/** + * Detect if we're running in Trigger.dev environment + * Trigger.dev sets specific env vars when running tasks + */ +const isTriggerDevRuntime = (): boolean => { + return !!( + process.env.TRIGGER_RUN_ID || + process.env.TRIGGER_TASK_ID || + process.env.TRIGGER_WORKER_ID + ); +}; + +/** + * Create a logger that works in both environments + * - In Trigger.dev: uses console methods (which Trigger.dev intercepts and formats) + * - In NestJS: uses NestJS Logger + */ +const createLogger = () => { + // Check if running in Trigger.dev - use console logging + if (isTriggerDevRuntime()) { + return { + info: (message: string, payload?: LogPayload): void => { + console.log(`[VectorStore] ${formatMessage(message, payload)}`); + }, + warn: (message: string, payload?: LogPayload): void => { + console.warn(`[VectorStore] ${formatMessage(message, payload)}`); + }, + error: (message: string, payload?: LogPayload): void => { + console.error(`[VectorStore] ${formatMessage(message, payload)}`); + }, + }; + } + + // In NestJS environment, try to use NestJS Logger + // We import dynamically to avoid issues in Trigger.dev runtime + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { Logger } = require('@nestjs/common'); + const baseLogger = new Logger('VectorStore'); + + return { + info: (message: string, payload?: LogPayload): void => { + baseLogger.log(formatMessage(message, payload)); + }, + warn: (message: string, payload?: LogPayload): void => { + baseLogger.warn(formatMessage(message, payload)); + }, + error: (message: string, payload?: LogPayload): void => { + baseLogger.error(formatMessage(message, payload)); + }, +}; + } catch { + // Fallback to console if NestJS is not available + return { + info: (message: string, payload?: LogPayload): void => { + console.log(`[VectorStore] ${formatMessage(message, payload)}`); + }, + warn: (message: string, payload?: LogPayload): void => { + console.warn(`[VectorStore] ${formatMessage(message, payload)}`); + }, + error: (message: string, payload?: LogPayload): void => { + console.error(`[VectorStore] ${formatMessage(message, payload)}`); + }, + }; + } +}; + +export const logger = createLogger(); diff --git a/apps/api/src/vector-store/vector-store.module.ts b/apps/api/src/vector-store/vector-store.module.ts new file mode 100644 index 000000000..896539916 --- /dev/null +++ b/apps/api/src/vector-store/vector-store.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { VectorStoreService } from './vector-store.service'; + +@Module({ + providers: [VectorStoreService], + exports: [VectorStoreService], +}) +export class VectorStoreModule {} diff --git a/apps/api/src/vector-store/vector-store.service.ts b/apps/api/src/vector-store/vector-store.service.ts new file mode 100644 index 000000000..f0e433517 --- /dev/null +++ b/apps/api/src/vector-store/vector-store.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@nestjs/common'; +import type { SimilarContentResult } from './lib'; +import { + countEmbeddings, + deleteManualAnswerFromVector, + findSimilarContent, + listManualAnswerEmbeddings, + syncManualAnswerToVector, + syncOrganizationEmbeddings, +} from './lib'; + +@Injectable() +export class VectorStoreService { + async syncOrganization(organizationId: string): Promise { + await syncOrganizationEmbeddings(organizationId); + } + + async syncManualAnswer(manualAnswerId: string, organizationId: string) { + return syncManualAnswerToVector(manualAnswerId, organizationId); + } + + async deleteManualAnswer(manualAnswerId: string, organizationId: string) { + return deleteManualAnswerFromVector(manualAnswerId, organizationId); + } + + async searchSimilarContent( + question: string, + organizationId: string, + ): Promise { + return findSimilarContent(question, organizationId); + } + + async countOrganizationEmbeddings( + organizationId: string, + sourceType?: 'policy' | 'context' | 'manual_answer', + ) { + return countEmbeddings(organizationId, sourceType); + } + + async listManualAnswers(organizationId: string) { + return listManualAnswerEmbeddings(organizationId); + } +} diff --git a/apps/api/trigger.config.ts b/apps/api/trigger.config.ts new file mode 100644 index 000000000..29d31d7d5 --- /dev/null +++ b/apps/api/trigger.config.ts @@ -0,0 +1,33 @@ +import { PrismaInstrumentation } from '@prisma/instrumentation'; +import { syncVercelEnvVars } from '@trigger.dev/build/extensions/core'; +import { defineConfig } from '@trigger.dev/sdk'; +import { prismaExtension } from './customPrismaExtension'; + +export default defineConfig({ + project: 'proj_zhioyrusqertqgafqgpj', // API project + logLevel: 'log', + instrumentations: [new PrismaInstrumentation()], + maxDuration: 300, // 5 minutes + build: { + extensions: [ + prismaExtension({ + version: '6.13.0', + dbPackageVersion: '^1.3.15', // Version of @trycompai/db package with compiled JS + }), + syncVercelEnvVars(), + ], + }, + retries: { + enabledInDev: true, + default: { + maxAttempts: 3, + minTimeoutInMs: 1000, + maxTimeoutInMs: 10000, + factor: 2, + randomize: true, + }, + }, + dirs: ['./src/vector-store/jobs', './src/questionnaire/vendors'], +}); + + diff --git a/apps/app/.env.example b/apps/app/.env.example index cf8f3fdf3..a691a2f90 100644 --- a/apps/app/.env.example +++ b/apps/app/.env.example @@ -8,7 +8,7 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/comp" # Should be th AUTH_SECRET="" # Used for auth, use something random and strong SECRET_KEY="" # Used for encrypting data, use something random and strong NEXT_PUBLIC_BETTER_AUTH_URL=http://localhost:3000 # Must point to the domain hosting the app - +NEXT_PUBLIC_API_URL=http://localhost:3333 # Upstash UPSTASH_REDIS_REST_URL="" # Optional, used for rate limiting UPSTASH_REDIS_REST_TOKEN="" # Optional, used for rate limiting @@ -37,6 +37,7 @@ APP_AWS_BUCKET_NAME="" # Required, for task attachments APP_AWS_REGION="" # Required, for task attachments APP_AWS_ACCESS_KEY_ID="" # Required, for task attachments APP_AWS_SECRET_ACCESS_KEY="" # Required, for task attachments +APP_AWS_ORG_ASSETS_BUCKET="" # Required, for org compliance and logo # TRIGGER REVAL REVALIDATION_SECRET="" # Revalidate server side, generate something random diff --git a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/components/PolicyDetails.tsx b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/components/PolicyDetails.tsx index ab303af21..cc784717a 100644 --- a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/components/PolicyDetails.tsx +++ b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/components/PolicyDetails.tsx @@ -10,22 +10,16 @@ import '@comp/ui/editor.css'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@comp/ui/tabs'; import type { PolicyDisplayFormat } from '@db'; import type { JSONContent } from '@tiptap/react'; -import { - DefaultChatTransport, - getToolName, - isToolUIPart, - type ToolUIPart, - type UIMessage, -} from 'ai'; +import { DefaultChatTransport } from 'ai'; import { structuredPatch } from 'diff'; import { CheckCircle, Loader2, Sparkles, X } from 'lucide-react'; import { useAction } from 'next-safe-action/hooks'; -import { useFeatureFlagEnabled } from 'posthog-js/react'; -import { useMemo, useState } from 'react'; +import { useCallback, useMemo, useRef, useState } from 'react'; import { toast } from 'sonner'; import { switchPolicyDisplayFormatAction } from '../../actions/switch-policy-display-format'; import { PdfViewer } from '../../components/PdfViewer'; import { updatePolicy } from '../actions/update-policy'; +import type { PolicyChatUIMessage } from '../types'; import { markdownToTipTapJSON } from './ai/markdown-utils'; import { PolicyAiAssistant } from './ai/policy-ai-assistant'; @@ -49,24 +43,32 @@ interface LatestProposal { key: string; content: string; summary: string; + title: string; + detail: string; + reviewHint: string; } -function getLatestProposedPolicy(messages: UIMessage[]): LatestProposal | null { +function getLatestProposedPolicy(messages: PolicyChatUIMessage[]): LatestProposal | null { const lastAssistantMessage = [...messages].reverse().find((m) => m.role === 'assistant'); if (!lastAssistantMessage?.parts) return null; let latest: LatestProposal | null = null; lastAssistantMessage.parts.forEach((part, index) => { - if (!isToolUIPart(part) || getToolName(part) !== 'proposePolicy') return; - const toolPart = part as ToolUIPart; - const input = toolPart.input as { content?: string; summary?: string } | undefined; + if (part.type !== 'tool-proposePolicy') return; + if (part.state === 'input-streaming' || part.state === 'output-error') return; + const input = part.input; if (!input?.content) return; latest = { key: `${lastAssistantMessage.id}:${index}`, content: input.content, summary: input.summary ?? 'Proposing policy changes', + title: input.title ?? input.summary ?? 'Policy updates ready for your review', + detail: + input.detail ?? + 'I have prepared an updated version of this policy based on your instructions.', + reviewHint: input.reviewHint ?? 'Review the proposed changes below before applying them.', }; }); @@ -100,13 +102,18 @@ export function PolicyContentManager({ const [dismissedProposalKey, setDismissedProposalKey] = useState(null); const [isApplying, setIsApplying] = useState(false); const [chatErrorMessage, setChatErrorMessage] = useState(null); - const isAiPolicyAssistantEnabled = useFeatureFlagEnabled('is-ai-policy-assistant-enabled'); + const diffViewerRef = useRef(null); + + const isAiPolicyAssistantEnabled = true; + const scrollToDiffViewer = useCallback(() => { + diffViewerRef.current?.scrollIntoView({ behavior: 'smooth', block: 'start' }); + }, []); const { messages, status, sendMessage: baseSendMessage, - } = useChat({ + } = useChat({ transport: new DefaultChatTransport({ api: `/api/policies/${policyId}/chat`, }), @@ -128,6 +135,20 @@ export function PolicyContentManager({ const proposedPolicyMarkdown = activeProposal?.content ?? null; + const hasPendingProposal = useMemo( + () => + messages.some( + (m) => + m.role === 'assistant' && + m.parts?.some( + (part) => + part.type === 'tool-proposePolicy' && + (part.state === 'input-streaming' || part.state === 'input-available'), + ), + ), + [messages], + ); + const switchFormat = useAction(switchPolicyDisplayFormatAction, { onError: () => toast.error('Failed to switch view.'), }); @@ -167,8 +188,8 @@ export function PolicyContentManager({
-
-
+
+
@@ -219,13 +240,15 @@ export function PolicyContentManager({
{showAiAssistant && isAiPolicyAssistantEnabled && ( -
+
setShowAiAssistant(false)} + onScrollToDiff={scrollToDiffViewer} + hasActiveProposal={!!activeProposal && !hasPendingProposal} />
)} @@ -233,8 +256,8 @@ export function PolicyContentManager({ - {proposedPolicyMarkdown && diffPatch && activeProposal && ( -
+ {proposedPolicyMarkdown && diffPatch && activeProposal && !hasPendingProposal && ( +
+ )} +

+ + ); + } + + return null; + })} +
+ )) + )} + {isLoading && !hasActiveTool && ( +
Thinking...
+ )} + + + {errorMessage && (
diff --git a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/tools/policy-tools.ts b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/tools/policy-tools.ts new file mode 100644 index 000000000..1b0e1b01b --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/tools/policy-tools.ts @@ -0,0 +1,45 @@ +import { type InferUITools, tool } from 'ai'; +import { z } from 'zod'; + +export function getPolicyTools() { + return { + proposePolicy: tool({ + description: + 'Propose an updated version of the policy. Use this tool whenever the user asks you to make changes, edits, or improvements to the policy. You must provide the COMPLETE policy content, not just the changes.', + inputSchema: z.object({ + content: z + .string() + .describe( + 'The complete updated policy content in markdown format. Must include the entire policy, not just the changed sections.', + ), + summary: z + .string() + .describe('One to two sentences summarizing the changes. No bullet points.'), + title: z + .string() + .describe( + 'A short, sentence-case heading (~4–10 words) that clearly states the main change, for use in a small review banner.', + ), + detail: z + .string() + .describe( + 'One or two plain-text sentences briefly explaining what changed and why, shown in the review banner.', + ), + reviewHint: z + .string() + .describe( + 'A very short imperative phrase that tells the user to review the updated policy content in the editor below.', + ), + }), + execute: async ({ summary, title, detail, reviewHint }) => ({ + success: true, + summary, + title, + detail, + reviewHint, + }), + }), + }; +} + +export type PolicyToolSet = InferUITools>; diff --git a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/types/index.ts b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/types/index.ts index d08b5b233..857b62bc7 100644 --- a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/types/index.ts +++ b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/editor/types/index.ts @@ -1,4 +1,8 @@ +import type { UIMessage } from 'ai'; import { z } from 'zod'; +import type { PolicyToolSet } from '../tools/policy-tools'; + +export type PolicyChatUIMessage = UIMessage; export const policyDetailsSchema = z.object({ id: z.string(), diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx deleted file mode 100644 index 13ab01bea..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx +++ /dev/null @@ -1,48 +0,0 @@ -'use client'; - -import { FileQuestion, FileText, ChevronRight } from 'lucide-react'; -import Link from 'next/link'; -import { useParams } from 'next/navigation'; - -interface QuestionnaireBreadcrumbProps { - filename: string; - organizationId: string; -} - -export function QuestionnaireBreadcrumb({ filename, organizationId }: QuestionnaireBreadcrumbProps) { - const params = useParams(); - const orgId = params.orgId as string; - - return ( - - ); -} - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/answer-single-question.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/answer-single-question.ts deleted file mode 100644 index ba60e148a..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/answer-single-question.ts +++ /dev/null @@ -1,71 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { answerQuestion } from '@/jobs/tasks/vendors/answer-question'; -import { z } from 'zod'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; - -const inputSchema = z.object({ - question: z.string(), - questionIndex: z.number(), - totalQuestions: z.number(), -}); - -export const answerSingleQuestionAction = authActionClient - .inputSchema(inputSchema) - .metadata({ - name: 'answer-single-question', - track: { - event: 'answer-single-question', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { question, questionIndex, totalQuestions } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId) { - throw new Error('No active organization'); - } - - const organizationId = session.activeOrganizationId; - - try { - // Call answerQuestion function directly - const result = await answerQuestion( - { - question, - organizationId, - questionIndex, - totalQuestions, - }, - { - useMetadata: false, - }, - ); - - // Revalidate the page to show updated answer - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - revalidatePath(path); - - return { - success: result.success, - data: { - questionIndex: result.questionIndex, - question: result.question, - answer: result.answer, - sources: result.sources, - error: result.error, - }, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Failed to answer question', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/create-trigger-token.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/create-trigger-token.ts index 3b6565c13..941ecbdc3 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/create-trigger-token.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/create-trigger-token.ts @@ -43,40 +43,3 @@ export const createTriggerToken = async (taskId: 'parse-questionnaire' | 'vendor }; } }; - -// Create public token with read permissions for a specific run -export const createRunReadToken = async (runId: string) => { - const session = await betterAuth.api.getSession({ - headers: await headers(), - }); - - if (!session) { - return { - success: false, - error: 'Unauthorized', - }; - } - - try { - const token = await auth.createPublicToken({ - scopes: { - read: { - runs: [runId], - }, - }, - expirationTime: '1hr', - }); - - return { - success: true, - token, - }; - } catch (error) { - console.error('Error creating run read token:', error); - return { - success: false, - error: error instanceof Error ? error.message : 'Failed to create run read token', - }; - } -}; - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/delete-questionnaire-answer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/delete-questionnaire-answer.ts deleted file mode 100644 index a2878b4aa..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/delete-questionnaire-answer.ts +++ /dev/null @@ -1,118 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { Prisma } from '@prisma/client'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; - -const deleteAnswerSchema = z.object({ - questionnaireId: z.string(), - questionAnswerId: z.string(), -}); - -export const deleteQuestionnaireAnswer = authActionClient - .inputSchema(deleteAnswerSchema) - .metadata({ - name: 'delete-questionnaire-answer', - track: { - event: 'delete-questionnaire-answer', - description: 'Delete Questionnaire Answer', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { questionnaireId, questionAnswerId } = parsedInput; - const { activeOrganizationId } = ctx.session; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - // Verify questionnaire exists and belongs to organization - const questionnaire = await db.questionnaire.findUnique({ - where: { - id: questionnaireId, - organizationId: activeOrganizationId, - }, - }); - - if (!questionnaire) { - return { - success: false, - error: 'Questionnaire not found', - }; - } - - // Verify question answer exists and belongs to questionnaire - const questionAnswer = await db.questionnaireQuestionAnswer.findUnique({ - where: { - id: questionAnswerId, - questionnaireId, - }, - }); - - if (!questionAnswer) { - return { - success: false, - error: 'Question answer not found', - }; - } - - // Delete the answer (set to null and status to untouched) - await db.questionnaireQuestionAnswer.update({ - where: { - id: questionAnswerId, - }, - data: { - answer: null, - status: 'untouched', - sources: Prisma.JsonNull, - generatedAt: null, - updatedBy: null, - updatedAt: new Date(), - }, - }); - - // Update answered questions count - const answeredCount = await db.questionnaireQuestionAnswer.count({ - where: { - questionnaireId, - answer: { - not: null, - }, - }, - }); - - await db.questionnaire.update({ - where: { - id: questionnaireId, - }, - data: { - answeredQuestions: answeredCount, - }, - }); - - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - - revalidatePath(path); - - return { - success: true, - }; - } catch (error) { - console.error('Error deleting answer:', error); - return { - success: false, - error: 'Failed to delete answer', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/export-questionnaire.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/export-questionnaire.ts deleted file mode 100644 index 3a7dba5b6..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/export-questionnaire.ts +++ /dev/null @@ -1,202 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import * as XLSX from 'xlsx'; -import { jsPDF } from 'jspdf'; -import { z } from 'zod'; - -const inputSchema = z.object({ - questionsAndAnswers: z.array( - z.object({ - question: z.string(), - answer: z.string().nullable(), - }), - ), - format: z.enum(['xlsx', 'csv', 'pdf']), -}); - -interface QuestionAnswer { - question: string; - answer: string | null; -} - -/** - * Generates XLSX file from questions and answers - */ -function generateXLSX(questionsAndAnswers: QuestionAnswer[]): Buffer { - const workbook = XLSX.utils.book_new(); - - // Create worksheet data - const worksheetData = [ - ['#', 'Question', 'Answer'], // Header row - ...questionsAndAnswers.map((qa, index) => [ - index + 1, - qa.question, - qa.answer || '', - ]), - ]; - - const worksheet = XLSX.utils.aoa_to_sheet(worksheetData); - - // Set column widths - worksheet['!cols'] = [ - { wch: 5 }, // # - { wch: 60 }, // Question - { wch: 60 }, // Answer - ]; - - XLSX.utils.book_append_sheet(workbook, worksheet, 'Questionnaire'); - - // Convert to buffer - return Buffer.from(XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })); -} - -/** - * Generates CSV file from questions and answers - */ -function generateCSV(questionsAndAnswers: QuestionAnswer[]): string { - const rows = [ - ['#', 'Question', 'Answer'], // Header row - ...questionsAndAnswers.map((qa, index) => [ - String(index + 1), - qa.question.replace(/"/g, '""'), // Escape quotes - (qa.answer || '').replace(/"/g, '""'), // Escape quotes - ]), - ]; - - return rows.map((row) => row.map((cell) => `"${cell}"`).join(',')).join('\n'); -} - -/** - * Generates PDF file from questions and answers - */ -function generatePDF(questionsAndAnswers: QuestionAnswer[], vendorName?: string): Buffer { - const doc = new jsPDF(); - const pageWidth = doc.internal.pageSize.getWidth(); - const pageHeight = doc.internal.pageSize.getHeight(); - const margin = 20; - const contentWidth = pageWidth - 2 * margin; - let yPosition = margin; - const lineHeight = 7; - - // Add title - doc.setFontSize(16); - doc.setFont('helvetica', 'bold'); - const title = vendorName ? `Questionnaire: ${vendorName}` : 'Questionnaire'; - doc.text(title, margin, yPosition); - yPosition += lineHeight * 2; - - // Add date - doc.setFontSize(10); - doc.setFont('helvetica', 'normal'); - doc.text(`Generated: ${new Date().toLocaleDateString()}`, margin, yPosition); - yPosition += lineHeight * 2; - - // Process each question-answer pair - doc.setFontSize(11); - - questionsAndAnswers.forEach((qa, index) => { - // Check if we need a new page - if (yPosition > pageHeight - 40) { - doc.addPage(); - yPosition = margin; - } - - // Question number and question - doc.setFont('helvetica', 'bold'); - const questionText = `Q${index + 1}: ${qa.question}`; - const questionLines = doc.splitTextToSize(questionText, contentWidth); - doc.text(questionLines, margin, yPosition); - yPosition += questionLines.length * lineHeight + 2; - - // Answer - doc.setFont('helvetica', 'normal'); - const answerText = qa.answer || 'No answer provided'; - const answerLines = doc.splitTextToSize(`A${index + 1}: ${answerText}`, contentWidth); - doc.text(answerLines, margin, yPosition); - yPosition += answerLines.length * lineHeight + 4; - }); - - // Convert to buffer - return Buffer.from(doc.output('arraybuffer')); -} - -export const exportQuestionnaire = authActionClient - .inputSchema(inputSchema) - .metadata({ - name: 'export-questionnaire', - track: { - event: 'export-questionnaire', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { questionsAndAnswers, format } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId) { - throw new Error('No active organization'); - } - - const organizationId = session.activeOrganizationId; - - try { - const vendorName = 'questionnaire'; - const sanitizedVendorName = vendorName.toLowerCase().replace(/[^a-z0-9]/g, '-'); - const timestamp = new Date().toISOString().split('T')[0]; - - let fileBuffer: Buffer; - let mimeType: string; - let fileExtension: string; - let filename: string; - - // Generate file based on format - switch (format) { - case 'xlsx': { - fileBuffer = generateXLSX(questionsAndAnswers); - mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; - fileExtension = 'xlsx'; - filename = `questionnaire-${sanitizedVendorName}-${timestamp}.xlsx`; - break; - } - - case 'csv': { - const csvContent = generateCSV(questionsAndAnswers); - fileBuffer = Buffer.from(csvContent, 'utf-8'); - mimeType = 'text/csv'; - fileExtension = 'csv'; - filename = `questionnaire-${sanitizedVendorName}-${timestamp}.csv`; - break; - } - - case 'pdf': { - fileBuffer = generatePDF(questionsAndAnswers, vendorName); - mimeType = 'application/pdf'; - fileExtension = 'pdf'; - filename = `questionnaire-${sanitizedVendorName}-${timestamp}.pdf`; - break; - } - - default: - throw new Error(`Unsupported format: ${format}`); - } - - // Convert buffer to base64 data URL for direct download - const base64Data = fileBuffer.toString('base64'); - const dataUrl = `data:${mimeType};base64,${base64Data}`; - - return { - success: true, - data: { - downloadUrl: dataUrl, - filename, - }, - }; - } catch (error) { - throw error instanceof Error - ? error - : new Error('Failed to export questionnaire'); - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/parse-questionnaire-ai.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/parse-questionnaire-ai.ts deleted file mode 100644 index af8d55b20..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/parse-questionnaire-ai.ts +++ /dev/null @@ -1,96 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { parseQuestionnaireTask } from '@/jobs/tasks/vendors/parse-questionnaire'; -import { tasks } from '@trigger.dev/sdk'; -import { z } from 'zod'; -import { APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET } from '@/app/s3'; - -const inputSchema = z.object({ - inputType: z.enum(['file', 'url', 'attachment', 's3']), - // For file uploads - fileData: z.string().optional(), // base64 encoded - fileName: z.string().optional(), - fileType: z.string().optional(), // MIME type - // For URLs - url: z.string().url().optional(), - // For attachments - attachmentId: z.string().optional(), - // For S3 keys (temporary questionnaire files) - s3Key: z.string().optional(), -}); - -export const parseQuestionnaireAI = authActionClient - .inputSchema(inputSchema) - .metadata({ - name: 'parse-questionnaire-ai', - track: { - event: 'parse-questionnaire-ai', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { inputType } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId) { - throw new Error('No active organization'); - } - - // Validate questionnaire upload bucket is configured - if (!APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET) { - throw new Error('Questionnaire upload service is not configured. Please set APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET environment variable to use this feature.'); - } - - const organizationId = session.activeOrganizationId; - - try { - // Trigger the parse questionnaire task in Trigger.dev - // Only include fileData if inputType is 'file' (for backward compatibility) - // Otherwise use attachmentId or url - const payload: { - inputType: 'file' | 'url' | 'attachment' | 's3'; - organizationId: string; - fileData?: string; - fileName?: string; - fileType?: string; - url?: string; - attachmentId?: string; - s3Key?: string; - } = { - inputType, - organizationId, - }; - - if (inputType === 'file' && parsedInput.fileData) { - payload.fileData = parsedInput.fileData; - payload.fileName = parsedInput.fileName; - payload.fileType = parsedInput.fileType; - } else if (inputType === 'url' && parsedInput.url) { - payload.url = parsedInput.url; - } else if (inputType === 'attachment' && parsedInput.attachmentId) { - payload.attachmentId = parsedInput.attachmentId; - } else if (inputType === 's3' && parsedInput.s3Key) { - payload.s3Key = parsedInput.s3Key; - payload.fileName = parsedInput.fileName; - payload.fileType = parsedInput.fileType; - } - - const handle = await tasks.trigger( - 'parse-questionnaire', - payload, - ); - - return { - success: true, - data: { - taskId: handle.id, // Return task ID for polling - }, - }; - } catch (error) { - throw error instanceof Error - ? error - : new Error('Failed to trigger parse questionnaire task'); - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/save-answer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/save-answer.ts deleted file mode 100644 index 08bb3256f..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/save-answer.ts +++ /dev/null @@ -1,209 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; -import { syncManualAnswerToVector } from '@/lib/vector/sync/sync-manual-answer'; -import { logger } from '@/utils/logger'; - -const saveAnswerSchema = z.object({ - questionnaireId: z.string(), - questionIndex: z.number(), - answer: z.string().nullable(), - sources: z - .array( - z.object({ - sourceType: z.string(), - sourceName: z.string().optional(), - sourceId: z.string().optional(), - policyName: z.string().optional(), - documentName: z.string().optional(), - score: z.number(), - }), - ) - .optional(), - status: z.enum(['generated', 'manual']), -}); - -export const saveAnswerAction = authActionClient - .inputSchema(saveAnswerSchema) - .metadata({ - name: 'save-questionnaire-answer', - track: { - event: 'save-questionnaire-answer', - description: 'Save Questionnaire Answer', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { questionnaireId, questionIndex, answer, sources, status } = parsedInput; - const { activeOrganizationId } = ctx.session; - const userId = ctx.user.id; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - // Verify questionnaire exists and belongs to organization - const questionnaire = await db.questionnaire.findUnique({ - where: { - id: questionnaireId, - organizationId: activeOrganizationId, - }, - include: { - questions: { - where: { - questionIndex, - }, - }, - }, - }); - - if (!questionnaire) { - return { - success: false, - error: 'Questionnaire not found', - }; - } - - const existingQuestion = questionnaire.questions[0]; - - if (existingQuestion) { - // Update existing question answer - await db.questionnaireQuestionAnswer.update({ - where: { - id: existingQuestion.id, - }, - data: { - answer: answer || null, - status: status === 'generated' ? 'generated' : 'manual', - sources: sources ? (sources as any) : null, - generatedAt: status === 'generated' ? new Date() : null, - updatedBy: status === 'manual' ? userId || null : null, - updatedAt: new Date(), - }, - }); - - const shouldPersistManualAnswer = - status === 'manual' && answer && answer.trim().length > 0 && existingQuestion.question; - - if (shouldPersistManualAnswer) { - try { - const manualAnswer = await db.securityQuestionnaireManualAnswer.upsert({ - where: { - organizationId_question: { - organizationId: activeOrganizationId, - question: existingQuestion.question.trim(), - }, - }, - create: { - question: existingQuestion.question.trim(), - answer: answer.trim(), - tags: [], - organizationId: activeOrganizationId, - sourceQuestionnaireId: questionnaireId, - createdBy: userId || null, - updatedBy: userId || null, - }, - update: { - answer: answer.trim(), - sourceQuestionnaireId: questionnaireId, - updatedBy: userId || null, - updatedAt: new Date(), - }, - }); - - // Sync to vector DB SYNCHRONOUSLY - logger.info('🔄 Syncing manual answer to vector DB from save-answer', { - manualAnswerId: manualAnswer.id, - organizationId: activeOrganizationId, - questionIndex, - }); - - const syncResult = await syncManualAnswerToVector( - manualAnswer.id, - activeOrganizationId, - ); - - if (!syncResult.success) { - logger.error('❌ Failed to sync manual answer to vector DB', { - manualAnswerId: manualAnswer.id, - organizationId: activeOrganizationId, - error: syncResult.error, - }); - // Don't fail the main operation - manual answer is saved in DB - } else { - logger.info('✅ Successfully synced manual answer to vector DB', { - manualAnswerId: manualAnswer.id, - embeddingId: syncResult.embeddingId, - organizationId: activeOrganizationId, - }); - } - } catch (error) { - // Log error but don't fail the main operation - logger.error('Error saving to manual answers:', { - error: error instanceof Error ? error.message : 'Unknown error', - questionIndex, - organizationId: activeOrganizationId, - }); - } - } - } else { - // Create new question answer (shouldn't happen, but handle it) - await db.questionnaireQuestionAnswer.create({ - data: { - questionnaireId, - question: '', // Will be updated from parse results - questionIndex, - answer: answer || null, - status: status === 'generated' ? 'generated' : 'manual', - sources: sources ? (sources as any) : null, - generatedAt: status === 'generated' ? new Date() : null, - updatedBy: status === 'manual' ? userId || null : null, - }, - }); - } - - // Update answered questions count - const answeredCount = await db.questionnaireQuestionAnswer.count({ - where: { - questionnaireId, - answer: { - not: null, - }, - }, - }); - - await db.questionnaire.update({ - where: { - id: questionnaireId, - }, - data: { - answeredQuestions: answeredCount, - }, - }); - - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - - revalidatePath(path); - - return { - success: true, - }; - } catch (error) { - console.error('Error saving answer:', error); - return { - success: false, - error: 'Failed to save answer', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/save-answers-batch.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/save-answers-batch.ts deleted file mode 100644 index f36c5b5ed..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/save-answers-batch.ts +++ /dev/null @@ -1,155 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; - -const saveAnswersBatchSchema = z.object({ - questionnaireId: z.string(), - answers: z.array( - z.object({ - questionIndex: z.number(), - answer: z.string().nullable(), - sources: z - .array( - z.object({ - sourceType: z.string(), - sourceName: z.string().optional(), - sourceId: z.string().optional(), - policyName: z.string().optional(), - documentName: z.string().optional(), - score: z.number(), - }), - ) - .optional(), - status: z.enum(['generated', 'manual']), - }), - ), -}); - -export const saveAnswersBatchAction = authActionClient - .inputSchema(saveAnswersBatchSchema) - .metadata({ - name: 'save-questionnaire-answers-batch', - track: { - event: 'save-questionnaire-answers-batch', - description: 'Save Questionnaire Answers Batch', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { questionnaireId, answers } = parsedInput; - const { activeOrganizationId } = ctx.session; - const userId = ctx.user.id; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - // Verify questionnaire exists and belongs to organization - const questionnaire = await db.questionnaire.findUnique({ - where: { - id: questionnaireId, - organizationId: activeOrganizationId, - }, - }); - - if (!questionnaire) { - return { - success: false, - error: 'Questionnaire not found', - }; - } - - // Get all existing questions for this questionnaire - const existingQuestions = await db.questionnaireQuestionAnswer.findMany({ - where: { - questionnaireId, - }, - }); - - const existingQuestionsMap = new Map( - existingQuestions.map((q) => [q.questionIndex, q]), - ); - - // Update or create answers - const updatePromises = answers.map(async (answerData) => { - const existing = existingQuestionsMap.get(answerData.questionIndex); - - if (existing) { - // Update existing - return db.questionnaireQuestionAnswer.update({ - where: { - id: existing.id, - }, - data: { - answer: answerData.answer || null, - status: answerData.status === 'generated' ? 'generated' : 'manual', - sources: answerData.sources ? (answerData.sources as any) : null, - generatedAt: answerData.status === 'generated' ? new Date() : null, - updatedBy: answerData.status === 'manual' ? userId || null : null, - updatedAt: new Date(), - }, - }); - } else { - // Create new (shouldn't happen, but handle it) - return db.questionnaireQuestionAnswer.create({ - data: { - questionnaireId, - question: '', // Will be updated from parse results - questionIndex: answerData.questionIndex, - answer: answerData.answer || null, - status: answerData.status === 'generated' ? 'generated' : 'manual', - sources: answerData.sources ? (answerData.sources as any) : null, - generatedAt: answerData.status === 'generated' ? new Date() : null, - updatedBy: answerData.status === 'manual' ? userId || null : null, - }, - }); - } - }); - - await Promise.all(updatePromises); - - // Update answered questions count - const answeredCount = await db.questionnaireQuestionAnswer.count({ - where: { - questionnaireId, - answer: { - not: null, - }, - }, - }); - - await db.questionnaire.update({ - where: { - id: questionnaireId, - }, - data: { - answeredQuestions: answeredCount, - }, - }); - - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - - revalidatePath(path); - - return { - success: true, - }; - } catch (error) { - console.error('Error saving answers batch:', error); - return { - success: false, - error: 'Failed to save answers', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/update-questionnaire-answer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/update-questionnaire-answer.ts deleted file mode 100644 index 6589c3a10..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/update-questionnaire-answer.ts +++ /dev/null @@ -1,202 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; -import { syncManualAnswerToVector } from '@/lib/vector/sync/sync-manual-answer'; -import { logger } from '@/utils/logger'; - -const updateAnswerSchema = z.object({ - questionnaireId: z.string(), - questionAnswerId: z.string(), - answer: z.string(), - status: z.enum(['generated', 'manual']).optional().default('manual'), - sources: z - .array( - z.object({ - sourceType: z.string(), - sourceName: z.string().optional(), - sourceId: z.string().optional(), - policyName: z.string().optional(), - documentName: z.string().optional(), - score: z.number(), - }), - ) - .optional(), -}); - -export const updateQuestionnaireAnswer = authActionClient - .inputSchema(updateAnswerSchema) - .metadata({ - name: 'update-questionnaire-answer', - track: { - event: 'update-questionnaire-answer', - description: 'Update Questionnaire Answer', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { questionnaireId, questionAnswerId, answer, status, sources } = parsedInput; - const { activeOrganizationId } = ctx.session; - const userId = ctx.user.id; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - // Verify questionnaire exists and belongs to organization - const questionnaire = await db.questionnaire.findUnique({ - where: { - id: questionnaireId, - organizationId: activeOrganizationId, - }, - }); - - if (!questionnaire) { - return { - success: false, - error: 'Questionnaire not found', - }; - } - - // Verify question answer exists and belongs to questionnaire - const questionAnswer = await db.questionnaireQuestionAnswer.findUnique({ - where: { - id: questionAnswerId, - questionnaireId, - }, - }); - - if (!questionAnswer) { - return { - success: false, - error: 'Question answer not found', - }; - } - - // Store the previous status to determine if this was written from scratch - const previousStatus = questionAnswer.status; - - // Update the answer - await db.questionnaireQuestionAnswer.update({ - where: { - id: questionAnswerId, - }, - data: { - answer: answer.trim() || null, - status: status === 'generated' ? 'generated' : 'manual', - sources: sources ? (sources as any) : null, - generatedAt: status === 'generated' ? new Date() : null, - updatedBy: status === 'manual' ? userId || null : null, - updatedAt: new Date(), - }, - }); - - const shouldPersistManualAnswer = - status === 'manual' && answer && answer.trim().length > 0 && questionAnswer.question; - - if (shouldPersistManualAnswer) { - try { - const manualAnswer = await db.securityQuestionnaireManualAnswer.upsert({ - where: { - organizationId_question: { - organizationId: activeOrganizationId, - question: questionAnswer.question.trim(), - }, - }, - create: { - question: questionAnswer.question.trim(), - answer: answer.trim(), - tags: [], - organizationId: activeOrganizationId, - sourceQuestionnaireId: questionnaireId, - createdBy: userId || null, - updatedBy: userId || null, - }, - update: { - answer: answer.trim(), - sourceQuestionnaireId: questionnaireId, - updatedBy: userId || null, - updatedAt: new Date(), - }, - }); - - // Sync to vector DB SYNCHRONOUSLY - logger.info('🔄 Syncing manual answer to vector DB from questionnaire update', { - manualAnswerId: manualAnswer.id, - organizationId: activeOrganizationId, - questionId: questionAnswerId, - }); - - const syncResult = await syncManualAnswerToVector( - manualAnswer.id, - activeOrganizationId, - ); - - if (!syncResult.success) { - logger.error('❌ Failed to sync manual answer to vector DB', { - manualAnswerId: manualAnswer.id, - organizationId: activeOrganizationId, - error: syncResult.error, - }); - // Don't fail the main operation - manual answer is saved in DB - } else { - logger.info('✅ Successfully synced manual answer to vector DB', { - manualAnswerId: manualAnswer.id, - embeddingId: syncResult.embeddingId, - organizationId: activeOrganizationId, - }); - } - } catch (error) { - // Log error but don't fail the main operation - logger.error('Error saving to manual answers:', { - error: error instanceof Error ? error.message : 'Unknown error', - questionAnswerId, - organizationId: activeOrganizationId, - }); - } - } - - // Update answered questions count - const answeredCount = await db.questionnaireQuestionAnswer.count({ - where: { - questionnaireId, - answer: { - not: null, - }, - }, - }); - - await db.questionnaire.update({ - where: { - id: questionnaireId, - }, - data: { - answeredQuestions: answeredCount, - }, - }); - - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - - revalidatePath(path); - - return { - success: true, - }; - } catch (error) { - console.error('Error updating answer:', error); - return { - success: false, - error: 'Failed to update answer', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/upload-questionnaire-file.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/upload-questionnaire-file.ts deleted file mode 100644 index 6e1945c60..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/upload-questionnaire-file.ts +++ /dev/null @@ -1,134 +0,0 @@ -'use server'; - -import { APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET, s3Client } from '@/app/s3'; -import { authActionClient } from '@/actions/safe-action'; -import { PutObjectCommand } from '@aws-sdk/client-s3'; -import { db } from '@db'; -import { AttachmentEntityType, AttachmentType } from '@db'; -import { z } from 'zod'; -import { randomBytes } from 'crypto'; - -const uploadSchema = z.object({ - fileName: z.string(), - fileType: z.string(), - fileData: z.string(), // base64 encoded - organizationId: z.string(), -}); - -function mapFileTypeToAttachmentType(fileType: string): AttachmentType { - const type = fileType.split('/')[0]; - switch (type) { - case 'image': - return AttachmentType.image; - case 'video': - return AttachmentType.video; - case 'audio': - return AttachmentType.audio; - case 'application': - return AttachmentType.document; - default: - return AttachmentType.other; - } -} - -export const uploadQuestionnaireFile = authActionClient - .inputSchema(uploadSchema) - .metadata({ - name: 'upload-questionnaire-file', - track: { - event: 'upload-questionnaire-file', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { fileName, fileType, fileData, organizationId } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId || session.activeOrganizationId !== organizationId) { - throw new Error('Unauthorized'); - } - - if (!s3Client) { - throw new Error('S3 client not configured'); - } - - if (!APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET) { - throw new Error('Questionnaire upload bucket is not configured. Please set APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET environment variable.'); - } - - try { - // Convert base64 to buffer - const fileBuffer = Buffer.from(fileData, 'base64'); - - // Validate file size (10MB limit) - const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; - if (fileBuffer.length > MAX_FILE_SIZE_BYTES) { - throw new Error(`File exceeds the ${MAX_FILE_SIZE_BYTES / (1024 * 1024)}MB limit`); - } - - // Generate unique file key - const fileId = randomBytes(16).toString('hex'); - const sanitizedFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_'); - const timestamp = Date.now(); - const s3Key = `${organizationId}/questionnaire-uploads/${timestamp}-${fileId}-${sanitizedFileName}`; - - // Upload to S3 - const putCommand = new PutObjectCommand({ - Bucket: APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET, - Key: s3Key, - Body: fileBuffer, - ContentType: fileType, - Metadata: { - originalFileName: fileName, - organizationId, - }, - }); - - await s3Client.send(putCommand); - - // Return S3 key directly instead of creating attachment record - // Questionnaire files are temporary processing files, not permanent attachments - return { - success: true, - data: { - s3Key, - fileName, - fileType, - }, - }; - } catch (error) { - // Provide more helpful error messages for common S3 errors - if (error && typeof error === 'object' && 'Code' in error) { - const awsError = error as { Code: string; message?: string }; - - if (awsError.Code === 'AccessDenied') { - throw new Error( - `Access denied to S3 bucket "${APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET}". ` + - `Please verify that:\n` + - `1. The bucket "${APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET}" exists\n` + - `2. Your AWS credentials have s3:PutObject permission for this bucket\n` + - `3. The bucket is in the correct region (${process.env.APP_AWS_REGION || 'not set'})\n` + - `4. The bucket name is correct` - ); - } - - if (awsError.Code === 'NoSuchBucket') { - throw new Error( - `S3 bucket "${APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET}" does not exist. ` + - `Please create the bucket or update APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET environment variable.` - ); - } - - if (awsError.Code === 'InvalidAccessKeyId' || awsError.Code === 'SignatureDoesNotMatch') { - throw new Error( - `Invalid AWS credentials. Please check APP_AWS_ACCESS_KEY_ID and APP_AWS_SECRET_ACCESS_KEY environment variables.` - ); - } - } - - throw error instanceof Error - ? error - : new Error('Failed to upload questionnaire file'); - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/vendor-questionnaire-orchestrator.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/actions/vendor-questionnaire-orchestrator.ts deleted file mode 100644 index 9d0c6d474..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/actions/vendor-questionnaire-orchestrator.ts +++ /dev/null @@ -1,136 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { answerQuestion } from '@/jobs/tasks/vendors/answer-question'; -import { syncOrganizationEmbeddings } from '@/lib/vector'; -import { logger } from '@/utils/logger'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; - -const inputSchema = z.object({ - questionsAndAnswers: z.array( - z.object({ - question: z.string(), - answer: z.string().nullable(), - _originalIndex: z.number().optional(), // Preserves original index from QuestionnaireResult - }), - ), -}); - -export const vendorQuestionnaireOrchestrator = authActionClient - .inputSchema(inputSchema) - .metadata({ - name: 'vendor-questionnaire-orchestrator', - track: { - event: 'vendor-questionnaire-orchestrator', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { questionsAndAnswers } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId) { - throw new Error('No active organization'); - } - - const organizationId = session.activeOrganizationId; - - try { - logger.info('Starting auto-answer questionnaire', { - organizationId, - questionCount: questionsAndAnswers.length, - }); - - // Sync organization embeddings before generating answers - // Uses incremental sync: only updates what changed (much faster than full sync) - try { - await syncOrganizationEmbeddings(organizationId); - logger.info('Organization embeddings synced successfully', { - organizationId, - }); - } catch (error) { - logger.warn('Failed to sync organization embeddings', { - organizationId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - // Continue with existing embeddings if sync fails - } - - // Filter questions that need answers (skip already answered) - // Preserve original index if provided (for single question answers) - const questionsToAnswer = questionsAndAnswers - .map((qa, index) => ({ - ...qa, - index: qa._originalIndex !== undefined ? qa._originalIndex : index, - })) - .filter((qa) => !qa.answer || qa.answer.trim().length === 0); - - logger.info('Questions to answer', { - total: questionsAndAnswers.length, - toAnswer: questionsToAnswer.length, - }); - - // Process all questions in parallel by calling answerQuestion directly - // Note: metadata updates are disabled since we're not in a Trigger.dev task context - const results = await Promise.all( - questionsToAnswer.map((qa) => - answerQuestion( - { - question: qa.question, - organizationId, - questionIndex: qa.index, - totalQuestions: questionsAndAnswers.length, - }, - { useMetadata: false }, - ), - ), - ); - - // Process results - const allAnswers: Array<{ - questionIndex: number; - question: string; - answer: string | null; - sources?: Array<{ - sourceType: string; - sourceName?: string; - score: number; - }>; - }> = results.map((result) => ({ - questionIndex: result.questionIndex, - question: result.question, - answer: result.answer, - sources: result.sources, - })); - - logger.info('Auto-answer questionnaire completed', { - organizationId, - totalQuestions: questionsAndAnswers.length, - answered: allAnswers.filter((a) => a.answer).length, - }); - - // Revalidate the page to show updated answers - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - revalidatePath(path); - - return { - success: true, - data: { - answers: allAnswers, - }, - }; - } catch (error) { - logger.error('Failed to answer questions', { - organizationId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - throw error instanceof Error - ? error - : new Error('Failed to answer questions'); - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/components/KnowledgeBaseDocumentLink.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/components/KnowledgeBaseDocumentLink.tsx index 9ce4ecb64..55817c3e9 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/components/KnowledgeBaseDocumentLink.tsx +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/components/KnowledgeBaseDocumentLink.tsx @@ -2,7 +2,7 @@ import { LinkIcon, Loader2 } from 'lucide-react'; import { useState } from 'react'; -import { getKnowledgeBaseDocumentViewUrlAction } from '../knowledge-base/additional-documents/actions/get-document-view-url'; +import { api } from '@/lib/api-client'; interface KnowledgeBaseDocumentLinkProps { documentId: string; @@ -25,12 +25,28 @@ export function KnowledgeBaseDocumentLink({ setIsLoading(true); try { - const result = await getKnowledgeBaseDocumentViewUrlAction({ - documentId, - }); + const response = await api.post<{ + signedUrl: string; + fileName: string; + fileType: string; + viewableInBrowser: boolean; + }>( + `/v1/knowledge-base/documents/${documentId}/view`, + { + organizationId: orgId, + }, + orgId, + ); - if (result?.data?.success && result.data.data) { - const { signedUrl, viewableInBrowser } = result.data.data; + if (response.error) { + // Fallback: navigate to knowledge base page + const knowledgeBaseUrl = `/${orgId}/questionnaire/knowledge-base`; + window.open(knowledgeBaseUrl, '_blank', 'noopener,noreferrer'); + return; + } + + if (response.data) { + const { signedUrl, viewableInBrowser } = response.data; if (viewableInBrowser && signedUrl) { // File can be viewed in browser - open it directly diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/usePersistGeneratedAnswers.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/usePersistGeneratedAnswers.ts deleted file mode 100644 index 58d7ac3e7..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/usePersistGeneratedAnswers.ts +++ /dev/null @@ -1,341 +0,0 @@ -"use client"; - -import { type Dispatch, type SetStateAction, useEffect, useRef } from 'react'; -import type { QuestionAnswer } from '../components/types'; - -type PersistedQuestionAnswer = QuestionAnswer & { - originalIndex?: number; - questionAnswerId?: string; - status?: 'untouched' | 'generated' | 'manual'; -}; - -type UpdateAnswerAction = { - execute: (...args: any[]) => unknown; - executeAsync: (...args: any[]) => Promise; -}; - -interface UsePersistGeneratedAnswersParams { - questionnaireId: string | null; - results: TResults; - setResults: Dispatch>; - autoAnswerRun: { - metadata?: Record; - status?: string; - output?: unknown; - } | null; - updateAnswerAction: UpdateAnswerAction; - setQuestionStatuses: React.Dispatch< - React.SetStateAction> - >; -} - -export function usePersistGeneratedAnswers({ - questionnaireId, - results, - setResults, - autoAnswerRun, - updateAnswerAction, - setQuestionStatuses, -}: UsePersistGeneratedAnswersParams) { - const processedMetadataAnswersRef = useRef>(new Set()); - const pendingMetadataUpdatesRef = useRef< - Map - >(new Map()); - const metadataUpdateTimeoutRef = useRef(null); - const updateQueueRef = useRef>(Promise.resolve()); - const resultsRef = useRef(results ?? []); - const previousResultsRef = useRef(results ?? []); - const processedResultsRef = useRef>(new Set()); - const resultsUpdateTimeoutRef = useRef(null); - const pendingResultsUpdatesRef = useRef< - Map - >(new Map()); - const pendingUpdatesWaitingForIdRef = useRef< - Map - >(new Map()); - - useEffect(() => { - resultsRef.current = results ?? []; - }, [results]); - - useEffect(() => { - previousResultsRef.current = results ?? []; - }, [results]); - - useEffect(() => { - return () => { - if (metadataUpdateTimeoutRef.current) { - clearTimeout(metadataUpdateTimeoutRef.current); - } - if (resultsUpdateTimeoutRef.current) { - clearTimeout(resultsUpdateTimeoutRef.current); - } - }; - }, []); - - const enqueueUpdate = ( - key: string, - payload: { questionAnswerId: string; answer: string; sources?: any[] }, - onError: () => void, - ) => { - if (!questionnaireId) { - return; - } - - updateQueueRef.current = updateQueueRef.current - .catch(() => { - // Swallow previous error to keep queue progressing - }) - .then(async () => { - try { - await updateAnswerAction.executeAsync({ - questionnaireId, - questionAnswerId: payload.questionAnswerId, - answer: payload.answer, - sources: payload.sources, - status: 'generated', - }); - console.log('Successfully updated answer in database:', { - questionAnswerId: payload.questionAnswerId, - }); - } catch (error) { - console.error('Failed to update answer in database', error); - onError(); - } - - await new Promise((resolve) => setTimeout(resolve, 200)); - }); - }; - - useEffect(() => { - if (!questionnaireId || !resultsRef.current.length || !autoAnswerRun?.metadata) { - return; - } - - const meta = autoAnswerRun.metadata as Record; - // Exclude _sources keys - they are handled separately - const answerKeys = Object.keys(meta).filter((key) => - key.startsWith('answer_') && !key.endsWith('_sources') - ); - - answerKeys.forEach((key) => { - if (processedMetadataAnswersRef.current.has(key)) { - return; - } - - const answerData = meta[key] as { - questionIndex?: number; - answer?: string | null; - sources?: any[]; - }; - - if (!answerData || answerData.questionIndex === undefined || !answerData.answer) { - return; - } - - // Sources are included in answerData.sources - const sourcesToUse = answerData.sources || []; - - const resultMatch = resultsRef.current.find((r) => r.originalIndex === answerData.questionIndex); - - if (!resultMatch?.questionAnswerId) { - pendingUpdatesWaitingForIdRef.current.set(answerData.questionIndex, { - answer: answerData.answer || '', - sources: sourcesToUse, - }); - return; - } - - processedMetadataAnswersRef.current.add(key); - pendingMetadataUpdatesRef.current.set(key, { - questionAnswerId: resultMatch.questionAnswerId, - answer: answerData.answer || '', - sources: sourcesToUse, - }); - }); - - if (metadataUpdateTimeoutRef.current) { - clearTimeout(metadataUpdateTimeoutRef.current); - } - - metadataUpdateTimeoutRef.current = setTimeout(() => { - const updates = Array.from(pendingMetadataUpdatesRef.current.entries()); - pendingMetadataUpdatesRef.current.clear(); - - updates.forEach(([key, update]) => { - enqueueUpdate(key, update, () => { - processedMetadataAnswersRef.current.delete(key); - }); - }); - }, 500); - }, [autoAnswerRun?.metadata, questionnaireId]); - - useEffect(() => { - if (!questionnaireId || !results?.length) { - return; - } - - results.forEach((result) => { - if (result.originalIndex == null) return; - - const pendingForIndex = pendingUpdatesWaitingForIdRef.current.get(result.originalIndex); - - if (pendingForIndex && result.questionAnswerId) { - const answerKey = `${result.questionAnswerId}-${result.originalIndex}`; - if (!processedResultsRef.current.has(answerKey)) { - processedResultsRef.current.add(answerKey); - pendingResultsUpdatesRef.current.set(answerKey, { - questionAnswerId: result.questionAnswerId, - answer: pendingForIndex.answer, - sources: pendingForIndex.sources, - }); - } - pendingUpdatesWaitingForIdRef.current.delete(result.originalIndex); - } - - if (!result.questionAnswerId) return; - - const prevResult = previousResultsRef.current.find((r) => r.originalIndex === result.originalIndex); - const answerKey = `${result.questionAnswerId}-${result.originalIndex}`; - - if (processedResultsRef.current.has(answerKey)) { - return; - } - - if ( - result.answer && - result.answer.trim().length > 0 && - result.status !== 'manual' && - (!prevResult || prevResult.answer !== result.answer) - ) { - processedResultsRef.current.add(answerKey); - // Use sources from current result - preserve existing sources if new ones are empty - // This prevents losing sources when updating answers incrementally - // Always pass an array (empty or with sources) to avoid undefined issues - const sourcesToSave = result.sources && result.sources.length > 0 - ? result.sources - : (prevResult?.sources && prevResult.sources.length > 0 ? prevResult.sources : []); - - pendingResultsUpdatesRef.current.set(answerKey, { - questionAnswerId: result.questionAnswerId, - answer: result.answer, - sources: sourcesToSave, - }); - } - }); - - if (resultsUpdateTimeoutRef.current) { - clearTimeout(resultsUpdateTimeoutRef.current); - } - - resultsUpdateTimeoutRef.current = setTimeout(() => { - const updates = Array.from(pendingResultsUpdatesRef.current.entries()); - pendingResultsUpdatesRef.current.clear(); - - updates.forEach(([answerKey, update]) => { - enqueueUpdate(answerKey, update, () => { - processedResultsRef.current.delete(answerKey); - }); - }); - }, 500); - }, [questionnaireId, results]); - - useEffect(() => { - if (!autoAnswerRun?.metadata || !resultsRef.current.length) { - return; - } - - const meta = autoAnswerRun.metadata as Record; - // Exclude _sources keys - they are handled separately - const answerKeys = Object.keys(meta).filter((key) => - key.startsWith('answer_') && !key.endsWith('_sources') - ); - - if (!answerKeys.length) { - return; - } - - const answers = answerKeys - .map((key) => { - const answer = meta[key] as { - questionIndex: number; - question: string; - answer: string | null; - sources?: Array<{ - sourceType: string; - sourceName?: string; - score: number; - }>; - }; - - return answer; - }) - .filter((answer): answer is NonNullable => Boolean(answer)) - .sort((a, b) => a.questionIndex - b.questionIndex); - - if (!answers.length) { - return; - } - - setResults((prevResults) => { - if (!prevResults) { - return prevResults; - } - - const updatedResults = [...prevResults]; - let hasChanges = false; - - answers.forEach((answer) => { - const targetIndex = updatedResults.findIndex( - (r) => r.originalIndex === answer.questionIndex, - ); - - if (targetIndex < 0 || targetIndex >= updatedResults.length) { - return; - } - - const currentAnswer = updatedResults[targetIndex]?.answer; - const originalQuestion = updatedResults[targetIndex]?.question; - - if (answer.answer) { - if (currentAnswer !== answer.answer) { - updatedResults[targetIndex] = { - ...updatedResults[targetIndex], - question: originalQuestion || answer.question, - answer: answer.answer, - sources: answer.sources, - failedToGenerate: false, - status: - updatedResults[targetIndex]?.status === 'manual' - ? 'manual' - : 'generated', - }; - hasChanges = true; - - const statusKey = updatedResults[targetIndex]?.originalIndex ?? targetIndex; - setQuestionStatuses((prevStatuses) => { - const newStatuses = new Map(prevStatuses); - if (prevStatuses.get(statusKey) !== 'completed') { - newStatuses.set(statusKey, 'completed'); - return newStatuses; - } - return prevStatuses; - }); - } - } else if (!currentAnswer) { - updatedResults[targetIndex] = { - ...updatedResults[targetIndex], - question: originalQuestion || answer.question, - answer: null, - failedToGenerate: true, - }; - hasChanges = true; - } - }); - - return (hasChanges ? updatedResults : prevResults) as TResults; - }); - }, [autoAnswerRun?.metadata, setQuestionStatuses, setResults]); -} - - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireActions.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireActions.ts index 7685ffa05..ae1fb7f4e 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireActions.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireActions.ts @@ -1,12 +1,12 @@ 'use client'; -import { useAction } from 'next-safe-action/hooks'; -import { useCallback, useEffect, useRef, useTransition } from 'react'; +import { useCallback, useEffect, useRef, useState, useTransition } from 'react'; import type { FileRejection } from 'react-dropzone'; import { toast } from 'sonner'; -import { exportQuestionnaire } from '../actions/export-questionnaire'; -import { saveAnswerAction } from '../actions/save-answer'; import type { QuestionAnswer } from '../components/types'; +import { api } from '@/lib/api-client'; +import { env } from '@/env.mjs'; +import { jwtManager } from '@/utils/jwt-manager'; interface UseQuestionnaireActionsProps { orgId: string; @@ -34,22 +34,23 @@ interface UseQuestionnaireActionsProps { setParseToken: (token: string | null) => void; uploadFileAction: { execute: (payload: any) => void; - status: 'idle' | 'executing' | 'hasSucceeded' | 'hasErrored' | 'transitioning' | 'hasNavigated'; + status: string; }; parseAction: { execute: (payload: any) => void; - status: 'idle' | 'executing' | 'hasSucceeded' | 'hasErrored' | 'transitioning' | 'hasNavigated'; + status: string; }; triggerAutoAnswer: (payload: { - vendorId: string; organizationId: string; questionsAndAnswers: QuestionAnswer[]; + questionnaireId?: string | null; }) => void; triggerSingleAnswer: (payload: { question: string; organizationId: string; questionIndex: number; totalQuestions: number; + questionnaireId?: string | null; }) => void; } @@ -80,40 +81,8 @@ export function useQuestionnaireActions({ triggerAutoAnswer, triggerSingleAnswer, }: UseQuestionnaireActionsProps) { - const saveAnswer = useAction(saveAnswerAction, { - onSuccess: () => { - // Answer saved successfully - }, - onError: ({ error }) => { - console.error('Error saving answer:', error); - // Don't show toast for every save - too noisy - }, - }); - const [isPending, startTransition] = useTransition(); - - const exportAction = useAction(exportQuestionnaire, { - onSuccess: ({ data }: { data: any }) => { - const responseData = data?.data || data; - const filename = responseData?.filename; - const downloadUrl = responseData?.downloadUrl; - - if (downloadUrl && filename) { - const link = document.createElement('a'); - link.href = downloadUrl; - link.download = filename; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - - toast.success(`Exported as ${filename}`); - } - }, - onError: ({ error }) => { - console.error('Export action error:', error); - toast.error(error.serverError || 'Failed to export questionnaire'); - }, - }); + const [isExporting, setIsExporting] = useState(false); const handleFileSelect = useCallback((acceptedFiles: File[], rejectedFiles: FileRejection[]) => { if (rejectedFiles.length > 0) { @@ -240,9 +209,9 @@ export function useQuestionnaireActions({ // Then trigger the actual task // Real metadata updates will refine the statuses as tasks actually start/complete triggerAutoAnswer({ - vendorId: `org_${orgId}`, organizationId: orgId, questionsAndAnswers: results, + questionnaireId, }); }; @@ -272,6 +241,7 @@ export function useQuestionnaireActions({ organizationId: orgId, questionIndex: index, totalQuestions: results.length, + questionnaireId, }); }; @@ -294,13 +264,23 @@ export function useQuestionnaireActions({ setEditingAnswer(''); // Save to database (use startTransition to avoid rendering issues) - startTransition(() => { - saveAnswer.execute({ + startTransition(async () => { + const response = await api.post( + '/v1/questionnaire/save-answer', + { questionnaireId, questionIndex: index, answer: answerText, status: 'manual', - }); + organizationId: orgId, + }, + orgId, + ); + + if (response.error) { + console.error('Error saving answer:', response.error); + toast.error('Failed to save answer'); + } }); toast.success('Answer updated'); @@ -312,15 +292,68 @@ export function useQuestionnaireActions({ }; const handleExport = async (format: 'xlsx' | 'csv' | 'pdf') => { - if (!results || results.length === 0) { - toast.error('No data to export'); + if (!questionnaireId) { + toast.error('No questionnaire to export'); return; } - await exportAction.execute({ - questionsAndAnswers: results, + setIsExporting(true); + + try { + // Get auth token for the request + const token = await jwtManager.getValidToken(); + + // Call the API to get the file as a blob + const response = await fetch( + `${env.NEXT_PUBLIC_API_URL || 'http://localhost:3333'}/v1/questionnaire/export`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + 'X-Organization-Id': orgId, + }, + body: JSON.stringify({ + questionnaireId, + organizationId: orgId, format, - }); + }), + }, + ); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.message || 'Failed to export questionnaire'); + } + + // Get filename from Content-Disposition header + const contentDisposition = response.headers.get('Content-Disposition'); + let filename = `questionnaire.${format}`; + if (contentDisposition) { + const match = contentDisposition.match(/filename="(.+)"/); + if (match) { + filename = match[1]; + } + } + + // Download the file + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + window.URL.revokeObjectURL(url); + + toast.success(`Exported as ${filename}`); + } catch (error) { + console.error('Export error:', error); + toast.error(error instanceof Error ? error.message : 'Failed to export questionnaire'); + } finally { + setIsExporting(false); + } }; const handleToggleSource = (index: number) => { @@ -333,6 +366,11 @@ export function useQuestionnaireActions({ setExpandedSources(newExpanded); }; + // Simulated exportAction for backward compatibility + const exportAction = { + status: isExporting ? 'executing' : 'idle', + }; + return { exportAction, handleFileSelect, diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireAutoAnswer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireAutoAnswer.ts index ab8219965..e2e5b82ad 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireAutoAnswer.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireAutoAnswer.ts @@ -1,10 +1,10 @@ 'use client'; -import { saveAnswersBatchAction } from '../actions/save-answers-batch'; -import { useAction } from 'next-safe-action/hooks'; -import { useTransition, useEffect, useMemo, useRef, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import { toast } from 'sonner'; import type { QuestionAnswer } from '../components/types'; +import { env } from '@/env.mjs'; +import { jwtManager } from '@/utils/jwt-manager'; interface UseQuestionnaireAutoAnswerProps { results: QuestionAnswer[] | null; @@ -35,21 +35,14 @@ export function useQuestionnaireAutoAnswer({ const [autoAnswerError, setAutoAnswerError] = useState(null); const completedAnswersRef = useRef>(new Set()); - // Action for saving answers batch - const saveAnswersBatch = useAction(saveAnswersBatchAction, { - onError: ({ error }) => { - console.error('Error saving answers batch:', error); - }, - }); - - const [isPending, startTransition] = useTransition(); - const triggerAutoAnswer = async (payload: { - vendorId: string; organizationId: string; + questionnaireId?: string | null; questionsAndAnswers: Array<{ question: string; answer: string | null; + _originalIndex?: number; + originalIndex?: number; }>; }) => { // Reset state @@ -85,16 +78,27 @@ export function useQuestionnaireAutoAnswer({ try { // Use fetch with ReadableStream for SSE (EventSource only supports GET) // credentials: 'include' is required to send cookies for authentication - const response = await fetch('/api/questionnaire/auto-answer', { + const token = await jwtManager.getValidToken(); + const response = await fetch( + `${env.NEXT_PUBLIC_API_URL || 'http://localhost:3333'}/v1/questionnaire/auto-answer`, + { method: 'POST', headers: { 'Content-Type': 'application/json', - }, - credentials: 'include', // Include cookies for authentication + ...(token ? { Authorization: `Bearer ${token}` } : {}), + 'X-Organization-Id': payload.organizationId, + }, body: JSON.stringify({ - questionsAndAnswers: payload.questionsAndAnswers, - }), - }); + organizationId: payload.organizationId, + questionnaireId: payload.questionnaireId ?? questionnaireId, + questionsAndAnswers: payload.questionsAndAnswers.map((qa, index) => ({ + question: qa.question, + answer: qa.answer ?? null, + _originalIndex: qa._originalIndex ?? qa.originalIndex ?? index, + })), + }), + }, + ); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); @@ -201,32 +205,6 @@ export function useQuestionnaireAutoAnswer({ setIsAutoAnswerProcessStarted(false); setAnsweringQuestionIndex(null); - // Save all answers in batch - if (questionnaireId && data.answers) { - const answersToSave = data.answers - .map((answer: any) => { - if (answer.answer) { - return { - questionIndex: answer.questionIndex, - answer: answer.answer, - sources: answer.sources || [], - status: 'generated' as const, - }; - } - return null; - }) - .filter((a: any): a is NonNullable => a !== null); - - if (answersToSave.length > 0) { - startTransition(() => { - saveAnswersBatch.execute({ - questionnaireId, - answers: answersToSave, - }); - }); - } - } - // Show final toast const totalQuestions = data.total; const answeredQuestions = data.answered; diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetail.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetail.ts index 623b0bbae..c26424cb3 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetail.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetail.ts @@ -5,7 +5,6 @@ import { useQuestionnaireActions } from '../useQuestionnaireActions'; import { useQuestionnaireAutoAnswer } from '../useQuestionnaireAutoAnswer'; import { useQuestionnaireSingleAnswer } from '../useQuestionnaireSingleAnswer'; import type { QuestionAnswer } from '../../components/types'; -import { usePersistGeneratedAnswers } from '../usePersistGeneratedAnswers'; import { useQuestionnaireDetailState } from './useQuestionnaireDetailState'; import { useQuestionnaireDetailHandlers } from './useQuestionnaireDetailHandlers'; import type { UseQuestionnaireDetailProps } from './types'; @@ -36,7 +35,7 @@ export function useQuestionnaireDetail({ }); // Wrapper for setResults that handles QuestionnaireResult[] with originalIndex - const setResultsWrapper = useCallback((updater: React.SetStateAction) => { + const setResultsWrapper = useCallback((updater: SetStateAction) => { state.setResults((prevResults) => { if (!prevResults) { const newResults = typeof updater === 'function' ? updater(null) : updater; @@ -140,23 +139,6 @@ export function useQuestionnaireDetail({ triggerSingleAnswer: singleAnswer.triggerSingleAnswer, }); - const persistenceAction = { - execute: () => {}, - executeAsync: (input: Parameters[0]) => - state.updateAnswerAction.executeAsync(input), - }; - - usePersistGeneratedAnswers({ - questionnaireId, - results: state.results as QuestionAnswer[] | null, - setResults: state.setResults as Dispatch>, - autoAnswerRun: autoAnswer.autoAnswerRun ?? null, - updateAnswerAction: persistenceAction as any, - setQuestionStatuses: state.setQuestionStatuses as Dispatch< - SetStateAction> - >, - }); - // Handlers const handlers = useQuestionnaireDetailHandlers({ questionnaireId, @@ -173,10 +155,7 @@ export function useQuestionnaireDetail({ editingAnswer: state.editingAnswer, setEditingIndex: state.setEditingIndex, setEditingAnswer: state.setEditingAnswer, - saveIndexRef: state.saveIndexRef, - saveAnswerRef: state.saveAnswerRef, - updateAnswerAction: state.updateAnswerAction, - deleteAnswerAction: state.deleteAnswerAction, + setSavingIndex: state.setSavingIndex, router: state.router, triggerAutoAnswer: autoAnswer.triggerAutoAnswer, triggerSingleAnswer: singleAnswer.triggerSingleAnswer, @@ -235,9 +214,8 @@ export function useQuestionnaireDetail({ autoAnswer.isAutoAnswerTriggering, ]); - const isSaving = state.updateAnswerAction.status === 'executing'; - const savingIndex = - isSaving && state.saveIndexRef.current !== null ? state.saveIndexRef.current : null; + const isSaving = state.savingIndex !== null; + const savingIndex = state.savingIndex; return { orgId: organizationId, diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailHandlers.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailHandlers.ts index 52f50a7fc..295ee4b84 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailHandlers.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailHandlers.ts @@ -1,7 +1,8 @@ 'use client'; -import { useCallback, useEffect } from 'react'; +import { useCallback, useEffect, type MutableRefObject } from 'react'; import { toast } from 'sonner'; +import { api } from '@/lib/api-client'; import type { QuestionnaireResult } from './types'; import type { Dispatch, SetStateAction } from 'react'; @@ -11,7 +12,7 @@ interface UseQuestionnaireDetailHandlersProps { results: QuestionnaireResult[]; answeringQuestionIndex: number | null; isAutoAnswerProcessStarted: boolean; - isAutoAnswerProcessStartedRef: React.MutableRefObject; + isAutoAnswerProcessStartedRef: MutableRefObject; setHasClickedAutoAnswer: (clicked: boolean) => void; setIsAutoAnswerProcessStarted: (started: boolean) => void; setAnsweringQuestionIndex: (index: number | null) => void; @@ -22,35 +23,23 @@ interface UseQuestionnaireDetailHandlersProps { editingAnswer: string; setEditingIndex: (index: number | null) => void; setEditingAnswer: (answer: string) => void; - saveIndexRef: React.MutableRefObject; - saveAnswerRef: React.MutableRefObject; - updateAnswerAction: { - execute: (payload: { - questionnaireId: string; - questionAnswerId: string; - answer: string; - }) => void; - }; - deleteAnswerAction: { - execute: ( - payload: { questionnaireId: string; questionAnswerId: string } - ) => Promise | void; - }; + setSavingIndex: (index: number | null) => void; router: { refresh: () => void }; triggerAutoAnswer: (payload: { - vendorId: string; organizationId: string; questionsAndAnswers: any[]; + questionnaireId?: string; }) => void; triggerSingleAnswer: (payload: { question: string; organizationId: string; questionIndex: number; totalQuestions: number; + questionnaireId?: string; }) => void; answerQueue: number[]; setAnswerQueue: Dispatch>; - answerQueueRef: React.MutableRefObject; + answerQueueRef: MutableRefObject; } export function useQuestionnaireDetailHandlers({ @@ -68,10 +57,7 @@ export function useQuestionnaireDetailHandlers({ editingAnswer, setEditingIndex, setEditingAnswer, - saveIndexRef, - saveAnswerRef, - updateAnswerAction, - deleteAnswerAction, + setSavingIndex, router, triggerAutoAnswer, triggerSingleAnswer, @@ -119,8 +105,8 @@ export function useQuestionnaireDetailHandlers({ try { triggerAutoAnswer({ - vendorId: `org_${organizationId}`, organizationId, + questionnaireId, questionsAndAnswers: questionsToAnswer.map((q) => ({ question: q.question, answer: q.answer, @@ -182,9 +168,10 @@ export function useQuestionnaireDetailHandlers({ organizationId, questionIndex: nextIndex, totalQuestions: results.length, + questionnaireId, }); }); - }, [results, organizationId, setAnswerQueue, setQuestionStatuses, triggerSingleAnswer]); + }, [results, organizationId, questionnaireId, setAnswerQueue, setQuestionStatuses, triggerSingleAnswer]); const handleAnswerSingleQuestion = (index: number) => { // Don't allow adding to queue if batch operation is running @@ -225,6 +212,7 @@ export function useQuestionnaireDetailHandlers({ organizationId, questionIndex: index, totalQuestions: results.length, + questionnaireId, }); }; @@ -237,19 +225,28 @@ export function useQuestionnaireDetailHandlers({ const handleDeleteAnswer = async (questionAnswerId: string, questionIndex: number) => { try { - await Promise.resolve( - deleteAnswerAction.execute({ + const response = await api.post( + '/v1/questionnaire/delete-answer', + { questionnaireId, questionAnswerId, - }) + organizationId, + }, + organizationId, ); + if (response.error) { + console.error('Failed to delete answer:', response.error); + toast.error(response.error || 'Failed to delete answer'); + return; + } + setResults((prev) => prev.map((r) => r.originalIndex === questionIndex - ? { ...r, answer: '', status: 'untouched' as const } - : r - ) + ? { ...r, answer: '', status: 'untouched' as const, sources: [] } + : r, + ), ); toast.success('Answer deleted. You can now generate a new answer.'); @@ -260,7 +257,7 @@ export function useQuestionnaireDetailHandlers({ } }; - const handleSaveAnswer = (index: number) => { + const handleSaveAnswer = async (index: number) => { const result = results[index]; if (!result) { @@ -283,14 +280,53 @@ export function useQuestionnaireDetailHandlers({ return; } - saveIndexRef.current = index; - saveAnswerRef.current = editingAnswer; + const trimmedAnswer = editingAnswer.trim(); + setSavingIndex(index); - updateAnswerAction.execute({ + try { + const response = await api.post( + '/v1/questionnaire/save-answer', + { questionnaireId, questionAnswerId: result.questionAnswerId, - answer: editingAnswer.trim(), - }); + organizationId, + answer: trimmedAnswer, + status: 'manual', + questionIndex: result.originalIndex, + }, + organizationId, + ); + + if (response.error) { + console.error('Failed to save answer:', response.error); + toast.error(response.error || 'Failed to save answer'); + return; + } + + setResults((prev) => + prev.map((r, i) => { + if (i === index) { + return { + ...r, + answer: trimmedAnswer || null, + status: trimmedAnswer ? ('manual' as const) : ('untouched' as const), + failedToGenerate: false, + sources: r.sources || [], + }; + } + return r; + }), + ); + + setEditingIndex(null); + setEditingAnswer(''); + toast.success('Answer saved'); + } catch (error) { + console.error('Failed to save answer:', error); + toast.error('Failed to save answer'); + } finally { + setSavingIndex(null); + } }; return { diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailState.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailState.ts index 3bc996059..f809c206f 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailState.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireDetail/useQuestionnaireDetailState.ts @@ -1,11 +1,7 @@ 'use client'; import { useEffect, useRef, useState } from 'react'; -import { useAction } from 'next-safe-action/hooks'; import { useRouter } from 'next/navigation'; -import { updateQuestionnaireAnswer } from '../../actions/update-questionnaire-answer'; -import { deleteQuestionnaireAnswer } from '../../actions/delete-questionnaire-answer'; -import { createTriggerToken } from '../../actions/create-trigger-token'; import type { QuestionnaireResult, QuestionnaireQuestionAnswer } from './types'; interface UseQuestionnaireDetailStateProps { @@ -57,52 +53,7 @@ export function useQuestionnaireDetailState({ const [answerQueue, setAnswerQueue] = useState([]); const answerQueueRef = useRef([]); - // Refs to capture values for save callback - const saveIndexRef = useRef(null); - const saveAnswerRef = useRef(''); - - // Actions - const updateAnswerAction = useAction(updateQuestionnaireAnswer, { - onSuccess: () => { - if (saveIndexRef.current !== null) { - const index = saveIndexRef.current; - const answer = saveAnswerRef.current; - - setResults((prev) => - prev.map((r, i) => { - if (i === index) { - const trimmedAnswer = answer.trim(); - return { - ...r, - answer: trimmedAnswer || null, - status: trimmedAnswer ? ('manual' as const) : ('untouched' as const), - failedToGenerate: false, - // Preserve sources when manually editing answer - sources: r.sources || [], - }; - } - return r; - }) - ); - - setEditingIndex(null); - setEditingAnswer(''); - router.refresh(); - - saveIndexRef.current = null; - saveAnswerRef.current = ''; - } - }, - onError: ({ error }) => { - console.error('Failed to update answer:', error); - if (saveIndexRef.current !== null) { - saveIndexRef.current = null; - saveAnswerRef.current = ''; - } - }, - }); - - const deleteAnswerAction = useAction(deleteQuestionnaireAnswer); + const [savingIndex, setSavingIndex] = useState(null); // No longer need trigger tokens - using server actions instead of Trigger.dev @@ -137,10 +88,8 @@ export function useQuestionnaireDetailState({ searchQuery, setSearchQuery, isAutoAnswerProcessStartedRef, - saveIndexRef, - saveAnswerRef, - updateAnswerAction, - deleteAnswerAction, + savingIndex, + setSavingIndex, router, answerQueue, setAnswerQueue, diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParse.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParse.ts index 34fcfa671..1c680072e 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParse.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParse.ts @@ -1,14 +1,10 @@ 'use client'; -import type { parseQuestionnaireTask } from '@/jobs/tasks/vendors/parse-questionnaire'; -import { useRealtimeRun } from '@trigger.dev/react-hooks'; -import { useAction } from 'next-safe-action/hooks'; +import { api } from '@/lib/api-client'; import { useRouter } from 'next/navigation'; -import { useEffect } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { toast } from 'sonner'; -import { createRunReadToken, createTriggerToken } from '../actions/create-trigger-token'; -import { parseQuestionnaireAI } from '../actions/parse-questionnaire-ai'; -import { uploadQuestionnaireFile } from '../actions/upload-questionnaire-file'; +import { createTriggerToken } from '../actions/create-trigger-token'; import type { QuestionAnswer } from '../components/types'; interface UseQuestionnaireParseProps { @@ -29,22 +25,20 @@ interface UseQuestionnaireParseProps { orgId: string; } +type ParseStatus = 'idle' | 'executing'; + export function useQuestionnaireParse({ - parseTaskId, - parseToken, autoAnswerToken, setAutoAnswerToken, setIsParseProcessStarted, - setParseTaskId, - setParseToken, - setResults, - setExtractedContent, - setQuestionStatuses, - setHasClickedAutoAnswer, setQuestionnaireId, orgId, }: UseQuestionnaireParseProps) { const router = useRouter(); + const [uploadStatus, setUploadStatus] = useState('idle'); + const [parseStatus, setParseStatus] = useState('idle'); + const abortControllerRef = useRef(null); + // Get trigger token for auto-answer (can trigger and read) useEffect(() => { async function getAutoAnswerToken() { @@ -58,145 +52,82 @@ export function useQuestionnaireParse({ } }, [autoAnswerToken, setAutoAnswerToken]); - - // Track parse task with realtime hook - const { run: parseRun, error: parseError } = useRealtimeRun( - parseTaskId || '', - { - accessToken: parseToken || undefined, - enabled: !!parseTaskId && !!parseToken, - onComplete: (run) => { - setIsParseProcessStarted(false); - - if (run.output) { - const questionsAndAnswers = run.output.questionsAndAnswers as - | Array<{ - question: string; - answer: string | null; - }> - | undefined; - const extractedContent = run.output.extractedContent as string | undefined; - const questionnaireId = run.output.questionnaireId as string | undefined; - - if (questionsAndAnswers && Array.isArray(questionsAndAnswers)) { - if (questionnaireId) { - // Navigate immediately to avoid showing results on new_questionnaire page - // The detail page will load the data from the database - setQuestionnaireId(questionnaireId); - toast.success( - `Successfully parsed ${questionsAndAnswers.length} question-answer pairs`, - ); - router.push(`/${orgId}/questionnaire/${questionnaireId}`); - } else { - // Fallback: if no questionnaireId, set results locally (shouldn't happen) - const initializedResults = questionsAndAnswers.map((qa) => ({ - ...qa, - failedToGenerate: false, - })); - setResults(initializedResults); - setExtractedContent(extractedContent || null); - setQuestionStatuses(new Map()); - setHasClickedAutoAnswer(false); - toast.success( - `Successfully parsed ${questionsAndAnswers.length} question-answer pairs`, - ); - } - } else { - toast.error('Parsed data is missing questions'); - } - } - }, - }, + const executeUploadAndParse = useCallback( + async (input: { fileName: string; fileType: string; fileData: string; organizationId: string }) => { + // Cancel any previous request + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + abortControllerRef.current = new AbortController(); + + setUploadStatus('executing'); + setParseStatus('executing'); + + try { + const response = await api.post<{ questionnaireId: string; totalQuestions: number }>( + '/v1/questionnaire/upload-and-parse', + { + organizationId: input.organizationId, + fileName: input.fileName, + fileType: input.fileType, + fileData: input.fileData, + source: 'internal', + }, + input.organizationId, ); - // Handle parse errors - useEffect(() => { - if (parseError) { - toast.error(`Failed to parse questionnaire: ${parseError.message}`); - } - }, [parseError]); - - // Handle parse task completion/failure - useEffect(() => { - if (parseRun?.status === 'FAILED' || parseRun?.status === 'CANCELED') { + if (response.error || !response.data) { setIsParseProcessStarted(false); - const errorMessage = - parseRun.error instanceof Error - ? parseRun.error.message - : typeof parseRun.error === 'string' - ? parseRun.error - : 'Task failed or was canceled'; - toast.error(`Failed to parse questionnaire: ${errorMessage}`); - } - }, [parseRun?.status, parseRun?.error, setIsParseProcessStarted]); - - const parseAction = useAction(parseQuestionnaireAI, { - onSuccess: async ({ data }: { data: any }) => { - const responseData = data?.data || data; - const taskId = responseData?.taskId as string | undefined; - - if (!taskId) { - setIsParseProcessStarted(false); - toast.error('Failed to start parse task'); + toast.error(response.error || 'Failed to parse questionnaire'); return; } - // ✅ Do NOT reset isParseProcessStarted here - task is started, need to wait for completion - // Clear old token before setting new task ID to prevent using wrong token with new run - setParseToken(null); - setParseTaskId(taskId); - - const tokenResult = await createRunReadToken(taskId); - if (tokenResult.success && tokenResult.token) { - setParseToken(tokenResult.token); - // ✅ Token created successfully - useRealtimeRun will connect and track the task - // isParseProcessStarted remains true until task completion (in onComplete) - } else { - // ✅ Only if token creation failed - reset state + const { questionnaireId, totalQuestions } = response.data; + setQuestionnaireId(questionnaireId); + toast.success(`Successfully parsed ${totalQuestions} questions`); + router.push(`/${orgId}/questionnaire/${questionnaireId}`); + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + return; // Request was cancelled + } setIsParseProcessStarted(false); - toast.error('Failed to create read token for parse task. The task may still be running - please check Trigger.dev dashboard.'); + console.error('Parse error:', error); + toast.error(error instanceof Error ? error.message : 'Failed to parse questionnaire'); + } finally { + setUploadStatus('idle'); + setParseStatus('idle'); } }, - onError: ({ error }) => { - // ✅ Only on task start error - reset state - setIsParseProcessStarted(false); - console.error('Parse action error:', error); - toast.error(error.serverError || 'Failed to start parse questionnaire'); - }, - }); - - const uploadFileAction = useAction(uploadQuestionnaireFile, { - onSuccess: ({ data }: { data: any }) => { - const responseData = data?.data || data; - const s3Key = responseData?.s3Key; - const fileName = responseData?.fileName; - const fileType = responseData?.fileType; + [orgId, router, setIsParseProcessStarted, setQuestionnaireId], + ); - if (s3Key && fileType) { - // ✅ isParseProcessStarted remains true - task continues - parseAction.execute({ - inputType: 's3', - s3Key, - fileName, - fileType, - }); - } else { - // ✅ Only if S3 key is missing - reset state - setIsParseProcessStarted(false); - toast.error('Failed to get S3 key after upload'); + // Cleanup on unmount + useEffect(() => { + return () => { + if (abortControllerRef.current) { + abortControllerRef.current.abort(); } + }; + }, []); + + // Simulated action objects to maintain compatibility with existing code + const uploadFileAction = { + status: uploadStatus, + execute: (input: { fileName: string; fileType: string; fileData: string; organizationId: string }) => { + executeUploadAndParse(input); }, - onError: ({ error }) => { - // ✅ On upload error - reset state - setIsParseProcessStarted(false); - console.error('Upload action error:', error); - toast.error(error.serverError || 'Failed to upload file'); + }; + + const parseAction = { + status: parseStatus, + execute: () => { + // No-op - parsing is now handled in uploadFileAction }, - }); + }; return { - parseRun, - parseError, + parseRun: null, + parseError: null, parseAction, uploadFileAction, }; diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParser.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParser.ts index a6ab4186e..2fd50b2d9 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParser.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireParser.ts @@ -79,44 +79,22 @@ export function useQuestionnaireParser() { triggerSingleAnswer: singleAnswer.triggerSingleAnswer, }); - // ✅ Improved isLoading logic - always shows loading until task completion + // isLoading logic - shows loading when parsing is in progress const isLoading = useMemo(() => { - // If parsing process has started, show loading until explicit completion + // If parsing process has started, show loading if (state.isParseProcessStarted) { - // Check if task is completed - const isCompleted = - parse.parseRun?.status === 'COMPLETED' || - parse.parseRun?.status === 'FAILED' || - parse.parseRun?.status === 'CANCELED'; - - // If task is completed, hide loading - if (isCompleted) { - return false; - } - - // Otherwise show loading (even if parseRun is not created yet) return true; } // Additional checks for reliability const isUploading = parse.uploadFileAction.status === 'executing'; const isParseActionExecuting = parse.parseAction.status === 'executing'; - const isParseRunActive = - parse.parseRun?.status === 'EXECUTING' || - parse.parseRun?.status === 'QUEUED' || - parse.parseRun?.status === 'WAITING'; - - if (isParseRunActive || isParseActionExecuting || isUploading) { - return true; - } - return false; + return isUploading || isParseActionExecuting; }, [ parse.uploadFileAction.status, parse.parseAction.status, - parse.parseRun?.status, state.isParseProcessStarted, - parse.parseRun, ]); const filteredResults = useMemo(() => { @@ -143,47 +121,23 @@ export function useQuestionnaireParser() { state.setShowExitDialog(false); }; - // ✅ Improved rawParseStatus logic - accounts for all statuses including cold start + // Simplified rawParseStatus logic for API-based parsing const rawParseStatus = useMemo(() => { - // If parsing process has started, always show status if (state.isParseProcessStarted) { - // Check statuses in priority order if (parse.uploadFileAction.status === 'executing') { return 'uploading'; } if (parse.parseAction.status === 'executing') { - return 'starting'; - } - if (parse.parseRun?.status === 'QUEUED') { - return 'queued'; - } - if (parse.parseRun?.status === 'EXECUTING') { return 'analyzing'; } - if (parse.parseRun?.status === 'WAITING') { - return 'processing'; - } - // If task is not created yet but process has started - show starting - if (!parse.parseRun) { - return 'starting'; - } - // If task is completed but isParseProcessStarted is still true - show processing - // (will be reset in onComplete) - if ( - parse.parseRun?.status === 'COMPLETED' || - parse.parseRun?.status === 'FAILED' || - parse.parseRun?.status === 'CANCELED' - ) { - return null; // Task completed, status will be reset - } + // Default to analyzing if process started but no specific status + return 'analyzing'; } return null; }, [ parse.uploadFileAction.status, parse.parseAction.status, - parse.parseRun?.status, state.isParseProcessStarted, - parse.parseRun, ]); // Throttled status for smooth transitions diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireSingleAnswer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireSingleAnswer.ts index 237611c75..5f2d967f8 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireSingleAnswer.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/hooks/useQuestionnaireSingleAnswer.ts @@ -1,10 +1,9 @@ 'use client'; -import { saveAnswerAction } from '../actions/save-answer'; -import { useAction } from 'next-safe-action/hooks'; import type { QuestionAnswer } from '../components/types'; import { toast } from 'sonner'; -import { useTransition, useRef } from 'react'; +import { useRef } from 'react'; +import { api } from '@/lib/api-client'; interface UseQuestionnaireSingleAnswerProps { results: QuestionAnswer[] | null; @@ -29,19 +28,12 @@ export function useQuestionnaireSingleAnswer({ const activeRequestsRef = useRef>(new Set()); // Action for saving answer - const saveAnswer = useAction(saveAnswerAction, { - onError: ({ error }) => { - console.error('Error saving answer:', error); - }, - }); - - const [isPending, startTransition] = useTransition(); - const triggerSingleAnswer = async (payload: { question: string; organizationId: string; questionIndex: number; totalQuestions: number; + questionnaireId?: string | null; }) => { const { questionIndex } = payload; @@ -63,24 +55,33 @@ export function useQuestionnaireSingleAnswer({ try { // Call server action directly via fetch for parallel processing - const response = await fetch('/api/questionnaire/answer-single', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - body: JSON.stringify({ + const response = await api.post<{ + success: boolean; + data?: { + questionIndex: number; + question: string; + answer: string | null; + sources?: QuestionAnswer['sources']; + error?: string; + }; + error?: string; + }>( + '/v1/questionnaire/answer-single', + { question: payload.question, questionIndex: payload.questionIndex, totalQuestions: payload.totalQuestions, - }), - }); + organizationId: payload.organizationId, + questionnaireId: payload.questionnaireId, + }, + payload.organizationId, + ); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); + if (response.error || !response.data) { + throw new Error(response.error || 'Failed to generate answer'); } - const result = await response.json(); + const result = response.data; if (result.success && result.data?.answer) { const output = result.data; @@ -124,18 +125,6 @@ export function useQuestionnaireSingleAnswer({ }); // Save answer to database - if (questionnaireId && output.answer) { - startTransition(() => { - saveAnswer.execute({ - questionnaireId, - questionIndex: targetIndex, - answer: output.answer, - sources: output.sources, - status: 'generated', - }); - }); - } - // Mark question as completed setQuestionStatuses((prev) => { const newStatuses = new Map(prev); diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/delete-document.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/delete-document.ts deleted file mode 100644 index 63e71e7dd..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/delete-document.ts +++ /dev/null @@ -1,116 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; -import { db } from '@db'; -import { DeleteObjectCommand } from '@aws-sdk/client-s3'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; -import { tasks } from '@trigger.dev/sdk'; -import { deleteKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/delete-knowledge-base-document'; - -const deleteDocumentSchema = z.object({ - documentId: z.string(), -}); - -export const deleteKnowledgeBaseDocumentAction = authActionClient - .inputSchema(deleteDocumentSchema) - .metadata({ - name: 'delete-knowledge-base-document', - track: { - event: 'delete-knowledge-base-document', - description: 'Delete Knowledge Base Document', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId } = parsedInput; - const { activeOrganizationId } = ctx.session; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - if (!s3Client) { - return { - success: false, - error: 'S3 client not configured', - }; - } - - if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { - return { - success: false, - error: 'Knowledge base bucket is not configured', - }; - } - - try { - // Find the document - const document = await db.knowledgeBaseDocument.findUnique({ - where: { - id: documentId, - organizationId: activeOrganizationId, - }, - }); - - if (!document) { - return { - success: false, - error: 'Document not found', - }; - } - - // Delete embeddings from vector database first (async, non-blocking) - let vectorDeletionRunId: string | undefined; - try { - const handle = await tasks.trigger( - 'delete-knowledge-base-document-from-vector', - { - documentId: document.id, - organizationId: activeOrganizationId, - }, - ); - vectorDeletionRunId = handle.id; - } catch (triggerError) { - // Log error but continue with deletion - console.error('Failed to trigger vector deletion task:', triggerError); - } - - // Delete from S3 - try { - const deleteCommand = new DeleteObjectCommand({ - Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, - Key: document.s3Key, - }); - await s3Client.send(deleteCommand); - } catch (s3Error) { - // Log error but continue with database deletion - console.error('Error deleting file from S3:', s3Error); - } - - // Delete from database - await db.knowledgeBaseDocument.delete({ - where: { - id: documentId, - }, - }); - - revalidatePath(`/${activeOrganizationId}/questionnaire/knowledge-base`); - - return { - success: true, - vectorDeletionRunId, // Return run ID for tracking deletion progress - }; - } catch (error) { - console.error('Error deleting knowledge base document:', error); - return { - success: false, - error: 'Failed to delete document', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/download-document.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/download-document.ts deleted file mode 100644 index 31e1dbd26..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/download-document.ts +++ /dev/null @@ -1,95 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; -import { db } from '@db'; -import { GetObjectCommand } from '@aws-sdk/client-s3'; -import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { z } from 'zod'; - -const downloadDocumentSchema = z.object({ - documentId: z.string(), -}); - -export const downloadKnowledgeBaseDocumentAction = authActionClient - .inputSchema(downloadDocumentSchema) - .metadata({ - name: 'download-knowledge-base-document', - track: { - event: 'download-knowledge-base-document', - description: 'Download Knowledge Base Document', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - if (!s3Client) { - return { - success: false, - error: 'S3 client not configured', - }; - } - - if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { - return { - success: false, - error: 'Knowledge base bucket is not configured', - }; - } - - try { - const document = await db.knowledgeBaseDocument.findUnique({ - where: { - id: documentId, - organizationId: session.activeOrganizationId, - }, - select: { - s3Key: true, - name: true, - fileType: true, - }, - }); - - if (!document) { - return { - success: false, - error: 'Document not found', - }; - } - - // Generate signed URL for download - const command = new GetObjectCommand({ - Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, - Key: document.s3Key, - ResponseContentDisposition: `attachment; filename="${encodeURIComponent(document.name)}"`, - }); - - const signedUrl = await getSignedUrl(s3Client, command, { - expiresIn: 3600, // URL expires in 1 hour - }); - - return { - success: true, - data: { - signedUrl, - fileName: document.name, - }, - }; - } catch (error) { - console.error('Error generating download URL:', error); - return { - success: false, - error: 'Failed to generate download URL', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts deleted file mode 100644 index b991568d9..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts +++ /dev/null @@ -1,119 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; -import { db } from '@db'; -import { GetObjectCommand } from '@aws-sdk/client-s3'; -import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { z } from 'zod'; - -const getDocumentViewUrlSchema = z.object({ - documentId: z.string(), -}); - -/** - * Gets a signed URL for viewing a knowledge base document (opens in browser, doesn't force download) - */ -export const getKnowledgeBaseDocumentViewUrlAction = authActionClient - .inputSchema(getDocumentViewUrlSchema) - .metadata({ - name: 'get-knowledge-base-document-view-url', - track: { - event: 'get-knowledge-base-document-view-url', - description: 'Get Knowledge Base Document View URL', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - if (!s3Client) { - return { - success: false, - error: 'S3 client not configured', - }; - } - - if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { - return { - success: false, - error: 'Knowledge base bucket is not configured', - }; - } - - try { - const document = await db.knowledgeBaseDocument.findUnique({ - where: { - id: documentId, - organizationId: session.activeOrganizationId, - }, - select: { - s3Key: true, - name: true, - fileType: true, - }, - }); - - if (!document) { - return { - success: false, - error: 'Document not found', - }; - } - - // Generate signed URL for viewing in browser - // Set Content-Type header so browser knows how to handle the file - // For PDFs, images, and text files: browser will display inline - // For DOCX, XLSX, etc.: browser may download or try to open with external app - const command = new GetObjectCommand({ - Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, - Key: document.s3Key, - ResponseContentDisposition: `inline; filename="${encodeURIComponent(document.name)}"`, - ResponseContentType: document.fileType || 'application/octet-stream', // Set Content-Type header - }); - - const signedUrl = await getSignedUrl(s3Client, command, { - expiresIn: 3600, // URL expires in 1 hour - }); - - // Determine if file can be viewed inline in browser - const viewableInBrowser = [ - 'application/pdf', - 'image/png', - 'image/jpeg', - 'image/jpg', - 'image/gif', - 'image/webp', - 'image/svg+xml', - 'text/plain', - 'text/html', - 'text/csv', - 'text/markdown', - ].includes(document.fileType); - - return { - success: true, - data: { - signedUrl, - fileName: document.name, - fileType: document.fileType, - viewableInBrowser, - }, - }; - } catch (error) { - console.error('Error generating view URL:', error); - return { - success: false, - error: 'Failed to generate view URL', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/process-documents.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/process-documents.ts deleted file mode 100644 index f66ceed12..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/process-documents.ts +++ /dev/null @@ -1,78 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { tasks } from '@trigger.dev/sdk'; -import { processKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/process-knowledge-base-document'; -import { processKnowledgeBaseDocumentsOrchestratorTask } from '@/jobs/tasks/vector/process-knowledge-base-documents-orchestrator'; -import { z } from 'zod'; - -const processDocumentsSchema = z.object({ - documentIds: z.array(z.string()).min(1), - organizationId: z.string(), -}); - -/** - * Server action to trigger document processing - * Uses orchestrator for multiple documents, individual task for single document - */ -export const processKnowledgeBaseDocumentsAction = authActionClient - .inputSchema(processDocumentsSchema) - .metadata({ - name: 'process-knowledge-base-documents', - track: { - event: 'process-knowledge-base-documents', - description: 'Process Knowledge Base Documents', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentIds, organizationId } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId || session.activeOrganizationId !== organizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - let runId: string | undefined; - - // Use orchestrator for multiple documents, individual task for single document - if (documentIds.length > 1) { - const handle = await tasks.trigger( - 'process-knowledge-base-documents-orchestrator', - { - documentIds, - organizationId, - }, - ); - runId = handle.id; - } else { - const handle = await tasks.trigger( - 'process-knowledge-base-document', - { - documentId: documentIds[0]!, - organizationId, - }, - ); - runId = handle.id; - } - - return { - success: true, - runId, - message: documentIds.length > 1 - ? `Processing ${documentIds.length} documents in parallel...` - : 'Processing document...', - }; - } catch (error) { - console.error('Failed to trigger document processing:', error); - return { - success: false, - error: 'Failed to trigger document processing', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/upload-document.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/upload-document.ts deleted file mode 100644 index ae8da911c..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/actions/upload-document.ts +++ /dev/null @@ -1,132 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; -import { db } from '@db'; -import { PutObjectCommand } from '@aws-sdk/client-s3'; -import { randomBytes } from 'crypto'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; - -const uploadDocumentSchema = z.object({ - fileName: z.string(), - fileType: z.string(), - fileData: z.string(), // base64 encoded file - description: z.string().optional(), - organizationId: z.string(), -}); - -export const uploadKnowledgeBaseDocumentAction = authActionClient - .inputSchema(uploadDocumentSchema) - .metadata({ - name: 'upload-knowledge-base-document', - track: { - event: 'upload-knowledge-base-document', - description: 'Upload Knowledge Base Document', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { fileName, fileType, fileData, description, organizationId } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId || session.activeOrganizationId !== organizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - if (!s3Client) { - return { - success: false, - error: 'S3 client not configured', - }; - } - - if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { - return { - success: false, - error: 'Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable.', - }; - } - - try { - // Convert base64 to buffer - const fileBuffer = Buffer.from(fileData, 'base64'); - - // Validate file size (10MB limit) - const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; - if (fileBuffer.length > MAX_FILE_SIZE_BYTES) { - return { - success: false, - error: `File exceeds the ${MAX_FILE_SIZE_BYTES / (1024 * 1024)}MB limit`, - }; - } - - // Generate unique file key - const fileId = randomBytes(16).toString('hex'); - const sanitizedFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_'); - const timestamp = Date.now(); - const s3Key = `${organizationId}/knowledge-base-documents/${timestamp}-${fileId}-${sanitizedFileName}`; - - // Sanitize filename for S3 metadata - // S3 metadata values must be valid HTTP header values - // To be absolutely safe, we'll encode the filename using a safe character set - // Remove control characters and non-ASCII characters, keep only safe printable ASCII - const sanitizedMetadataFileName = Buffer.from(fileName, 'utf8') - .toString('ascii') // Convert to ASCII, replacing non-ASCII with '?' - .replace(/[\x00-\x1F\x7F]/g, '') // Remove control characters - .replace(/\?/g, '_') // Replace '?' (from non-ASCII conversion) with '_' - .trim() - .substring(0, 1024); // S3 metadata values have a 2KB limit per value - - // Upload to S3 - const putCommand = new PutObjectCommand({ - Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, - Key: s3Key, - Body: fileBuffer, - ContentType: fileType, - Metadata: { - originalFileName: sanitizedMetadataFileName, - organizationId, - }, - }); - - await s3Client.send(putCommand); - - // Create database record - const document = await db.knowledgeBaseDocument.create({ - data: { - name: fileName, - description: description || null, - s3Key, - fileType, - fileSize: fileBuffer.length, - organizationId, - processingStatus: 'pending', - }, - }); - - // Note: Processing is triggered by orchestrator in the component - // when multiple files are uploaded, or individually for single files - - revalidatePath(`/${organizationId}/questionnaire/knowledge-base`); - - return { - success: true, - data: { - id: document.id, - name: document.name, - s3Key: document.s3Key, - }, - }; - } catch (error) { - console.error('Error uploading knowledge base document:', error); - return { - success: false, - error: 'Failed to upload document', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx index 05ec51f4c..e72c6bf30 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx @@ -15,12 +15,9 @@ import { import { Button } from '@comp/ui/button'; import { Card } from '@comp/ui'; import { ChevronLeft, ChevronRight, Download, FileText, Trash2, Upload } from 'lucide-react'; -import { useState, useRef } from 'react'; +import { useState, useRef, useCallback } from 'react'; import { toast } from 'sonner'; -import { uploadKnowledgeBaseDocumentAction } from '../actions/upload-document'; -import { downloadKnowledgeBaseDocumentAction } from '../actions/download-document'; -import { deleteKnowledgeBaseDocumentAction } from '../actions/delete-document'; -import { processKnowledgeBaseDocumentsAction } from '../actions/process-documents'; +import { api } from '@/lib/api-client'; import { useRouter } from 'next/navigation'; import { usePagination } from '../../hooks/usePagination'; import { format } from 'date-fns'; @@ -36,6 +33,13 @@ interface AdditionalDocumentsSectionProps { documents: Awaited>; } +// Simple state for active run tracking +interface ActiveRun { + runId: string; + token: string; + documentIds: string[]; +} + export function AdditionalDocumentsSection({ organizationId, documents, @@ -51,31 +55,28 @@ export function AdditionalDocumentsSection({ null, ); - // Track processing and deletion run IDs - const [processingRunIds, setProcessingRunIds] = useState>(new Map()); // documentId -> runId - const [deletionRunIds, setDeletionRunIds] = useState>(new Map()); // documentId -> runId + // Simple state for active processing and deletion runs + const [activeProcessingRun, setActiveProcessingRun] = useState(null); + const [activeDeletionRun, setActiveDeletionRun] = useState(null); - // Track processing/deletion progress for current document - const currentProcessingRunId = Array.from(processingRunIds.values())[0] || null; - const currentDeletionRunId = deletionRunIds.get(deletingId || '') || null; + // Stable callbacks for the hook + const handleProcessingComplete = useCallback(() => { + setActiveProcessingRun(null); + router.refresh(); + toast.success('Document processing completed'); + }, [router]); - const { isProcessing, isDeleting, processingStatus, deletionStatus } = useDocumentProcessing({ - processingRunId: currentProcessingRunId, - deletionRunId: currentDeletionRunId, - onProcessingComplete: () => { - // Clear processing run ID and refresh - setProcessingRunIds(new Map()); - router.refresh(); - toast.success('Document processed successfully'); - }, - onDeletionComplete: () => { - // Clear deletion run ID - const newDeletionRunIds = new Map(deletionRunIds); - if (deletingId) { - newDeletionRunIds.delete(deletingId); - } - setDeletionRunIds(newDeletionRunIds); - }, + const handleDeletionComplete = useCallback(() => { + setActiveDeletionRun(null); + }, []); + + const { isProcessing, isDeleting } = useDocumentProcessing({ + processingRunId: activeProcessingRun?.runId || null, + processingToken: activeProcessingRun?.token || null, + deletionRunId: activeDeletionRun?.runId || null, + deletionToken: activeDeletionRun?.token || null, + onProcessingComplete: handleProcessingComplete, + onDeletionComplete: handleDeletionComplete, }); const { currentPage, totalPages, paginatedItems, handlePageChange } = usePagination({ @@ -91,7 +92,7 @@ export function AdditionalDocumentsSection({ behavior: 'smooth', block: 'start', }); - }, 100); // Small delay to allow accordion animation to start + }, 100); } }; @@ -119,20 +120,32 @@ export function AdditionalDocumentsSection({ setUploadProgress({ ...newProgress }); // Upload file - const result = await uploadKnowledgeBaseDocumentAction({ - fileName: file.name, - fileType: file.type, - fileData, + const response = await api.post<{ + id: string; + name: string; + s3Key: string; + }>( + '/v1/knowledge-base/documents/upload', + { + fileName: file.name, + fileType: file.type, + fileData, + organizationId, + }, organizationId, - }); + ); + + if (response.error) { + throw new Error(response.error || 'Failed to upload file'); + } - if (result?.data?.success && result.data.data?.id) { - uploadedDocumentIds.push(result.data.data.id); + if (response.data?.id) { + uploadedDocumentIds.push(response.data.id); newProgress[file.name] = 100; setUploadProgress({ ...newProgress }); toast.success(`Successfully uploaded ${file.name}`); } else { - throw new Error(result?.data?.error || 'Failed to upload file'); + throw new Error('Failed to upload file: invalid response'); } } catch (error) { console.error(`Error uploading ${file.name}:`, error); @@ -144,28 +157,36 @@ export function AdditionalDocumentsSection({ } } - // Trigger processing for uploaded documents (orchestrator for multiple, individual for single) + // Trigger processing for uploaded documents if (uploadedDocumentIds.length > 0) { try { - const result = await processKnowledgeBaseDocumentsAction({ - documentIds: uploadedDocumentIds, + const response = await api.post<{ + success: boolean; + runId?: string; + publicAccessToken?: string; + message?: string; + }>( + '/v1/knowledge-base/documents/process', + { + documentIds: uploadedDocumentIds, + organizationId, + }, organizationId, - }); + ); - if (result?.data?.success) { - // Store run ID for tracking progress - const runId = result.data.runId; - if (runId) { - const newProcessingRunIds = new Map(processingRunIds); - // For orchestrator, track all documents with the same run ID - uploadedDocumentIds.forEach((docId) => { - newProcessingRunIds.set(docId, runId); - }); - setProcessingRunIds(newProcessingRunIds); - } - toast.success(result.data.message || 'Processing documents...'); - } else { - console.error('Failed to trigger document processing:', result?.data?.error); + if (response.error) { + console.error('Failed to trigger document processing:', response.error); + return; + } + + if (response.data?.success && response.data.runId && response.data.publicAccessToken) { + // Set active processing run + setActiveProcessingRun({ + runId: response.data.runId, + token: response.data.publicAccessToken, + documentIds: uploadedDocumentIds, + }); + toast.success(response.data.message || 'Processing documents...'); } } catch (error) { console.error('Failed to trigger document processing:', error); @@ -195,19 +216,33 @@ export function AdditionalDocumentsSection({ setDownloadingIds((prev) => new Set(prev).add(documentId)); try { - const result = await downloadKnowledgeBaseDocumentAction({ documentId }); + const response = await api.post<{ + signedUrl: string; + fileName: string; + }>( + `/v1/knowledge-base/documents/${documentId}/download`, + { + organizationId, + }, + organizationId, + ); + + if (response.error) { + toast.error(response.error || 'Failed to download file'); + return; + } - if (result?.data?.success && result.data.data?.signedUrl) { + if (response.data?.signedUrl) { // Create a temporary link and trigger download const link = document.createElement('a'); - link.href = result.data.data.signedUrl; + link.href = response.data.signedUrl; link.download = fileName; document.body.appendChild(link); link.click(); document.body.removeChild(link); toast.success(`Downloading ${fileName}...`); } else { - toast.error(result?.data?.error || 'Failed to download file'); + toast.error('Failed to download file: invalid response'); } } catch (error) { console.error('Error downloading file:', error); @@ -234,23 +269,37 @@ export function AdditionalDocumentsSection({ setIsDeleteDialogOpen(false); try { - const result = await deleteKnowledgeBaseDocumentAction({ - documentId: documentToDelete.id, - }); + const response = await api.post<{ + success: boolean; + vectorDeletionRunId?: string; + publicAccessToken?: string; + }>( + `/v1/knowledge-base/documents/${documentToDelete.id}/delete`, + { + organizationId, + }, + organizationId, + ); + + if (response.error) { + toast.error(response.error || 'Failed to delete document'); + return; + } - if (result?.data?.success) { - // Store deletion run ID for tracking progress - const vectorDeletionRunId = result.data.vectorDeletionRunId; - if (vectorDeletionRunId) { - const newDeletionRunIds = new Map(deletionRunIds); - newDeletionRunIds.set(documentToDelete.id, vectorDeletionRunId); - setDeletionRunIds(newDeletionRunIds); + if (response.data?.success) { + // Set active deletion run if we have the run info + if (response.data.vectorDeletionRunId && response.data.publicAccessToken) { + setActiveDeletionRun({ + runId: response.data.vectorDeletionRunId, + token: response.data.publicAccessToken, + documentIds: [documentToDelete.id], + }); } toast.success(`Successfully deleted ${documentToDelete.name}`); router.refresh(); } else { - toast.error(result?.data?.error || 'Failed to delete document'); + toast.error('Failed to delete document: invalid response'); } } catch (error) { console.error('Error deleting document:', error); @@ -275,6 +324,16 @@ export function AdditionalDocumentsSection({ }); }; + // Helper to check if a document is being processed + const isDocumentProcessing = (docId: string) => { + return activeProcessingRun?.documentIds.includes(docId) && isProcessing; + }; + + // Helper to check if a document's vectors are being deleted + const isDocumentDeletingVectors = (docId: string) => { + return activeDeletionRun?.documentIds.includes(docId) && isDeleting; + }; + return ( <> @@ -290,7 +349,7 @@ export function AdditionalDocumentsSection({

- Upload documents or images to enhance your knowledge base. Supported formats: PDF, Word (.docx), Excel, CSV, text files, and images (PNG, JPG, GIF, WebP, SVG). Click on a document to download it. + Upload documents or images to enhance your knowledge base. Supported formats: PDF, Word (.doc, .docx), Excel (.xlsx, .xls), CSV, text files (.txt, .md), and images (PNG, JPG, GIF, WebP, SVG). Click on a document to download it.

@@ -300,8 +359,8 @@ export function AdditionalDocumentsSection({ {paginatedItems.map((document: KnowledgeBaseDocument) => { const isDownloading = downloadingIds.has(document.id); const isDeleting = deletingId === document.id; - const isProcessingDocument = processingRunIds.has(document.id); - const isDeletingVector = deletionRunIds.has(document.id); + const isProcessingDoc = isDocumentProcessing(document.id); + const isDeletingVector = isDocumentDeletingVectors(document.id); const formattedDate = format(new Date(document.createdAt), 'MMM dd, yyyy'); return ( @@ -331,7 +390,7 @@ export function AdditionalDocumentsSection({
- {(isProcessingDocument || isDeletingVector) ? ( + {(isProcessingDoc || isDeletingVector) ? (
@@ -364,6 +423,9 @@ export function AdditionalDocumentsSection({ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': [ '.docx', ], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'], + 'application/vnd.ms-excel': ['.xls'], + 'text/csv': ['.csv'], 'text/plain': ['.txt'], 'text/markdown': ['.md'], 'image/png': ['.png'], diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts index 0ab979a77..4347ba491 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts @@ -1,84 +1,82 @@ 'use client'; import { useRealtimeRun } from '@trigger.dev/react-hooks'; -import { useEffect, useState } from 'react'; -import { createRunReadToken } from '../../../actions/create-trigger-token'; -import type { processKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/process-knowledge-base-document'; -import type { processKnowledgeBaseDocumentsOrchestratorTask } from '@/jobs/tasks/vector/process-knowledge-base-documents-orchestrator'; -import type { deleteKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/delete-knowledge-base-document'; +import { useCallback, useRef, useEffect } from 'react'; interface UseDocumentProcessingOptions { processingRunId?: string | null; + processingToken?: string | null; deletionRunId?: string | null; + deletionToken?: string | null; onProcessingComplete?: () => void; onDeletionComplete?: () => void; } +/** + * Hook to track document processing and deletion runs using Trigger.dev realtime + * + * The tokens should be obtained from the API response (e.g., from processDocuments or deleteDocument endpoints) + * which return `publicAccessToken` along with the `runId`. + */ export function useDocumentProcessing({ processingRunId, + processingToken, deletionRunId, + deletionToken, onProcessingComplete, onDeletionComplete, }: UseDocumentProcessingOptions) { - const [processingToken, setProcessingToken] = useState(null); - const [deletionToken, setDeletionToken] = useState(null); + // Use refs to avoid stale closure issues + const onProcessingCompleteRef = useRef(onProcessingComplete); + const onDeletionCompleteRef = useRef(onDeletionComplete); - // Get read token for processing run + // Keep refs updated useEffect(() => { - async function getProcessingToken() { - if (processingRunId) { - const result = await createRunReadToken(processingRunId); - if (result.success && result.token) { - setProcessingToken(result.token); - } - } - } - getProcessingToken(); - }, [processingRunId]); + onProcessingCompleteRef.current = onProcessingComplete; + }, [onProcessingComplete]); - // Get read token for deletion run useEffect(() => { - async function getDeletionToken() { - if (deletionRunId) { - const result = await createRunReadToken(deletionRunId); - if (result.success && result.token) { - setDeletionToken(result.token); - } - } - } - getDeletionToken(); - }, [deletionRunId]); + onDeletionCompleteRef.current = onDeletionComplete; + }, [onDeletionComplete]); + + // Stable callbacks that use refs + const handleProcessingComplete = useCallback(() => { + onProcessingCompleteRef.current?.(); + }, []); + + const handleDeletionComplete = useCallback(() => { + onDeletionCompleteRef.current?.(); + }, []); // Track processing run - const { run: processingRun } = useRealtimeRun< - typeof processKnowledgeBaseDocumentTask | typeof processKnowledgeBaseDocumentsOrchestratorTask - >(processingRunId || '', { + const { run: processingRun } = useRealtimeRun(processingRunId || '', { accessToken: processingToken || undefined, enabled: !!processingRunId && !!processingToken, - onComplete: () => { - onProcessingComplete?.(); - }, + onComplete: handleProcessingComplete, }); // Track deletion run - const { run: deletionRun } = useRealtimeRun( - deletionRunId || '', - { - accessToken: deletionToken || undefined, - enabled: !!deletionRunId && !!deletionToken, - onComplete: () => { - onDeletionComplete?.(); - }, - }, - ); + const { run: deletionRun } = useRealtimeRun(deletionRunId || '', { + accessToken: deletionToken || undefined, + enabled: !!deletionRunId && !!deletionToken, + onComplete: handleDeletionComplete, + }); + + // Check if processing is active (handle orchestrator child tasks) + const isProcessing = processingRun + ? ['EXECUTING', 'QUEUED', 'PENDING', 'WAITING'].includes(processingRun.status) + : false; + + const isDeleting = deletionRun + ? ['EXECUTING', 'QUEUED', 'PENDING', 'WAITING'].includes(deletionRun.status) + : false; return { processingRun, deletionRun, - isProcessing: processingRun?.status === 'EXECUTING' || processingRun?.status === 'QUEUED', - isDeleting: deletionRun?.status === 'EXECUTING' || deletionRun?.status === 'QUEUED', + isProcessing, + isDeleting, processingStatus: processingRun?.status, deletionStatus: deletionRun?.status, }; } - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/BackButton.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/BackButton.tsx deleted file mode 100644 index 547bafb43..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/BackButton.tsx +++ /dev/null @@ -1,22 +0,0 @@ -'use client'; - -import { Button } from '@comp/ui/button'; -import { ArrowLeft } from 'lucide-react'; -import { useRouter } from 'next/navigation'; - -export function BackButton() { - const router = useRouter(); - - return ( - - ); -} - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx deleted file mode 100644 index 59b15e216..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx +++ /dev/null @@ -1,19 +0,0 @@ -'use client'; - -import { BookOpen } from 'lucide-react'; - -export function KnowledgeBaseBreadcrumb() { - return ( - - ); -} - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/index.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/index.ts deleted file mode 100644 index ad5444742..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/components/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { BackButton } from './BackButton'; -export { KnowledgeBaseBreadcrumb } from './KnowledgeBaseBreadcrumb'; - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts deleted file mode 100644 index ae3054ac5..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts +++ /dev/null @@ -1,105 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { tasks } from '@trigger.dev/sdk'; -import { logger } from '@/utils/logger'; -import { z } from 'zod'; - -// Empty schema since this action doesn't need input -const deleteAllManualAnswersSchema = z.object({}); - -export const deleteAllManualAnswers = authActionClient - .inputSchema(deleteAllManualAnswersSchema) - .metadata({ - name: 'delete-all-manual-answers', - track: { - event: 'delete-all-manual-answers', - description: 'Delete All Manual Answers', - channel: 'server', - }, - }) - .action(async ({ ctx }) => { - const { activeOrganizationId } = ctx.session; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - // First, get all manual answer IDs BEFORE deletion - // This ensures the orchestrator has the IDs to delete from vector DB - const manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ - where: { - organizationId: activeOrganizationId, - }, - select: { - id: true, - }, - }); - - logger.info('Found manual answers to delete', { - organizationId: activeOrganizationId, - count: manualAnswers.length, - ids: manualAnswers.map((ma) => ma.id), - }); - - // Trigger orchestrator task to delete all manual answers from vector DB in parallel - // Pass the IDs directly to avoid race condition - // This runs in the background and processes deletions efficiently - if (manualAnswers.length > 0) { - try { - await tasks.trigger('delete-all-manual-answers-orchestrator', { - organizationId: activeOrganizationId, - manualAnswerIds: manualAnswers.map((ma) => ma.id), // Pass IDs directly - }); - logger.info('Triggered delete all manual answers orchestrator task', { - organizationId: activeOrganizationId, - count: manualAnswers.length, - }); - } catch (error) { - // Log error but continue with DB deletion - logger.warn('Failed to trigger delete all manual answers orchestrator', { - organizationId: activeOrganizationId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - // Continue with DB deletion even if orchestrator trigger fails - } - } else { - logger.info('No manual answers to delete', { - organizationId: activeOrganizationId, - }); - } - - // Delete all manual answers from main DB - // Vector DB deletion happens in background via orchestrator - await db.securityQuestionnaireManualAnswer.deleteMany({ - where: { - organizationId: activeOrganizationId, - }, - }); - - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - - revalidatePath(path); - revalidatePath(`/${activeOrganizationId}/questionnaire/knowledge-base`); - - return { - success: true, - }; - } catch (error) { - console.error('Error deleting all manual answers:', error); - return { - success: false, - error: 'Failed to delete all manual answers', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts deleted file mode 100644 index 6acac9f07..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts +++ /dev/null @@ -1,98 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; -import { tasks } from '@trigger.dev/sdk'; -import { logger } from '@/utils/logger'; - -const deleteManualAnswerSchema = z.object({ - manualAnswerId: z.string(), -}); - -export const deleteManualAnswer = authActionClient - .inputSchema(deleteManualAnswerSchema) - .metadata({ - name: 'delete-manual-answer', - track: { - event: 'delete-manual-answer', - description: 'Delete Manual Answer', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { manualAnswerId } = parsedInput; - const { activeOrganizationId } = ctx.session; - - if (!activeOrganizationId) { - return { - success: false, - error: 'Not authorized', - }; - } - - try { - // Verify manual answer exists and belongs to organization - const manualAnswer = await db.securityQuestionnaireManualAnswer.findUnique({ - where: { - id: manualAnswerId, - organizationId: activeOrganizationId, - }, - }); - - if (!manualAnswer) { - return { - success: false, - error: 'Manual answer not found', - }; - } - - // Trigger Trigger.dev task to delete from vector DB in background - // This runs asynchronously and doesn't block the main DB deletion - try { - await tasks.trigger('delete-manual-answer-from-vector', { - manualAnswerId, - organizationId: activeOrganizationId, - }); - logger.info('Triggered delete manual answer from vector DB task', { - manualAnswerId, - organizationId: activeOrganizationId, - }); - } catch (error) { - // Log error but continue with DB deletion - logger.warn('Failed to trigger delete manual answer from vector DB task', { - manualAnswerId, - organizationId: activeOrganizationId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - // Continue with DB deletion even if task trigger fails - } - - // Delete the manual answer from main DB - await db.securityQuestionnaireManualAnswer.delete({ - where: { - id: manualAnswerId, - }, - }); - - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - - revalidatePath(path); - revalidatePath(`/${activeOrganizationId}/questionnaire/knowledge-base`); - - return { - success: true, - }; - } catch (error) { - console.error('Error deleting manual answer:', error); - return { - success: false, - error: 'Failed to delete manual answer', - }; - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx index 35ca233d7..b601b231d 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx @@ -19,10 +19,8 @@ import { useParams, useRouter } from 'next/navigation'; import { useRef, useState, useEffect } from 'react'; import { usePagination } from '../../hooks/usePagination'; import { format } from 'date-fns'; -import { useAction } from 'next-safe-action/hooks'; import { toast } from 'sonner'; -import { deleteManualAnswer } from '../actions/delete-manual-answer'; -import { deleteAllManualAnswers } from '../actions/delete-all-manual-answers'; +import { api } from '@/lib/api-client'; interface ManualAnswersSectionProps { manualAnswers: Awaited>; @@ -38,42 +36,9 @@ export function ManualAnswersSection({ manualAnswers }: ManualAnswersSectionProp const [deleteAllDialogOpen, setDeleteAllDialogOpen] = useState(false); const [answerIdToDelete, setAnswerIdToDelete] = useState(null); const [isDeleting, setIsDeleting] = useState(false); + const [isDeletingAll, setIsDeletingAll] = useState(false); const [accordionValue, setAccordionValue] = useState(''); - const deleteAction = useAction(deleteManualAnswer, { - onSuccess: ({ data }) => { - if (data?.success) { - toast.success('Manual answer deleted successfully'); - setDeleteDialogOpen(false); - setAnswerIdToDelete(null); - setIsDeleting(false); - router.refresh(); - } else { - toast.error(data?.error || 'Failed to delete manual answer'); - setIsDeleting(false); - } - }, - onError: ({ error }) => { - toast.error(error.serverError || 'Failed to delete manual answer'); - setIsDeleting(false); - }, - }); - - const deleteAllAction = useAction(deleteAllManualAnswers, { - onSuccess: ({ data }) => { - if (data?.success) { - toast.success('All manual answers deleted successfully'); - setDeleteAllDialogOpen(false); - router.refresh(); - } else { - toast.error(data?.error || 'Failed to delete all manual answers'); - } - }, - onError: ({ error }) => { - toast.error(error.serverError || 'Failed to delete all manual answers'); - }, - }); - const { currentPage, totalPages, paginatedItems, handlePageChange } = usePagination({ items: manualAnswers, itemsPerPage: 10, @@ -84,10 +49,38 @@ export function ManualAnswersSection({ manualAnswers }: ManualAnswersSectionProp setDeleteDialogOpen(true); }; - const handleConfirmDelete = () => { - if (answerIdToDelete) { - setIsDeleting(true); - deleteAction.execute({ manualAnswerId: answerIdToDelete }); + const handleConfirmDelete = async () => { + if (!answerIdToDelete) return; + + setIsDeleting(true); + try { + const response = await api.post<{ success: boolean; error?: string }>( + `/v1/knowledge-base/manual-answers/${answerIdToDelete}/delete`, + { + organizationId: orgId, + }, + orgId, + ); + + if (response.error) { + toast.error(response.error || 'Failed to delete manual answer'); + setIsDeleting(false); + return; + } + + if (response.data?.success) { + toast.success('Manual answer deleted successfully'); + setDeleteDialogOpen(false); + setAnswerIdToDelete(null); + router.refresh(); + } else { + toast.error(response.data?.error || 'Failed to delete manual answer'); + } + } catch (error) { + console.error('Error deleting manual answer:', error); + toast.error('Failed to delete manual answer'); + } finally { + setIsDeleting(false); } }; @@ -95,8 +88,36 @@ export function ManualAnswersSection({ manualAnswers }: ManualAnswersSectionProp setDeleteAllDialogOpen(true); }; - const handleConfirmDeleteAll = () => { - deleteAllAction.execute({}); + const handleConfirmDeleteAll = async () => { + setIsDeletingAll(true); + try { + const response = await api.post<{ success: boolean; error?: string }>( + '/v1/knowledge-base/manual-answers/delete-all', + { + organizationId: orgId, + }, + orgId, + ); + + if (response.error) { + toast.error(response.error || 'Failed to delete all manual answers'); + setIsDeletingAll(false); + return; + } + + if (response.data?.success) { + toast.success('All manual answers deleted successfully'); + setDeleteAllDialogOpen(false); + router.refresh(); + } else { + toast.error(response.data?.error || 'Failed to delete all manual answers'); + } + } catch (error) { + console.error('Error deleting all manual answers:', error); + toast.error('Failed to delete all manual answers'); + } finally { + setIsDeletingAll(false); + } }; const handleAccordionChange = (value: string) => { @@ -323,13 +344,13 @@ export function ManualAnswersSection({ manualAnswers }: ManualAnswersSectionProp - Cancel + Cancel - {deleteAllAction.status === 'executing' ? 'Deleting...' : 'Delete All'} + {isDeletingAll ? 'Deleting...' : 'Delete All'} diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/approve-soa-document.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/approve-soa-document.ts deleted file mode 100644 index 471fa410a..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/approve-soa-document.ts +++ /dev/null @@ -1,88 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { z } from 'zod'; -import 'server-only'; - -const approveSOADocumentSchema = z.object({ - documentId: z.string(), -}); - -export const approveSOADocument = authActionClient - .inputSchema(approveSOADocumentSchema) - .metadata({ - name: 'approve-soa-document', - track: { - event: 'approve-soa-document', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId } = parsedInput; - const { session, user } = ctx; - - if (!session?.activeOrganizationId || !user?.id) { - throw new Error('Unauthorized'); - } - - const organizationId = session.activeOrganizationId; - const userId = user.id; - - // Check if user is owner or admin - const member = await db.member.findFirst({ - where: { - organizationId, - userId, - deactivated: false, - }, - }); - - if (!member) { - throw new Error('Member not found'); - } - - // Check if user has owner or admin role - const isOwnerOrAdmin = member.role.includes('owner') || member.role.includes('admin'); - - if (!isOwnerOrAdmin) { - throw new Error('Only owners and admins can approve SOA documents'); - } - - // Get the document - const document = await db.sOADocument.findFirst({ - where: { - id: documentId, - organizationId, - }, - }); - - if (!document) { - throw new Error('SOA document not found'); - } - - // Check if document is pending approval and current member is the approver - if (!(document as any).approverId || (document as any).approverId !== member.id) { - throw new Error('Document is not pending your approval'); - } - - if ((document as any).status !== 'needs_review') { - throw new Error('Document is not in needs_review status'); - } - - // Approve the document - keep approverId to track who approved, set status to completed, set approvedAt - const updatedDocument = await db.sOADocument.update({ - where: { id: documentId }, - data: { - // Keep approverId to track who approved it - status: 'completed', - approvedAt: new Date(), - }, - }); - - return { - success: true, - data: updatedDocument, - }; - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/auto-fill-soa.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/auto-fill-soa.ts deleted file mode 100644 index c4e7b3cdd..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/auto-fill-soa.ts +++ /dev/null @@ -1,328 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { syncOrganizationEmbeddings } from '@/lib/vector'; -import { db } from '@db'; -import { logger } from '@/utils/logger'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; -import { generateSOAAnswerWithRAG } from '../utils/generate-soa-answer'; -import 'server-only'; - -const inputSchema = z.object({ - documentId: z.string(), -}); - -export const autoFillSOA = authActionClient - .inputSchema(inputSchema) - .metadata({ - name: 'auto-fill-soa', - track: { - event: 'auto-fill-soa', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId } = parsedInput; - const { session, user } = ctx; - - if (!session?.activeOrganizationId) { - throw new Error('No active organization'); - } - - if (!user?.id) { - throw new Error('User not authenticated'); - } - - const organizationId = session.activeOrganizationId; - const userId = user.id; - - try { - // Fetch SOA document with configuration - const document = await db.sOADocument.findFirst({ - where: { - id: documentId, - organizationId, - }, - include: { - framework: true, - configuration: true, - answers: { - where: { - isLatestAnswer: true, - }, - }, - }, - }); - - if (!document) { - throw new Error('SOA document not found'); - } - - const configuration = document.configuration; - const questions = configuration.questions as Array<{ - id: string; - text: string; - columnMapping: { - title: string; - control_objective: string | null; - isApplicable: boolean | null; - }; - }>; - - // Process ALL questions - determine applicability for all - // If isApplicable is already set, we can still regenerate if needed - // For now, process all questions to ensure completeness - const questionsToAnswer = questions; - - logger.info('Starting auto-fill SOA', { - organizationId, - documentId, - totalQuestions: questions.length, - questionsToAnswer: questionsToAnswer.length, - }); - - // Sync organization embeddings before generating answers - try { - await syncOrganizationEmbeddings(organizationId); - logger.info('Organization embeddings synced successfully', { - organizationId, - }); - } catch (error) { - logger.warn('Failed to sync organization embeddings', { - organizationId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - // Continue with existing embeddings if sync fails - } - - // Process questions in batches to avoid overwhelming the system - // Process all questions to determine applicability - const batchSize = 10; - const results: Array<{ - questionId: string; - isApplicable: boolean | null; - justification: string | null; - sources: unknown; - success: boolean; - error: string | null; - }> = []; - - for (let i = 0; i < questionsToAnswer.length; i += batchSize) { - const batch = questionsToAnswer.slice(i, i + batchSize); - - const batchResults = await Promise.all( - batch.map((question, batchIndex) => { - const globalIndex = i + batchIndex; - - // First, determine if the control is applicable based on organization's context - const applicabilityQuestion = `Based on our organization's policies, documentation, business context, and operations, is the control "${question.columnMapping.title}" (${question.text}) applicable to our organization? - -Consider: -- Our business type and industry -- Our operational scope and scale -- Our risk profile -- Our regulatory requirements -- Our technical infrastructure - -Respond with ONLY "YES" or "NO" - no additional explanation.`; - - return generateSOAAnswerWithRAG( - applicabilityQuestion, - organizationId, - ).then(async (applicabilityResult) => { - if (!applicabilityResult.answer) { - return { - questionId: question.id, - isApplicable: null, - justification: null, - sources: applicabilityResult.sources, - success: false, - error: 'Failed to determine applicability - no answer generated', - }; - } - - // Parse YES/NO from answer - const answerText = applicabilityResult.answer.trim().toUpperCase(); - const isApplicable = answerText.includes('YES') || answerText.includes('APPLICABLE'); - const isNotApplicable = answerText.includes('NO') || answerText.includes('NOT APPLICABLE') || answerText.includes('NOT APPLICABLE'); - - let finalIsApplicable: boolean | null = null; - if (isApplicable && !isNotApplicable) { - finalIsApplicable = true; - } else if (isNotApplicable && !isApplicable) { - finalIsApplicable = false; - } - - // If not applicable, generate justification - let justification: string | null = null; - if (finalIsApplicable === false) { - const justificationQuestion = `Why is the control "${question.columnMapping.title}" not applicable to our organization? - -Provide a clear, professional justification explaining: -- Why this control does not apply to our business context -- Our operational characteristics that make it irrelevant -- Our risk profile considerations -- Any other relevant factors - -Keep the justification concise (2-3 sentences).`; - - const justificationResult = await generateSOAAnswerWithRAG( - justificationQuestion, - organizationId, - ); - - if (justificationResult.answer) { - justification = justificationResult.answer; - } - } - - return { - questionId: question.id, - isApplicable: finalIsApplicable, - justification, - sources: applicabilityResult.sources, - success: true, - error: null, - }; - }); - }), - ); - - results.push(...batchResults); - } - - // Save answers to database - const answersToSave = results - .filter((r) => r.success && r.isApplicable !== null) - .map((result) => { - const question = questionsToAnswer.find((q) => q.id === result.questionId); - if (!question) return null; - - // Get existing answer to determine version - return db.sOAAnswer.findFirst({ - where: { - documentId, - questionId: question.id, - isLatestAnswer: true, - }, - orderBy: { - answerVersion: 'desc', - }, - }).then(async (existingAnswer: { id: string; answerVersion: number } | null) => { - const nextVersion = existingAnswer ? existingAnswer.answerVersion + 1 : 1; - - // Mark existing answer as not latest if it exists - if (existingAnswer) { - await db.sOAAnswer.update({ - where: { id: existingAnswer.id }, - data: { isLatestAnswer: false }, - }); - } - - // Store justification in answer field if not applicable - // If applicable, answer is null (we don't need justification) - const answerValue = result.isApplicable === false ? result.justification : null; - - // Create new answer with justification (if not applicable) - const newAnswer = await db.sOAAnswer.create({ - data: { - documentId, - questionId: question.id, - answer: answerValue, // Store justification here if not applicable - status: 'generated', - sources: result.sources || undefined, - generatedAt: new Date(), - answerVersion: nextVersion, - isLatestAnswer: true, - createdBy: userId, - }, - }); - - return newAnswer; - }); - }) - .filter((promise) => promise !== null); - - await Promise.all(answersToSave); - - // Update the configuration's question mapping with all generated isApplicable values - const configQuestions = configuration.questions as Array<{ - id: string; - text: string; - columnMapping: { - title: string; - control_objective: string | null; - isApplicable: boolean | null; - justification: string | null; - }; - }>; - - // Create a map of results for easy lookup - const resultsMap = new Map( - results - .filter((r) => r.success && r.isApplicable !== null) - .map((r) => [r.questionId, r]) - ); - - // Update all questions in the configuration - const updatedQuestions = configQuestions.map((q) => { - const result = resultsMap.get(q.id); - if (result) { - return { - ...q, - columnMapping: { - ...q.columnMapping, - isApplicable: result.isApplicable, - justification: result.justification, - }, - }; - } - return q; - }); - - // Update configuration with new isApplicable values - await db.sOAFrameworkConfiguration.update({ - where: { id: configuration.id }, - data: { - questions: updatedQuestions, - }, - }); - - // Update document answered questions count - // Count questions that have isApplicable determined (not null) - const answeredCount = results.filter((r) => r.success && r.isApplicable !== null).length; - - await db.sOADocument.update({ - where: { id: documentId }, - data: { - answeredQuestions: answeredCount, - status: answeredCount === document.totalQuestions ? 'completed' : 'in_progress', - completedAt: answeredCount === document.totalQuestions ? new Date() : null, - }, - }); - - // Revalidate the page - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - revalidatePath(path); - - return { - success: true, - data: { - answered: results.filter((r) => r.success).length, - total: questionsToAnswer.length, - }, - }; - } catch (error) { - logger.error('Failed to auto-fill SOA', { - organizationId, - documentId, - error: error instanceof Error ? error.message : 'Unknown error', - }); - throw error instanceof Error ? error : new Error('Failed to auto-fill SOA'); - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/create-soa-document.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/create-soa-document.ts deleted file mode 100644 index a0692f4d5..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/create-soa-document.ts +++ /dev/null @@ -1,89 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { z } from 'zod'; -import 'server-only'; - -const createSOADocumentSchema = z.object({ - frameworkId: z.string(), - organizationId: z.string(), -}); - -export const createSOADocument = authActionClient - .inputSchema(createSOADocumentSchema) - .metadata({ - name: 'create-soa-document', - track: { - event: 'create-soa-document', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { frameworkId, organizationId } = parsedInput; - const { session } = ctx; - - if (!session?.activeOrganizationId || session.activeOrganizationId !== organizationId) { - throw new Error('Unauthorized'); - } - - // Get the latest SOA configuration for this framework - const configuration = await db.sOAFrameworkConfiguration.findFirst({ - where: { - frameworkId, - isLatest: true, - }, - }); - - if (!configuration) { - throw new Error('No SOA configuration found for this framework'); - } - - // Check if there's already a latest document for this framework and organization - const existingLatestDocument = await db.sOADocument.findFirst({ - where: { - frameworkId, - organizationId, - isLatest: true, - }, - }); - - // Determine the next version number - let nextVersion = 1; - if (existingLatestDocument) { - // Mark existing document as not latest - await db.sOADocument.update({ - where: { id: existingLatestDocument.id }, - data: { isLatest: false }, - }); - nextVersion = existingLatestDocument.version + 1; - } - - // Get questions from configuration to calculate totalQuestions - const questions = configuration.questions as Array<{ id: string }>; - const totalQuestions = Array.isArray(questions) ? questions.length : 0; - - // Create new SOA document - const document = await db.sOADocument.create({ - data: { - frameworkId, - organizationId, - configurationId: configuration.id, - version: nextVersion, - isLatest: true, - status: 'draft', - totalQuestions, - answeredQuestions: 0, - }, - include: { - framework: true, - configuration: true, - }, - }); - - return { - success: true, - data: document, - }; - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/decline-soa-document.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/decline-soa-document.ts deleted file mode 100644 index 1b50d32dd..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/decline-soa-document.ts +++ /dev/null @@ -1,88 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { z } from 'zod'; -import 'server-only'; - -const declineSOADocumentSchema = z.object({ - documentId: z.string(), -}); - -export const declineSOADocument = authActionClient - .inputSchema(declineSOADocumentSchema) - .metadata({ - name: 'decline-soa-document', - track: { - event: 'decline-soa-document', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId } = parsedInput; - const { session, user } = ctx; - - if (!session?.activeOrganizationId || !user?.id) { - throw new Error('Unauthorized'); - } - - const organizationId = session.activeOrganizationId; - const userId = user.id; - - // Check if user is owner or admin - const member = await db.member.findFirst({ - where: { - organizationId, - userId, - deactivated: false, - }, - }); - - if (!member) { - throw new Error('Member not found'); - } - - // Check if user has owner or admin role - const isOwnerOrAdmin = member.role.includes('owner') || member.role.includes('admin'); - - if (!isOwnerOrAdmin) { - throw new Error('Only owners and admins can decline SOA documents'); - } - - // Get the document - const document = await db.sOADocument.findFirst({ - where: { - id: documentId, - organizationId, - }, - }); - - if (!document) { - throw new Error('SOA document not found'); - } - - // Check if document is pending approval and current member is the approver - if (!(document as any).approverId || (document as any).approverId !== member.id) { - throw new Error('Document is not pending your approval'); - } - - if ((document as any).status !== 'needs_review') { - throw new Error('Document is not in needs_review status'); - } - - // Decline the document - clear approverId and set status back to completed (or in_progress) - const updatedDocument = await db.sOADocument.update({ - where: { id: documentId }, - data: { - approverId: null, // Clear approver - approvedAt: null, // Clear approved date - status: 'completed', // Set back to completed so it can be edited and resubmitted - }, - }); - - return { - success: true, - data: updatedDocument, - }; - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/ensure-soa-setup.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/ensure-soa-setup.ts deleted file mode 100644 index 9b8235c85..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/ensure-soa-setup.ts +++ /dev/null @@ -1,142 +0,0 @@ -'use server'; - -import { db } from '@db'; -import { seedISO27001SOAConfig } from './seed-soa-config'; -import 'server-only'; - -/** - * Direct server function to create SOA document without revalidation - * Used during page render, so cannot use server actions with revalidatePath - */ -async function createSOADocumentDirect(frameworkId: string, organizationId: string, configurationId: string) { - // Check if there's already a latest document for this framework and organization - const existingLatestDocument = await db.sOADocument.findFirst({ - where: { - frameworkId, - organizationId, - isLatest: true, - }, - }); - - // Determine the next version number - let nextVersion = 1; - if (existingLatestDocument) { - // Mark existing document as not latest - await db.sOADocument.update({ - where: { id: existingLatestDocument.id }, - data: { isLatest: false }, - }); - nextVersion = existingLatestDocument.version + 1; - } - - // Get questions from configuration to calculate totalQuestions - const configuration = await db.sOAFrameworkConfiguration.findUnique({ - where: { id: configurationId }, - }); - - if (!configuration) { - throw new Error('Configuration not found'); - } - - const questions = configuration.questions as Array<{ id: string }>; - const totalQuestions = Array.isArray(questions) ? questions.length : 0; - - // Create new SOA document - const document = await db.sOADocument.create({ - data: { - frameworkId, - organizationId, - configurationId: configuration.id, - version: nextVersion, - isLatest: true, - status: 'draft', - totalQuestions, - answeredQuestions: 0, - }, - include: { - answers: { - where: { - isLatestAnswer: true, - }, - }, - }, - }); - - return document; -} - -/** - * Ensures SOA configuration and document exist for a framework - * Currently only supports ISO 27001 - */ -export async function ensureSOASetup(frameworkId: string, organizationId: string) { - // Get framework to check if it's ISO - const framework = await db.frameworkEditorFramework.findUnique({ - where: { id: frameworkId }, - }); - - if (!framework) { - throw new Error('Framework not found'); - } - - // Check if framework is ISO 27001 (currently only supported framework) - const isISO27001 = ['ISO 27001', 'iso27001', 'ISO27001'].includes(framework.name); - - if (!isISO27001) { - return { - success: false, - error: 'Only ISO 27001 framework is currently supported', - configuration: null, - document: null, - }; - } - - // Check if configuration exists - let configuration = await db.sOAFrameworkConfiguration.findFirst({ - where: { - frameworkId, - isLatest: true, - }, - }); - - // Create configuration if it doesn't exist - if (!configuration) { - try { - configuration = await seedISO27001SOAConfig(); - } catch (error) { - throw new Error(`Failed to create SOA configuration: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - // Check if document exists - let document = await db.sOADocument.findFirst({ - where: { - frameworkId, - organizationId, - isLatest: true, - }, - include: { - answers: { - where: { - isLatestAnswer: true, - }, - }, - }, - }); - - // Create document if it doesn't exist (using direct function to avoid revalidation during render) - if (!document && configuration) { - try { - document = await createSOADocumentDirect(frameworkId, organizationId, configuration.id); - } catch (error) { - throw new Error(`Failed to create SOA document: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - return { - success: true, - configuration, - document, - }; -} - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/save-soa-answer.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/save-soa-answer.ts deleted file mode 100644 index 97b2942fe..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/save-soa-answer.ts +++ /dev/null @@ -1,183 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { headers } from 'next/headers'; -import { revalidatePath } from 'next/cache'; -import { z } from 'zod'; -import 'server-only'; - -const saveAnswerSchema = z.object({ - documentId: z.string(), - questionId: z.string(), - answer: z.string().nullable(), - isApplicable: z.boolean().nullable().optional(), - justification: z.string().nullable().optional(), -}); - -export const saveSOAAnswer = authActionClient - .inputSchema(saveAnswerSchema) - .metadata({ - name: 'save-soa-answer', - track: { - event: 'save-soa-answer', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId, questionId, answer, isApplicable, justification } = parsedInput; - const { session, user } = ctx; - - if (!session?.activeOrganizationId) { - throw new Error('No active organization'); - } - - if (!user?.id) { - throw new Error('User not authenticated'); - } - - const organizationId = session.activeOrganizationId; - const userId = user.id; - - try { - // Verify document exists and belongs to organization - const document = await db.sOADocument.findFirst({ - where: { - id: documentId, - organizationId, - }, - include: { - configuration: true, - }, - }); - - if (!document) { - throw new Error('SOA document not found'); - } - - // Get existing answer to determine version - const existingAnswer = await db.sOAAnswer.findFirst({ - where: { - documentId, - questionId, - isLatestAnswer: true, - }, - orderBy: { - answerVersion: 'desc', - }, - }); - - const nextVersion = existingAnswer ? existingAnswer.answerVersion + 1 : 1; - - // Mark existing answer as not latest if it exists - if (existingAnswer) { - await db.sOAAnswer.update({ - where: { id: existingAnswer.id }, - data: { isLatestAnswer: false }, - }); - } - - // Determine answer value: if isApplicable is NO, use justification; otherwise use provided answer or null - let finalAnswer: string | null = null; - if (isApplicable !== undefined) { - // If isApplicable is provided, use justification if NO, otherwise null - finalAnswer = isApplicable === false ? (justification || answer || null) : null; - } else { - // Fallback to provided answer - finalAnswer = answer || null; - } - - // Create or update answer - await db.sOAAnswer.create({ - data: { - documentId, - questionId, - answer: finalAnswer, - status: finalAnswer && finalAnswer.trim().length > 0 ? 'manual' : 'untouched', - answerVersion: nextVersion, - isLatestAnswer: true, - createdBy: existingAnswer ? undefined : userId, - updatedBy: userId, - }, - }); - - // Update configuration's question mapping if isApplicable or justification provided - // This needs to happen before counting answered questions - if (isApplicable !== undefined || justification !== undefined) { - const configuration = document.configuration; - const questions = configuration.questions as Array<{ - id: string; - text: string; - columnMapping: { - closure: string; - title: string; - control_objective: string | null; - isApplicable: boolean | null; - justification: string | null; - }; - }>; - - const updatedQuestions = questions.map((q) => { - if (q.id === questionId) { - return { - ...q, - columnMapping: { - ...q.columnMapping, - isApplicable: isApplicable !== undefined ? isApplicable : q.columnMapping.isApplicable, - justification: justification !== undefined ? justification : q.columnMapping.justification, - }, - }; - } - return q; - }); - - await db.sOAFrameworkConfiguration.update({ - where: { id: configuration.id }, - data: { - questions: updatedQuestions, - }, - }); - } - - // Update document answered questions count (count questions with isApplicable set in configuration) - const updatedConfiguration = await db.sOAFrameworkConfiguration.findUnique({ - where: { id: document.configurationId }, - }); - - let answeredCount = 0; - if (updatedConfiguration) { - const configQuestions = updatedConfiguration.questions as Array<{ - id: string; - columnMapping: { - isApplicable: boolean | null; - }; - }>; - answeredCount = configQuestions.filter(q => q.columnMapping.isApplicable !== null).length; - } - - await db.sOADocument.update({ - where: { id: documentId }, - data: { - answeredQuestions: answeredCount, - status: answeredCount === document.totalQuestions ? 'completed' : 'in_progress', - completedAt: answeredCount === document.totalQuestions ? new Date() : null, - // Clear approval when answers are edited - approverId: null, - approvedAt: null, - }, - }); - - // Revalidate the page - const headersList = await headers(); - let path = headersList.get('x-pathname') || headersList.get('referer') || ''; - path = path.replace(/\/[a-z]{2}\//, '/'); - revalidatePath(path); - - return { - success: true, - }; - } catch (error) { - throw error instanceof Error ? error : new Error('Failed to save SOA answer'); - } - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/seed-soa-config.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/seed-soa-config.ts deleted file mode 100644 index da425e372..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/seed-soa-config.ts +++ /dev/null @@ -1,55 +0,0 @@ -'use server'; - -import { db } from '@db'; -import { loadISOConfig } from '../utils/transform-iso-config'; -import 'server-only'; - -/** - * Seeds SOA configuration for ISO 27001 framework - * This creates the initial configuration if it doesn't exist - */ -export async function seedISO27001SOAConfig() { - // Find ISO 27001 framework by name - const iso27001Framework = await db.frameworkEditorFramework.findFirst({ - where: { - OR: [ - { name: 'ISO 27001' }, - { name: 'iso27001' }, - { name: 'ISO27001' }, - ], - }, - }); - - if (!iso27001Framework) { - throw new Error('ISO 27001 framework not found'); - } - - // Check if configuration already exists - const existingConfig = await db.sOAFrameworkConfiguration.findFirst({ - where: { - frameworkId: iso27001Framework.id, - isLatest: true, - }, - }); - - if (existingConfig) { - return existingConfig; // Return existing config - } - - // Load and transform ISO config - const soaConfig = await loadISOConfig(); - - // Create new SOA configuration - const newConfig = await db.sOAFrameworkConfiguration.create({ - data: { - frameworkId: iso27001Framework.id, - version: 1, - isLatest: true, - columns: soaConfig.columns, - questions: soaConfig.questions, - }, - }); - - return newConfig; -} - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/submit-soa-for-approval.ts b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/submit-soa-for-approval.ts deleted file mode 100644 index 1f806eb41..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/actions/submit-soa-for-approval.ts +++ /dev/null @@ -1,81 +0,0 @@ -'use server'; - -import { authActionClient } from '@/actions/safe-action'; -import { db } from '@db'; -import { z } from 'zod'; -import 'server-only'; - -const submitSOAForApprovalSchema = z.object({ - documentId: z.string(), - approverId: z.string(), -}); - -export const submitSOAForApproval = authActionClient - .inputSchema(submitSOAForApprovalSchema) - .metadata({ - name: 'submit-soa-for-approval', - track: { - event: 'submit-soa-for-approval', - channel: 'server', - }, - }) - .action(async ({ parsedInput, ctx }) => { - const { documentId, approverId } = parsedInput; - const { session, user } = ctx; - - if (!session?.activeOrganizationId || !user?.id) { - throw new Error('Unauthorized'); - } - - const organizationId = session.activeOrganizationId; - - // Verify approver is a member of the organization - const approverMember = await db.member.findFirst({ - where: { - id: approverId, - organizationId, - deactivated: false, - }, - }); - - if (!approverMember) { - throw new Error('Approver not found in organization'); - } - - // Check if approver is owner or admin - const isOwnerOrAdmin = approverMember.role.includes('owner') || approverMember.role.includes('admin'); - if (!isOwnerOrAdmin) { - throw new Error('Approver must be an owner or admin'); - } - - // Get the document - const document = await db.sOADocument.findFirst({ - where: { - id: documentId, - organizationId, - }, - }); - - if (!document) { - throw new Error('SOA document not found'); - } - - if ((document as any).status === 'needs_review') { - throw new Error('Document is already pending approval'); - } - - // Submit for approval - set approverId and status to needs_review - const updatedDocument = await db.sOADocument.update({ - where: { id: documentId }, - data: { - approverId, - status: 'needs_review', - }, - }); - - return { - success: true, - data: updatedDocument, - }; - }); - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/CreateSOADocument.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/CreateSOADocument.tsx index 1b0898286..21ca9edf9 100644 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/CreateSOADocument.tsx +++ b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/CreateSOADocument.tsx @@ -6,7 +6,7 @@ import { Plus, Loader2 } from 'lucide-react'; import { useRouter } from 'next/navigation'; import { useState } from 'react'; import { toast } from 'sonner'; -import { createSOADocument } from '../actions/create-soa-document'; +import { api } from '@/lib/api-client'; interface CreateSOADocumentProps { frameworkId: string; @@ -26,17 +26,23 @@ export function CreateSOADocument({ setIsCreating(true); try { - const result = await createSOADocument({ - frameworkId, + const response = await api.post<{ success: boolean; data?: { id: string } }>( + '/v1/soa/create-document', + { + frameworkId, + organizationId, + }, organizationId, - }); + ); - if (result?.data?.success && result?.data?.data) { + if (response.error) { + toast.error(response.error || 'Failed to create SOA document'); + } else if (response.data?.success && response.data?.data) { toast.success('SOA document created successfully'); - router.push(`/${organizationId}/questionnaire/soa/${result.data.data.id}`); + router.push(`/${organizationId}/questionnaire/soa/${response.data.data.id}`); router.refresh(); } else { - toast.error(result?.serverError || 'Failed to create SOA document'); + toast.error('Failed to create SOA document'); } } catch (error) { toast.error('An error occurred while creating the SOA document'); diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/EditableIsApplicable.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/EditableIsApplicable.tsx deleted file mode 100644 index 63f610591..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/EditableIsApplicable.tsx +++ /dev/null @@ -1,150 +0,0 @@ -'use client'; - -import { useState } from 'react'; -import { Button } from '@comp/ui/button'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@comp/ui/select'; -import { Edit2, Loader2 } from 'lucide-react'; -import { useAction } from 'next-safe-action/hooks'; -import { saveSOAAnswer } from '../actions/save-soa-answer'; -import { toast } from 'sonner'; - -interface EditableIsApplicableProps { - documentId: string; - questionId: string; - isApplicable: boolean | null; - isPendingApproval: boolean; - isControl7?: boolean; - isFullyRemote?: boolean; - onUpdate?: () => void; -} - -export function EditableIsApplicable({ - documentId, - questionId, - isApplicable: initialIsApplicable, - isPendingApproval, - isControl7 = false, - isFullyRemote = false, - onUpdate, -}: EditableIsApplicableProps) { - const [isEditing, setIsEditing] = useState(false); - const [isApplicable, setIsApplicable] = useState(initialIsApplicable); - - const saveAction = useAction(saveSOAAnswer, { - onSuccess: () => { - setIsEditing(false); - toast.success('Answer saved successfully'); - // Refresh page to update configuration - if (typeof window !== 'undefined') { - window.location.reload(); - } - onUpdate?.(); - }, - onError: ({ error }) => { - toast.error(error.serverError || 'Failed to save answer'); - }, - }); - - // If control 7.* and fully remote, disable editing - const isDisabled = isPendingApproval || (isControl7 && isFullyRemote); - - const handleSave = async () => { - await saveAction.execute({ - documentId, - questionId, - answer: null, // Answer is stored in justification field when NO - isApplicable, - justification: null, // Justification is handled separately in EditableJustification - }); - }; - - const handleCancel = () => { - setIsApplicable(initialIsApplicable); - setIsEditing(false); - }; - - if (isDisabled && !isEditing) { - return ( - - {isApplicable === true ? 'YES' : isApplicable === false ? 'NO' : '—'} - - ); - } - - if (!isEditing) { - return ( -
- - {isApplicable === true ? 'YES' : isApplicable === false ? 'NO' : '—'} - - -
- ); - } - - return ( -
- - - -
- ); -} - diff --git a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/EditableJustification.tsx b/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/EditableJustification.tsx deleted file mode 100644 index 2d8af4833..000000000 --- a/apps/app/src/app/(app)/[orgId]/questionnaire/soa/components/EditableJustification.tsx +++ /dev/null @@ -1,158 +0,0 @@ -'use client'; - -import { useState } from 'react'; -import { Button } from '@comp/ui/button'; -import { Textarea } from '@comp/ui/textarea'; -import { Check, X, Loader2, Edit2 } from 'lucide-react'; -import { useAction } from 'next-safe-action/hooks'; -import { saveSOAAnswer } from '../actions/save-soa-answer'; -import { toast } from 'sonner'; - -interface EditableJustificationProps { - documentId: string; - questionId: string; - isApplicable: boolean | null; - justification: string | null; - isPendingApproval: boolean; - isControl7?: boolean; - isFullyRemote?: boolean; - onUpdate?: () => void; -} - -export function EditableJustification({ - documentId, - questionId, - isApplicable, - justification: initialJustification, - isPendingApproval, - isControl7 = false, - isFullyRemote = false, - onUpdate, -}: EditableJustificationProps) { - const [isEditing, setIsEditing] = useState(false); - const [justification, setJustification] = useState(initialJustification); - const [error, setError] = useState(null); - - const saveAction = useAction(saveSOAAnswer, { - onSuccess: () => { - setIsEditing(false); - setError(null); - toast.success('Answer saved successfully'); - // Refresh page to update configuration - if (typeof window !== 'undefined') { - window.location.reload(); - } - onUpdate?.(); - }, - onError: ({ error }) => { - setError(error.serverError || 'Failed to save answer'); - toast.error(error.serverError || 'Failed to save answer'); - }, - }); - - // If control 7.* and fully remote, disable editing - const isDisabled = isPendingApproval || (isControl7 && isFullyRemote); - - const handleSave = async () => { - // Validate: if NO, justification is required - if (isApplicable === false && (!justification || justification.trim().length === 0)) { - setError('Justification is required when Applicable is NO'); - return; - } - - const answerValue = isApplicable === false ? justification : null; - - await saveAction.execute({ - documentId, - questionId, - answer: answerValue, - isApplicable, - justification: isApplicable === false ? justification : null, - }); - }; - - const handleCancel = () => { - setJustification(initialJustification); - setIsEditing(false); - setError(null); - }; - - // Only show if isApplicable is NO - if (isApplicable !== false) { - return —; - } - - if (isDisabled && !isEditing) { - return ( -

- {justification || '—'} -

- ); - } - - if (!isEditing) { - return ( -
-
-

- {justification || '—'} -

- -
-
- ); - } - - return ( -
-