From 1a095afe4edc0cec25e0bf4a9f9ac40ceb06bb34 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 18:24:08 -0500 Subject: [PATCH 1/7] [dev] [tofikwest] tofik/remove-unrelevant-files-from-v2-questionnare (#1807) * feat(security-questionnaire-knowledge-base): add knowledge base UI part and update security-questionnaire * fix(knowledge-base): remove cache logic * feat(security-questionnaire-knowledge-base): add manual answers management and UI components * feat(security-questionnaire): sync manual answers to vector DB and enhance management * feat(db): add migrations for knowledge base and questionnaire tables * feat(knowledge-base): add document processing and orchestration tasks * feat(knowledge-base): update UI parts * feat(knowledge-base): enhance document management with new actions and UI components * feat(security-questionnaire): enhance knowledge base UI and document management features * chore: remove all unrelevant changes * chore: remove file * chore: fix decodeBasicHtmlEntities * chore(dependencies): add mammoth and update package.json --------- Co-authored-by: Tofik Hasanov Co-authored-by: Mariano Fuentes --- .env.example | 2 +- SELF_HOSTING.md | 2 + apps/app/package.json | 1 + apps/app/src/actions/safe-action.ts | 71 ++- .../app/(app)/[orgId]/knowledge-base/page.tsx | 63 +++ .../components/QuestionnaireBreadcrumb.tsx | 48 ++ .../components/QuestionnaireDetailClient.tsx | 116 ++++ .../[questionnaireId]/data/queries.ts | 46 ++ .../[questionnaireId]/page.tsx | 53 ++ .../actions/delete-questionnaire-answer.ts | 118 ++++ .../actions/save-answer.ts | 207 +++++++ .../actions/save-answers-batch.ts | 155 +++++ .../actions/update-questionnaire-answer.ts | 182 ++++++ .../components/KnowledgeBaseDocumentLink.tsx | 69 +++ .../components/QuestionnaireResults.tsx | 8 + .../components/QuestionnaireResultsCards.tsx | 104 +++- .../components/QuestionnaireResultsTable.tsx | 112 +++- .../SecurityQuestionnaireBreadcrumb.tsx | 19 + .../components/types.ts | 2 + .../hooks/usePersistGeneratedAnswers.ts | 324 +++++++++++ .../hooks/useQuestionnaireActions.ts | 33 +- .../hooks/useQuestionnaireAutoAnswer.ts | 319 +++++++++-- .../hooks/useQuestionnaireDetail.ts | 529 ++++++++++++++++++ .../hooks/useQuestionnaireParse.ts | 14 + .../hooks/useQuestionnaireParser.ts | 5 + .../hooks/useQuestionnaireSingleAnswer.ts | 114 +++- .../hooks/useQuestionnaireState.ts | 4 + .../actions/delete-document.ts | 116 ++++ .../actions/download-document.ts | 95 ++++ .../actions/get-document-view-url.ts | 119 ++++ .../actions/process-documents.ts | 78 +++ .../actions/upload-document.ts | 132 +++++ .../components/AdditionalDocumentsSection.tsx | 436 +++++++++++++++ .../additional-documents/components/index.ts | 2 + .../hooks/useDocumentProcessing.ts | 84 +++ .../knowledge-base/components/BackButton.tsx | 22 + .../components/KnowledgeBaseBreadcrumb.tsx | 19 + .../components/KnowledgeBaseHeader.tsx | 18 + .../knowledge-base/components/index.ts | 3 + .../context/components/ContextSection.tsx | 178 ++++++ .../context/components/index.ts | 2 + .../knowledge-base/data/queries.ts | 131 +++++ .../knowledge-base/hooks/usePagination.ts | 39 ++ .../actions/delete-all-manual-answers.ts | 105 ++++ .../actions/delete-manual-answer.ts | 98 ++++ .../actions/save-manual-answer.ts | 142 +++++ .../components/ManualAnswersSection.tsx | 289 ++++++++++ .../manual-answers/components/index.ts | 2 + .../knowledge-base/page.tsx | 62 ++ .../components/PublishedPoliciesSection.tsx | 123 ++++ .../published-policies/components/index.ts | 2 + .../[orgId]/security-questionnaire/layout.tsx | 29 + .../new_questionnaire/page.tsx | 98 ++++ .../[orgId]/security-questionnaire/page.tsx | 24 +- .../start_page/README.md | 28 + .../actions/delete-questionnaire.ts | 65 +++ .../components/QuestionnaireHistory.tsx | 330 +++++++++++ .../components/QuestionnaireOverview.tsx | 70 +++ .../start_page/components/index.ts | 7 + .../start_page/data/queries.ts | 45 ++ .../hooks/useQuestionnaireHistory.ts | 59 ++ .../utils/deduplicate-sources.ts | 111 ++++ apps/app/src/app/s3.ts | 1 + apps/app/src/components/app-onboarding.tsx | 28 +- apps/app/src/env.mjs | 2 + .../delete-all-manual-answers-orchestrator.ts | 167 ++++++ .../vector/delete-knowledge-base-document.ts | 264 +++++++++ .../jobs/tasks/vector/delete-manual-answer.ts | 61 ++ .../helpers/extract-content-from-file.ts | 231 ++++++++ .../vector/process-knowledge-base-document.ts | 283 ++++++++++ ...s-knowledge-base-documents-orchestrator.ts | 160 ++++++ .../tasks/vendors/answer-question-helpers.ts | 102 ++-- .../jobs/tasks/vendors/parse-questionnaire.ts | 48 ++ .../src/lib/vector/README-MANUAL-ANSWERS.md | 110 ++++ .../src/lib/vector/core/count-embeddings.ts | 142 +++++ .../vector/core/find-existing-embeddings.ts | 355 ++++++++++-- apps/app/src/lib/vector/core/find-similar.ts | 10 +- .../src/lib/vector/core/upsert-embedding.ts | 72 ++- apps/app/src/lib/vector/index.ts | 2 + .../src/lib/vector/sync/sync-manual-answer.ts | 174 ++++++ .../src/lib/vector/sync/sync-organization.ts | 179 +++++- bun.lock | 24 +- .../migration.sql | 31 + .../migration.sql | 5 + .../migration.sql | 64 +++ .../migration.sql | 36 ++ .../schema/knowledge-base-document.prisma | 32 ++ packages/db/prisma/schema/organization.prisma | 3 + .../db/prisma/schema/questionnaire.prisma | 61 ++ ...ecurity-questionnaire-manual-answer.prisma | 29 + 90 files changed, 8265 insertions(+), 294 deletions(-) create mode 100644 apps/app/src/app/(app)/[orgId]/knowledge-base/page.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireDetailClient.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/data/queries.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/page.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/delete-questionnaire-answer.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answer.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answers-batch.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/update-questionnaire-answer.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/components/KnowledgeBaseDocumentLink.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/components/SecurityQuestionnaireBreadcrumb.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/usePersistGeneratedAnswers.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireDetail.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/delete-document.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/download-document.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/process-documents.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/upload-document.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/index.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/BackButton.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseHeader.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/index.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/ContextSection.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/index.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/data/queries.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/hooks/usePagination.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/save-manual-answer.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/index.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/page.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/PublishedPoliciesSection.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/index.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/layout.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/new_questionnaire/page.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/README.md create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/actions/delete-questionnaire.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireHistory.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireOverview.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/index.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/data/queries.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/hooks/useQuestionnaireHistory.ts create mode 100644 apps/app/src/app/(app)/[orgId]/security-questionnaire/utils/deduplicate-sources.ts create mode 100644 apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts create mode 100644 apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts create mode 100644 apps/app/src/jobs/tasks/vector/delete-manual-answer.ts create mode 100644 apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts create mode 100644 apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts create mode 100644 apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts create mode 100644 apps/app/src/lib/vector/README-MANUAL-ANSWERS.md create mode 100644 apps/app/src/lib/vector/core/count-embeddings.ts create mode 100644 apps/app/src/lib/vector/sync/sync-manual-answer.ts create mode 100644 packages/db/prisma/migrations/20251118160603_add_supplement_documents/migration.sql create mode 100644 packages/db/prisma/migrations/20251118171710_add_trigger_run_id_to_knowledge_base_document/migration.sql create mode 100644 packages/db/prisma/migrations/20251118183908_add_questionnaire_tables/migration.sql create mode 100644 packages/db/prisma/migrations/20251118220527_add_security_questionnaire_manual_answers/migration.sql create mode 100644 packages/db/prisma/schema/knowledge-base-document.prisma create mode 100644 packages/db/prisma/schema/questionnaire.prisma create mode 100644 packages/db/prisma/schema/security-questionnaire-manual-answer.prisma diff --git a/.env.example b/.env.example index c4da7a6f2..886df5796 100644 --- a/.env.example +++ b/.env.example @@ -18,7 +18,7 @@ APP_AWS_SECRET_ACCESS_KEY="" # AWS Secret Access Key APP_AWS_REGION="" # AWS Region APP_AWS_BUCKET_NAME="" # AWS Bucket Name APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET="" # AWS, Required for Security Questionnaire feature - +APP_AWS_KNOWLEDGE_BASE_BUCKET="" # AWS Required for the Knowledge Base feature in Security Questionnaire TRIGGER_SECRET_KEY="" # For background jobs. Self-host or use cloud-version @ https://trigger.dev # TRIGGER_API_URL="" # Only set if you are self-hosting TRIGGER_API_KEY="" # API key from Trigger.dev diff --git a/SELF_HOSTING.md b/SELF_HOSTING.md index c225133e9..c33f1a28e 100644 --- a/SELF_HOSTING.md +++ b/SELF_HOSTING.md @@ -45,6 +45,7 @@ App (`apps/app`): - **APP_AWS_REGION**, **APP_AWS_ACCESS_KEY_ID**, **APP_AWS_SECRET_ACCESS_KEY**, **APP_AWS_BUCKET_NAME**: AWS S3 credentials for file storage (attachments, general uploads). - **APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET**: AWS S3 bucket name specifically for questionnaire file uploads. Required for the Security Questionnaire feature. If not set, users will see an error when trying to parse questionnaires. +- **APP_AWS_KNOWLEDGE_BASE_BUCKET**: AWS S3 bucket name specifically for knowledge base documents. Required for the Knowledge Base feature in Security Questionnaire. If not set, users will see an error when trying to upload knowledge base documents. - **OPENAI_API_KEY**: Enables AI features that call OpenAI models. - **UPSTASH_REDIS_REST_URL**, **UPSTASH_REDIS_REST_TOKEN**: Optional Redis (Upstash) used for rate limiting/queues/caching. - **NEXT_PUBLIC_POSTHOG_KEY**, **NEXT_PUBLIC_POSTHOG_HOST**: Client analytics via PostHog; leave unset to disable. @@ -151,6 +152,7 @@ NEXT_PUBLIC_BETTER_AUTH_URL_PORTAL=http://localhost:3002 # APP_AWS_SECRET_ACCESS_KEY= # APP_AWS_BUCKET_NAME= # APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET= +# APP_AWS_KNOWLEDGE_BASE_BUCKET= # OPENAI_API_KEY= # UPSTASH_REDIS_REST_URL= # UPSTASH_REDIS_REST_TOKEN= diff --git a/apps/app/package.json b/apps/app/package.json index 99ab867b8..16482e763 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -75,6 +75,7 @@ "geist": "^1.3.1", "jspdf": "^3.0.2", "lucide-react": "^0.544.0", + "mammoth": "^1.11.0", "motion": "^12.9.2", "next": "^15.4.6", "next-safe-action": "^8.0.3", diff --git a/apps/app/src/actions/safe-action.ts b/apps/app/src/actions/safe-action.ts index 3814b4f65..656010ebb 100644 --- a/apps/app/src/actions/safe-action.ts +++ b/apps/app/src/actions/safe-action.ts @@ -68,7 +68,7 @@ export const authActionClient = actionClientWithMeta }, }); - const { fileData: _, ...inputForLog } = clientInput as any; + const { fileData: _, ...inputForLog } = (clientInput || {}) as any; logger.info('Input ->', JSON.stringify(inputForLog, null, 2)); logger.info('Result ->', JSON.stringify(result.data, null, 2)); @@ -79,7 +79,7 @@ export const authActionClient = actionClientWithMeta return result; }) - .use(async ({ next, metadata }) => { + .use(async ({ next, metadata, ctx }) => { const headersList = await headers(); let remaining: number | undefined; @@ -97,6 +97,7 @@ export const authActionClient = actionClientWithMeta return next({ ctx: { + ...ctx, ip: headersList.get('x-forwarded-for'), userAgent: headersList.get('user-agent'), ratelimit: { @@ -106,58 +107,51 @@ export const authActionClient = actionClientWithMeta }); }) .use(async ({ next, metadata, ctx }) => { - const session = await auth.api.getSession({ - headers: await headers(), - }); - - if (!session) { + // Use user and session from previous middleware instead of re-fetching + // This ensures consistency and avoids potential security issues from stale data + if (!ctx.user || !ctx.session) { throw new Error('Unauthorized'); } if (metadata.track) { - track(session.user.id, metadata.track.event, { + track(ctx.user.id, metadata.track.event, { channel: metadata.track.channel, - email: session.user.email, - name: session.user.name, - organizationId: session.session.activeOrganizationId, + email: ctx.user.email, + name: ctx.user.name, + organizationId: ctx.session.activeOrganizationId, }); } - return next({ - ctx: { - user: session.user, - }, - }); + return next({ ctx }); }) - .use(async ({ next, metadata, clientInput }) => { + .use(async ({ next, metadata, clientInput, ctx }) => { const headersList = await headers(); - const session = await auth.api.getSession({ - headers: headersList, - }); - - const member = await auth.api.getActiveMember({ - headers: headersList, - }); - - if (!session) { + + // Use user and session from previous middleware for consistency + // Only fetch activeMember as it may require fresh data + if (!ctx.user || !ctx.session) { throw new Error('Unauthorized'); } - if (!session.session.activeOrganizationId) { + if (!ctx.session.activeOrganizationId) { throw new Error('Organization not found'); } + const member = await auth.api.getActiveMember({ + headers: headersList, + }); + if (!member) { throw new Error('Member not found'); } - const { fileData: _, ...inputForAuditLog } = clientInput as any; + const { fileData: _, ...inputForAuditLog } = (clientInput || {}) as any; const data = { - userId: session.user.id, - email: session.user.email, - name: session.user.name, - organizationId: session.session.activeOrganizationId, + userId: ctx.user.id, + email: ctx.user.email, + name: ctx.user.name, + organizationId: ctx.session.activeOrganizationId, action: metadata.name, input: inputForAuditLog, ipAddress: headersList.get('x-forwarded-for') || null, @@ -203,9 +197,9 @@ export const authActionClient = actionClientWithMeta data: { data: JSON.stringify(data), memberId: member.id, - userId: session.user.id, + userId: ctx.user.id, description: metadata.track?.description || null, - organizationId: session.session.activeOrganizationId, + organizationId: ctx.session.activeOrganizationId, entityId, entityType, }, @@ -220,7 +214,7 @@ export const authActionClient = actionClientWithMeta revalidatePath(path); - return next(); + return next({ ctx }); }); // New action client that includes organization access check @@ -246,6 +240,7 @@ export const authWithOrgAccessClient = authActionClient.use(async ({ next, clien return next({ ctx: { + ...ctx, member, organizationId, }, @@ -272,7 +267,7 @@ export const authActionClientWithoutOrg = actionClientWithMeta }, }); - const { fileData: _, ...inputForLog } = clientInput as any; + const { fileData: _, ...inputForLog } = (clientInput || {}) as any; logger.info('Input ->', JSON.stringify(inputForLog, null, 2)); logger.info('Result ->', JSON.stringify(result.data, null, 2)); @@ -283,7 +278,7 @@ export const authActionClientWithoutOrg = actionClientWithMeta return result; }) - .use(async ({ next, metadata }) => { + .use(async ({ next, metadata, ctx }) => { const headersList = await headers(); let remaining: number | undefined; @@ -301,6 +296,7 @@ export const authActionClientWithoutOrg = actionClientWithMeta return next({ ctx: { + ...ctx, ip: headersList.get('x-forwarded-for'), userAgent: headersList.get('user-agent'), ratelimit: { @@ -330,6 +326,7 @@ export const authActionClientWithoutOrg = actionClientWithMeta return next({ ctx: { + ...ctx, user: session.user, }, }); diff --git a/apps/app/src/app/(app)/[orgId]/knowledge-base/page.tsx b/apps/app/src/app/(app)/[orgId]/knowledge-base/page.tsx new file mode 100644 index 000000000..50e87f488 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/knowledge-base/page.tsx @@ -0,0 +1,63 @@ +import PageWithBreadcrumb from '@/components/pages/PageWithBreadcrumb'; +import { auth } from '@/utils/auth'; +import { headers } from 'next/headers'; +import { notFound } from 'next/navigation'; +import { AdditionalDocumentsSection } from '../security-questionnaire/knowledge-base/additional-documents/components'; +import { ContextSection } from '../security-questionnaire/knowledge-base/context/components'; +import { ManualAnswersSection } from '../security-questionnaire/knowledge-base/manual-answers/components'; +import { PublishedPoliciesSection } from '../security-questionnaire/knowledge-base/published-policies/components'; +import { KnowledgeBaseHeader } from '../security-questionnaire/knowledge-base/components/KnowledgeBaseHeader'; +import { + getContextEntries, + getKnowledgeBaseDocuments, + getManualAnswers, + getPublishedPolicies, +} from '../security-questionnaire/knowledge-base/data/queries'; + +export default async function KnowledgeBasePage() { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.user?.id || !session?.session?.activeOrganizationId) { + return notFound(); + } + + const organizationId = session.session.activeOrganizationId; + + // Fetch all data in parallel + const [policies, contextEntries, manualAnswers, documents] = await Promise.all([ + getPublishedPolicies(organizationId), + getContextEntries(organizationId), + getManualAnswers(organizationId), + getKnowledgeBaseDocuments(organizationId), + ]); + + return ( + + + +
+ {/* Published Policies and Context Sections - Side by Side */} +
+ + +
+ + {/* Manual Answers Section */} + + + {/* Additional Documents Section */} + +
+ + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx new file mode 100644 index 000000000..763ed0251 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireBreadcrumb.tsx @@ -0,0 +1,48 @@ +'use client'; + +import { FileQuestion, FileText, ChevronRight } from 'lucide-react'; +import Link from 'next/link'; +import { useParams } from 'next/navigation'; + +interface QuestionnaireBreadcrumbProps { + filename: string; + organizationId: string; +} + +export function QuestionnaireBreadcrumb({ filename, organizationId }: QuestionnaireBreadcrumbProps) { + const params = useParams(); + const orgId = params.orgId as string; + + return ( + + ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireDetailClient.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireDetailClient.tsx new file mode 100644 index 000000000..471f28808 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/components/QuestionnaireDetailClient.tsx @@ -0,0 +1,116 @@ +'use client'; + +import { QuestionnaireResults } from '../../components/QuestionnaireResults'; +import { useQuestionnaireDetail } from '../../hooks/useQuestionnaireDetail'; + +interface QuestionnaireDetailClientProps { + questionnaireId: string; + organizationId: string; + initialQuestions: Array<{ + id: string; + question: string; + answer: string | null; + status: 'untouched' | 'generated' | 'manual'; + questionIndex: number; + sources: any; + }>; + filename: string; +} + +export function QuestionnaireDetailClient({ + questionnaireId, + organizationId, + initialQuestions, + filename, +}: QuestionnaireDetailClientProps) { + const { + results, + searchQuery, + setSearchQuery, + editingIndex, + editingAnswer, + setEditingAnswer, + expandedSources, + questionStatuses, + answeringQuestionIndex, + hasClickedAutoAnswer, + isLoading, + isAutoAnswering, + isExporting, + isSaving, + savingIndex, + filteredResults, + answeredCount, + totalCount, + progressPercentage, + handleAutoAnswer, + handleAnswerSingleQuestion, + handleEditAnswer, + handleSaveAnswer, + handleCancelEdit, + handleExport, + handleToggleSource, + } = useQuestionnaireDetail({ + questionnaireId, + organizationId, + initialQuestions, + }); + + return ( +
+
+

{filename}

+

+ Review and manage answers for this questionnaire +

+
+ ({ + question: r.question, + answer: r.answer, + sources: r.sources, + failedToGenerate: (r as any).failedToGenerate ?? false, // Preserve failedToGenerate from result + status: (r as any).status ?? 'untouched', // Preserve status field for UI behavior + _originalIndex: (r as any).originalIndex ?? index, // Preserve originalIndex for reference, fallback to map index + }))} + filteredResults={filteredResults?.map((r, index) => ({ + question: r.question, + answer: r.answer, + sources: r.sources, + failedToGenerate: (r as any).failedToGenerate ?? false, // Preserve failedToGenerate from result + status: (r as any).status ?? 'untouched', // Preserve status field for UI behavior + _originalIndex: (r as any).originalIndex ?? index, // Preserve originalIndex for reference, fallback to map index + }))} + searchQuery={searchQuery} + onSearchChange={setSearchQuery} + editingIndex={editingIndex} + editingAnswer={editingAnswer} + onEditingAnswerChange={setEditingAnswer} + expandedSources={expandedSources} + questionStatuses={questionStatuses} + answeringQuestionIndex={answeringQuestionIndex} + hasClickedAutoAnswer={hasClickedAutoAnswer} + isLoading={isLoading} + isAutoAnswering={isAutoAnswering} + isExporting={isExporting} + isSaving={isSaving} + savingIndex={savingIndex} + showExitDialog={false} + onShowExitDialogChange={() => {}} + onExit={() => {}} + onAutoAnswer={handleAutoAnswer} + onAnswerSingleQuestion={handleAnswerSingleQuestion} + onEditAnswer={handleEditAnswer} + onSaveAnswer={handleSaveAnswer} + onCancelEdit={handleCancelEdit} + onExport={handleExport} + onToggleSource={handleToggleSource} + totalCount={totalCount} + answeredCount={answeredCount} + progressPercentage={progressPercentage} + /> +
+ ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/data/queries.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/data/queries.ts new file mode 100644 index 000000000..dd90016ee --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/data/queries.ts @@ -0,0 +1,46 @@ +'use server'; + +import { auth } from '@/utils/auth'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { notFound } from 'next/navigation'; +import 'server-only'; + +export const getQuestionnaireById = async (questionnaireId: string, organizationId: string) => { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.session?.activeOrganizationId || session.session.activeOrganizationId !== organizationId) { + return null; + } + + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: questionnaireId, + organizationId, + }, + include: { + questions: { + orderBy: { + questionIndex: 'asc', + }, + select: { + id: true, + question: true, + answer: true, + status: true, + questionIndex: true, + sources: true, + }, + }, + }, + }); + + if (!questionnaire) { + return null; + } + + return questionnaire; +}; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/page.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/page.tsx new file mode 100644 index 000000000..77808dd13 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/[questionnaireId]/page.tsx @@ -0,0 +1,53 @@ +import PageWithBreadcrumb from '@/components/pages/PageWithBreadcrumb'; +import { auth } from '@/utils/auth'; +import { headers } from 'next/headers'; +import { notFound } from 'next/navigation'; +import { QuestionnaireResults } from '../components/QuestionnaireResults'; +import { useQuestionnaireDetail } from '../hooks/useQuestionnaireDetail'; +import { getQuestionnaireById } from './data/queries'; +import { QuestionnaireDetailClient } from './components/QuestionnaireDetailClient'; + +export default async function QuestionnaireDetailPage({ + params, +}: { + params: Promise<{ questionnaireId: string; orgId: string }>; +}) { + const { questionnaireId, orgId } = await params; + + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.user?.id || !session?.session?.activeOrganizationId) { + return notFound(); + } + + const organizationId = session.session.activeOrganizationId; + + if (organizationId !== orgId) { + return notFound(); + } + + const questionnaire = await getQuestionnaireById(questionnaireId, organizationId); + + if (!questionnaire) { + return notFound(); + } + + return ( + + + + ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/delete-questionnaire-answer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/delete-questionnaire-answer.ts new file mode 100644 index 000000000..a2878b4aa --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/delete-questionnaire-answer.ts @@ -0,0 +1,118 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { Prisma } from '@prisma/client'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; + +const deleteAnswerSchema = z.object({ + questionnaireId: z.string(), + questionAnswerId: z.string(), +}); + +export const deleteQuestionnaireAnswer = authActionClient + .inputSchema(deleteAnswerSchema) + .metadata({ + name: 'delete-questionnaire-answer', + track: { + event: 'delete-questionnaire-answer', + description: 'Delete Questionnaire Answer', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { questionnaireId, questionAnswerId } = parsedInput; + const { activeOrganizationId } = ctx.session; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // Verify questionnaire exists and belongs to organization + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: questionnaireId, + organizationId: activeOrganizationId, + }, + }); + + if (!questionnaire) { + return { + success: false, + error: 'Questionnaire not found', + }; + } + + // Verify question answer exists and belongs to questionnaire + const questionAnswer = await db.questionnaireQuestionAnswer.findUnique({ + where: { + id: questionAnswerId, + questionnaireId, + }, + }); + + if (!questionAnswer) { + return { + success: false, + error: 'Question answer not found', + }; + } + + // Delete the answer (set to null and status to untouched) + await db.questionnaireQuestionAnswer.update({ + where: { + id: questionAnswerId, + }, + data: { + answer: null, + status: 'untouched', + sources: Prisma.JsonNull, + generatedAt: null, + updatedBy: null, + updatedAt: new Date(), + }, + }); + + // Update answered questions count + const answeredCount = await db.questionnaireQuestionAnswer.count({ + where: { + questionnaireId, + answer: { + not: null, + }, + }, + }); + + await db.questionnaire.update({ + where: { + id: questionnaireId, + }, + data: { + answeredQuestions: answeredCount, + }, + }); + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + + return { + success: true, + }; + } catch (error) { + console.error('Error deleting answer:', error); + return { + success: false, + error: 'Failed to delete answer', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answer.ts new file mode 100644 index 000000000..0774151a9 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answer.ts @@ -0,0 +1,207 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; +import { syncManualAnswerToVector } from '@/lib/vector/sync/sync-manual-answer'; +import { logger } from '@/utils/logger'; + +const saveAnswerSchema = z.object({ + questionnaireId: z.string(), + questionIndex: z.number(), + answer: z.string().nullable(), + sources: z + .array( + z.object({ + sourceType: z.string(), + sourceName: z.string().optional(), + sourceId: z.string().optional(), + policyName: z.string().optional(), + documentName: z.string().optional(), + score: z.number(), + }), + ) + .optional(), + status: z.enum(['generated', 'manual']), +}); + +export const saveAnswerAction = authActionClient + .inputSchema(saveAnswerSchema) + .metadata({ + name: 'save-questionnaire-answer', + track: { + event: 'save-questionnaire-answer', + description: 'Save Questionnaire Answer', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { questionnaireId, questionIndex, answer, sources, status } = parsedInput; + const { activeOrganizationId } = ctx.session; + const userId = ctx.user.id; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // Verify questionnaire exists and belongs to organization + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: questionnaireId, + organizationId: activeOrganizationId, + }, + include: { + questions: { + where: { + questionIndex, + }, + }, + }, + }); + + if (!questionnaire) { + return { + success: false, + error: 'Questionnaire not found', + }; + } + + const existingQuestion = questionnaire.questions[0]; + + if (existingQuestion) { + // Update existing question answer + await db.questionnaireQuestionAnswer.update({ + where: { + id: existingQuestion.id, + }, + data: { + answer: answer || null, + status: status === 'generated' ? 'generated' : 'manual', + sources: sources ? (sources as any) : null, + generatedAt: status === 'generated' ? new Date() : null, + updatedBy: status === 'manual' ? userId || null : null, + updatedAt: new Date(), + }, + }); + + // If status is manual and answer exists, also save to SecurityQuestionnaireManualAnswer + if (status === 'manual' && answer && answer.trim().length > 0 && existingQuestion.question) { + try { + const manualAnswer = await db.securityQuestionnaireManualAnswer.upsert({ + where: { + organizationId_question: { + organizationId: activeOrganizationId, + question: existingQuestion.question.trim(), + }, + }, + create: { + question: existingQuestion.question.trim(), + answer: answer.trim(), + tags: [], + organizationId: activeOrganizationId, + sourceQuestionnaireId: questionnaireId, + createdBy: userId || null, + updatedBy: userId || null, + }, + update: { + answer: answer.trim(), + sourceQuestionnaireId: questionnaireId, + updatedBy: userId || null, + updatedAt: new Date(), + }, + }); + + // Sync to vector DB SYNCHRONOUSLY + logger.info('🔄 Syncing manual answer to vector DB from save-answer', { + manualAnswerId: manualAnswer.id, + organizationId: activeOrganizationId, + questionIndex, + }); + + const syncResult = await syncManualAnswerToVector( + manualAnswer.id, + activeOrganizationId, + ); + + if (!syncResult.success) { + logger.error('❌ Failed to sync manual answer to vector DB', { + manualAnswerId: manualAnswer.id, + organizationId: activeOrganizationId, + error: syncResult.error, + }); + // Don't fail the main operation - manual answer is saved in DB + } else { + logger.info('✅ Successfully synced manual answer to vector DB', { + manualAnswerId: manualAnswer.id, + embeddingId: syncResult.embeddingId, + organizationId: activeOrganizationId, + }); + } + } catch (error) { + // Log error but don't fail the main operation + logger.error('Error saving to manual answers:', { + error: error instanceof Error ? error.message : 'Unknown error', + questionIndex, + organizationId: activeOrganizationId, + }); + } + } + } else { + // Create new question answer (shouldn't happen, but handle it) + await db.questionnaireQuestionAnswer.create({ + data: { + questionnaireId, + question: '', // Will be updated from parse results + questionIndex, + answer: answer || null, + status: status === 'generated' ? 'generated' : 'manual', + sources: sources ? (sources as any) : null, + generatedAt: status === 'generated' ? new Date() : null, + updatedBy: status === 'manual' ? userId || null : null, + }, + }); + } + + // Update answered questions count + const answeredCount = await db.questionnaireQuestionAnswer.count({ + where: { + questionnaireId, + answer: { + not: null, + }, + }, + }); + + await db.questionnaire.update({ + where: { + id: questionnaireId, + }, + data: { + answeredQuestions: answeredCount, + }, + }); + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + + return { + success: true, + }; + } catch (error) { + console.error('Error saving answer:', error); + return { + success: false, + error: 'Failed to save answer', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answers-batch.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answers-batch.ts new file mode 100644 index 000000000..f36c5b5ed --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/save-answers-batch.ts @@ -0,0 +1,155 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; + +const saveAnswersBatchSchema = z.object({ + questionnaireId: z.string(), + answers: z.array( + z.object({ + questionIndex: z.number(), + answer: z.string().nullable(), + sources: z + .array( + z.object({ + sourceType: z.string(), + sourceName: z.string().optional(), + sourceId: z.string().optional(), + policyName: z.string().optional(), + documentName: z.string().optional(), + score: z.number(), + }), + ) + .optional(), + status: z.enum(['generated', 'manual']), + }), + ), +}); + +export const saveAnswersBatchAction = authActionClient + .inputSchema(saveAnswersBatchSchema) + .metadata({ + name: 'save-questionnaire-answers-batch', + track: { + event: 'save-questionnaire-answers-batch', + description: 'Save Questionnaire Answers Batch', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { questionnaireId, answers } = parsedInput; + const { activeOrganizationId } = ctx.session; + const userId = ctx.user.id; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // Verify questionnaire exists and belongs to organization + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: questionnaireId, + organizationId: activeOrganizationId, + }, + }); + + if (!questionnaire) { + return { + success: false, + error: 'Questionnaire not found', + }; + } + + // Get all existing questions for this questionnaire + const existingQuestions = await db.questionnaireQuestionAnswer.findMany({ + where: { + questionnaireId, + }, + }); + + const existingQuestionsMap = new Map( + existingQuestions.map((q) => [q.questionIndex, q]), + ); + + // Update or create answers + const updatePromises = answers.map(async (answerData) => { + const existing = existingQuestionsMap.get(answerData.questionIndex); + + if (existing) { + // Update existing + return db.questionnaireQuestionAnswer.update({ + where: { + id: existing.id, + }, + data: { + answer: answerData.answer || null, + status: answerData.status === 'generated' ? 'generated' : 'manual', + sources: answerData.sources ? (answerData.sources as any) : null, + generatedAt: answerData.status === 'generated' ? new Date() : null, + updatedBy: answerData.status === 'manual' ? userId || null : null, + updatedAt: new Date(), + }, + }); + } else { + // Create new (shouldn't happen, but handle it) + return db.questionnaireQuestionAnswer.create({ + data: { + questionnaireId, + question: '', // Will be updated from parse results + questionIndex: answerData.questionIndex, + answer: answerData.answer || null, + status: answerData.status === 'generated' ? 'generated' : 'manual', + sources: answerData.sources ? (answerData.sources as any) : null, + generatedAt: answerData.status === 'generated' ? new Date() : null, + updatedBy: answerData.status === 'manual' ? userId || null : null, + }, + }); + } + }); + + await Promise.all(updatePromises); + + // Update answered questions count + const answeredCount = await db.questionnaireQuestionAnswer.count({ + where: { + questionnaireId, + answer: { + not: null, + }, + }, + }); + + await db.questionnaire.update({ + where: { + id: questionnaireId, + }, + data: { + answeredQuestions: answeredCount, + }, + }); + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + + return { + success: true, + }; + } catch (error) { + console.error('Error saving answers batch:', error); + return { + success: false, + error: 'Failed to save answers', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/update-questionnaire-answer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/update-questionnaire-answer.ts new file mode 100644 index 000000000..ad55ed9b3 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/actions/update-questionnaire-answer.ts @@ -0,0 +1,182 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; +import { syncManualAnswerToVector } from '@/lib/vector/sync/sync-manual-answer'; +import { logger } from '@/utils/logger'; + +const updateAnswerSchema = z.object({ + questionnaireId: z.string(), + questionAnswerId: z.string(), + answer: z.string(), +}); + +export const updateQuestionnaireAnswer = authActionClient + .inputSchema(updateAnswerSchema) + .metadata({ + name: 'update-questionnaire-answer', + track: { + event: 'update-questionnaire-answer', + description: 'Update Questionnaire Answer', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { questionnaireId, questionAnswerId, answer } = parsedInput; + const { activeOrganizationId } = ctx.session; + const userId = ctx.user.id; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // Verify questionnaire exists and belongs to organization + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: questionnaireId, + organizationId: activeOrganizationId, + }, + }); + + if (!questionnaire) { + return { + success: false, + error: 'Questionnaire not found', + }; + } + + // Verify question answer exists and belongs to questionnaire + const questionAnswer = await db.questionnaireQuestionAnswer.findUnique({ + where: { + id: questionAnswerId, + questionnaireId, + }, + }); + + if (!questionAnswer) { + return { + success: false, + error: 'Question answer not found', + }; + } + + // Update the answer + await db.questionnaireQuestionAnswer.update({ + where: { + id: questionAnswerId, + }, + data: { + answer: answer.trim() || null, + status: 'manual', + updatedBy: userId || null, + updatedAt: new Date(), + }, + }); + + // Also save to SecurityQuestionnaireManualAnswer if answer exists + if (answer && answer.trim().length > 0 && questionAnswer.question) { + try { + const manualAnswer = await db.securityQuestionnaireManualAnswer.upsert({ + where: { + organizationId_question: { + organizationId: activeOrganizationId, + question: questionAnswer.question.trim(), + }, + }, + create: { + question: questionAnswer.question.trim(), + answer: answer.trim(), + tags: [], + organizationId: activeOrganizationId, + sourceQuestionnaireId: questionnaireId, + createdBy: userId || null, + updatedBy: userId || null, + }, + update: { + answer: answer.trim(), + sourceQuestionnaireId: questionnaireId, + updatedBy: userId || null, + updatedAt: new Date(), + }, + }); + + // Sync to vector DB SYNCHRONOUSLY + logger.info('🔄 Syncing manual answer to vector DB from questionnaire update', { + manualAnswerId: manualAnswer.id, + organizationId: activeOrganizationId, + questionId: questionAnswerId, + }); + + const syncResult = await syncManualAnswerToVector( + manualAnswer.id, + activeOrganizationId, + ); + + if (!syncResult.success) { + logger.error('❌ Failed to sync manual answer to vector DB', { + manualAnswerId: manualAnswer.id, + organizationId: activeOrganizationId, + error: syncResult.error, + }); + // Don't fail the main operation - manual answer is saved in DB + } else { + logger.info('✅ Successfully synced manual answer to vector DB', { + manualAnswerId: manualAnswer.id, + embeddingId: syncResult.embeddingId, + organizationId: activeOrganizationId, + }); + } + } catch (error) { + // Log error but don't fail the main operation + logger.error('Error saving to manual answers:', { + error: error instanceof Error ? error.message : 'Unknown error', + questionAnswerId, + organizationId: activeOrganizationId, + }); + } + } + + // Update answered questions count + const answeredCount = await db.questionnaireQuestionAnswer.count({ + where: { + questionnaireId, + answer: { + not: null, + }, + }, + }); + + await db.questionnaire.update({ + where: { + id: questionnaireId, + }, + data: { + answeredQuestions: answeredCount, + }, + }); + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + + return { + success: true, + }; + } catch (error) { + console.error('Error updating answer:', error); + return { + success: false, + error: 'Failed to update answer', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/KnowledgeBaseDocumentLink.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/KnowledgeBaseDocumentLink.tsx new file mode 100644 index 000000000..1fdf1fcdd --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/KnowledgeBaseDocumentLink.tsx @@ -0,0 +1,69 @@ +'use client'; + +import { LinkIcon, Loader2 } from 'lucide-react'; +import { useState } from 'react'; +import { getKnowledgeBaseDocumentViewUrlAction } from '../knowledge-base/additional-documents/actions/get-document-view-url'; + +interface KnowledgeBaseDocumentLinkProps { + documentId: string; + sourceName: string; + orgId: string; + className?: string; // Allow custom className for different contexts (cards vs table) +} + +export function KnowledgeBaseDocumentLink({ + documentId, + sourceName, + orgId, + className = 'font-medium text-primary hover:underline inline-flex items-center gap-1 disabled:opacity-50 disabled:cursor-not-allowed', +}: KnowledgeBaseDocumentLinkProps) { + const [isLoading, setIsLoading] = useState(false); + + const handleClick = async (e: React.MouseEvent) => { + e.preventDefault(); + if (isLoading) return; + + setIsLoading(true); + try { + const result = await getKnowledgeBaseDocumentViewUrlAction({ + documentId, + }); + + if (result?.data?.success && result.data.data) { + const { signedUrl, viewableInBrowser } = result.data.data; + + if (viewableInBrowser && signedUrl) { + // File can be viewed in browser - open it directly + window.open(signedUrl, '_blank', 'noopener,noreferrer'); + } else { + // File cannot be viewed in browser - navigate to knowledge base page + const knowledgeBaseUrl = `/${orgId}/security-questionnaire/knowledge-base`; + window.open(knowledgeBaseUrl, '_blank', 'noopener,noreferrer'); + } + } + } catch (error) { + console.error('Error opening knowledge base document:', error); + // Fallback: navigate to knowledge base page + const knowledgeBaseUrl = `/${orgId}/security-questionnaire/knowledge-base`; + window.open(knowledgeBaseUrl, '_blank', 'noopener,noreferrer'); + } finally { + setIsLoading(false); + } + }; + + return ( + + ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResults.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResults.tsx index d35854f8c..3f276b371 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResults.tsx +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResults.tsx @@ -23,6 +23,8 @@ interface QuestionnaireResultsProps { isLoading: boolean; isAutoAnswering: boolean; isExporting: boolean; + isSaving?: boolean; + savingIndex?: number | null; showExitDialog: boolean; onShowExitDialogChange: (show: boolean) => void; onExit: () => void; @@ -54,6 +56,8 @@ export function QuestionnaireResults({ isLoading, isAutoAnswering, isExporting, + isSaving, + savingIndex, showExitDialog, onShowExitDialogChange, onExit, @@ -109,6 +113,8 @@ export function QuestionnaireResults({ answeringQuestionIndex={answeringQuestionIndex} isAutoAnswering={isAutoAnswering} hasClickedAutoAnswer={hasClickedAutoAnswer} + isSaving={isSaving} + savingIndex={savingIndex} onEditAnswer={onEditAnswer} onSaveAnswer={onSaveAnswer} onCancelEdit={onCancelEdit} @@ -129,6 +135,8 @@ export function QuestionnaireResults({ answeringQuestionIndex={answeringQuestionIndex} isAutoAnswering={isAutoAnswering} hasClickedAutoAnswer={hasClickedAutoAnswer} + isSaving={isSaving} + savingIndex={savingIndex} onEditAnswer={onEditAnswer} onSaveAnswer={onSaveAnswer} onCancelEdit={onCancelEdit} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsCards.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsCards.tsx index dbf26c2a9..ced669304 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsCards.tsx +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsCards.tsx @@ -2,9 +2,11 @@ import { Button } from '@comp/ui/button'; import { Textarea } from '@comp/ui/textarea'; -import { BookOpen, ChevronDown, ChevronUp, Link as LinkIcon, Loader2 } from 'lucide-react'; +import { BookOpen, ChevronDown, ChevronUp, Link as LinkIcon, Loader2, Pencil } from 'lucide-react'; import Link from 'next/link'; import type { QuestionAnswer } from './types'; +import { deduplicateSources } from '../utils/deduplicate-sources'; +import { KnowledgeBaseDocumentLink } from './KnowledgeBaseDocumentLink'; interface QuestionnaireResultsCardsProps { orgId: string; @@ -18,6 +20,8 @@ interface QuestionnaireResultsCardsProps { answeringQuestionIndex: number | null; isAutoAnswering: boolean; hasClickedAutoAnswer: boolean; + isSaving?: boolean; + savingIndex?: number | null; onEditAnswer: (index: number) => void; onSaveAnswer: (index: number) => void; onCancelEdit: () => void; @@ -37,6 +41,8 @@ export function QuestionnaireResultsCards({ answeringQuestionIndex, isAutoAnswering, hasClickedAutoAnswer, + isSaving, + savingIndex, onEditAnswer, onSaveAnswer, onCancelEdit, @@ -46,19 +52,35 @@ export function QuestionnaireResultsCards({ return (
{filteredResults.map((qa, index) => { - const originalIndex = results.findIndex((r) => r === qa); - const isEditing = editingIndex === originalIndex; - const questionStatus = questionStatuses.get(originalIndex); - const isProcessing = questionStatus === 'processing'; + // Use originalIndex if available (from detail page), otherwise find by question text + const originalIndex = (qa as any)._originalIndex !== undefined + ? (qa as any)._originalIndex + : results.findIndex((r) => r.question === qa.question); + // Fallback to index if not found (shouldn't happen, but safety check) + const safeIndex = originalIndex >= 0 ? originalIndex : index; + + // Deduplicate sources for this question + const uniqueSources = qa.sources ? deduplicateSources(qa.sources) : []; + const isEditing = editingIndex === safeIndex; + const questionStatus = questionStatuses.get(safeIndex); + // Determine if this question is being processed + // It's processing if: + // 1. Status is explicitly 'processing' + // 2. This is the single question being answered + // 3. Auto-answer is running and this question doesn't have an answer yet (or has empty answer) + const isProcessing = + questionStatus === 'processing' || + answeringQuestionIndex === safeIndex || + (isAutoAnswering && hasClickedAutoAnswer && (!qa.answer || qa.answer.trim().length === 0) && questionStatus !== 'completed'); return (
- Question {originalIndex + 1} + Question {safeIndex + 1}

{qa.question}

@@ -74,27 +96,47 @@ export function QuestionnaireResultsCards({ autoFocus />
- -
) : ( <> - {qa.answer ? ( + {qa.answer && qa.answer.trim().length > 0 ? (
onEditAnswer(originalIndex)} + className="group relative rounded-xs p-3 bg-muted/30 border border-border/30 cursor-pointer transition-colors duration-150 ease-in-out hover:bg-muted/50 hover:border-primary/40" + onClick={() => onEditAnswer(safeIndex)} + title="Click to edit" > -

{qa.answer}

+
+

{qa.answer}

+ +
) : isProcessing ? ( -
+
- Generating answer... + Finding answer...
) : (
@@ -103,10 +145,10 @@ export function QuestionnaireResultsCards({ size="sm" onClick={(e) => { e.stopPropagation(); - onAnswerSingleQuestion(originalIndex); + onAnswerSingleQuestion(safeIndex); }} disabled={ - answeringQuestionIndex === originalIndex || + answeringQuestionIndex === safeIndex || (isAutoAnswering && hasClickedAutoAnswer) } className="w-full justify-center" @@ -124,7 +166,7 @@ export function QuestionnaireResultsCards({
- {qa.sources && qa.sources.length > 0 && ( -
+ {uniqueSources.length > 0 && ( +
- {expandedSources.has(originalIndex) && ( + {expandedSources.has(safeIndex) && (
- {qa.sources.map((source, sourceIndex) => { + {uniqueSources.map((source, sourceIndex) => { const isPolicy = source.sourceType === 'policy' && source.sourceId; + const isKnowledgeBaseDocument = + source.sourceType === 'knowledge_base_document' && source.sourceId; const sourceContent = source.sourceName || source.sourceType; return ( @@ -175,6 +219,12 @@ export function QuestionnaireResultsCards({ {sourceContent} + ) : isKnowledgeBaseDocument && source.sourceId ? ( + ) : ( {sourceContent} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsTable.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsTable.tsx index 165f5052c..8e3e877d2 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsTable.tsx +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/QuestionnaireResultsTable.tsx @@ -3,9 +3,11 @@ import { Button } from '@comp/ui/button'; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@comp/ui/table'; import { Textarea } from '@comp/ui/textarea'; -import { BookOpen, ChevronDown, ChevronUp, Link as LinkIcon, Loader2, Zap } from 'lucide-react'; +import { BookOpen, ChevronDown, ChevronUp, Link as LinkIcon, Loader2, Zap, Pencil } from 'lucide-react'; import Link from 'next/link'; import type { QuestionAnswer } from './types'; +import { deduplicateSources } from '../utils/deduplicate-sources'; +import { KnowledgeBaseDocumentLink } from './KnowledgeBaseDocumentLink'; interface QuestionnaireResultsTableProps { orgId: string; @@ -19,6 +21,8 @@ interface QuestionnaireResultsTableProps { answeringQuestionIndex: number | null; isAutoAnswering: boolean; hasClickedAutoAnswer: boolean; + isSaving?: boolean; + savingIndex?: number | null; onEditAnswer: (index: number) => void; onSaveAnswer: (index: number) => void; onCancelEdit: () => void; @@ -38,6 +42,8 @@ export function QuestionnaireResultsTable({ answeringQuestionIndex, isAutoAnswering, hasClickedAutoAnswer, + isSaving, + savingIndex, onEditAnswer, onSaveAnswer, onCancelEdit, @@ -56,15 +62,31 @@ export function QuestionnaireResultsTable({ {filteredResults.map((qa, index) => { - const originalIndex = results.findIndex((r) => r === qa); - const isEditing = editingIndex === originalIndex; - const questionStatus = questionStatuses.get(originalIndex); - const isProcessing = questionStatus === 'processing'; + // Use originalIndex if available (from detail page), otherwise find by question text + const originalIndex = (qa as any)._originalIndex !== undefined + ? (qa as any)._originalIndex + : results.findIndex((r) => r.question === qa.question); + // Fallback to index if not found (shouldn't happen, but safety check) + const safeIndex = originalIndex >= 0 ? originalIndex : index; + const isEditing = editingIndex === safeIndex; + const questionStatus = questionStatuses.get(safeIndex); + // Determine if this question is being processed + // It's processing if: + // 1. Status is explicitly 'processing' + // 2. This is the single question being answered + // 3. Auto-answer is running and this question doesn't have an answer yet (or has empty answer) + const isProcessing = + questionStatus === 'processing' || + answeringQuestionIndex === safeIndex || + (isAutoAnswering && hasClickedAutoAnswer && (!qa.answer || qa.answer.trim().length === 0) && questionStatus !== 'completed'); + + // Deduplicate sources for this question + const uniqueSources = qa.sources ? deduplicateSources(qa.sources) : []; return ( - + - {originalIndex + 1} + {safeIndex + 1}

{qa.question}

@@ -79,25 +101,48 @@ export function QuestionnaireResultsTable({ autoFocus />
- -
) : (
- {qa.answer ? ( -
onEditAnswer(originalIndex)}> -

- {qa.answer} -

+ {qa.answer && qa.answer.trim().length > 0 ? ( +
onEditAnswer(safeIndex)} + title="Click to edit" + > +
+

+ {qa.answer} +

+ +
) : isProcessing ? ( -
- +
+ Finding answer...
) : qa.failedToGenerate ? ( @@ -108,7 +153,7 @@ export function QuestionnaireResultsTable({
)} - {qa.sources && qa.sources.length > 0 && ( -
+ {uniqueSources.length > 0 && ( +
- {expandedSources.has(originalIndex) && ( + {expandedSources.has(safeIndex) && (
- {qa.sources.map((source, sourceIndex) => { + {uniqueSources.map((source, sourceIndex) => { const isPolicy = source.sourceType === 'policy' && source.sourceId; + const isKnowledgeBaseDocument = + source.sourceType === 'knowledge_base_document' && source.sourceId; const sourceContent = source.sourceName || source.sourceType; return ( @@ -187,6 +234,13 @@ export function QuestionnaireResultsTable({ {sourceContent} + ) : isKnowledgeBaseDocument && source.sourceId ? ( + ) : ( {sourceContent} )} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/SecurityQuestionnaireBreadcrumb.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/SecurityQuestionnaireBreadcrumb.tsx new file mode 100644 index 000000000..3ae039ce4 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/SecurityQuestionnaireBreadcrumb.tsx @@ -0,0 +1,19 @@ +'use client'; + +import { FileQuestion } from 'lucide-react'; + +export function SecurityQuestionnaireBreadcrumb() { + return ( + + ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/types.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/types.ts index 34e37f3c5..e08acb40b 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/types.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/components/types.ts @@ -6,8 +6,10 @@ export interface QuestionAnswer { sourceName?: string; sourceId?: string; policyName?: string; + documentName?: string; score: number; }>; failedToGenerate?: boolean; // Track if auto-generation was attempted but failed + status?: 'untouched' | 'generated' | 'manual'; // Track answer source: untouched, AI-generated, or manually edited } diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/usePersistGeneratedAnswers.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/usePersistGeneratedAnswers.ts new file mode 100644 index 000000000..541796671 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/usePersistGeneratedAnswers.ts @@ -0,0 +1,324 @@ +"use client"; + +import { type Dispatch, type SetStateAction, useEffect, useRef } from 'react'; +import type { QuestionAnswer } from '../components/types'; + +type PersistedQuestionAnswer = QuestionAnswer & { + originalIndex?: number; + questionAnswerId?: string; + status?: 'untouched' | 'generated' | 'manual'; +}; + +type UpdateAnswerAction = { + execute: (...args: any[]) => unknown; + executeAsync: (...args: any[]) => Promise; +}; + +interface UsePersistGeneratedAnswersParams { + questionnaireId: string | null; + results: TResults; + setResults: Dispatch>; + autoAnswerRun: { + metadata?: Record; + status?: string; + output?: unknown; + } | null; + updateAnswerAction: UpdateAnswerAction; + setQuestionStatuses: React.Dispatch< + React.SetStateAction> + >; +} + +export function usePersistGeneratedAnswers({ + questionnaireId, + results, + setResults, + autoAnswerRun, + updateAnswerAction, + setQuestionStatuses, +}: UsePersistGeneratedAnswersParams) { + const processedMetadataAnswersRef = useRef>(new Set()); + const pendingMetadataUpdatesRef = useRef< + Map + >(new Map()); + const metadataUpdateTimeoutRef = useRef(null); + const updateQueueRef = useRef>(Promise.resolve()); + const resultsRef = useRef(results ?? []); + const previousResultsRef = useRef(results ?? []); + const processedResultsRef = useRef>(new Set()); + const resultsUpdateTimeoutRef = useRef(null); + const pendingResultsUpdatesRef = useRef< + Map + >(new Map()); + const pendingUpdatesWaitingForIdRef = useRef< + Map + >(new Map()); + + useEffect(() => { + resultsRef.current = results ?? []; + }, [results]); + + useEffect(() => { + previousResultsRef.current = results ?? []; + }, [results]); + + useEffect(() => { + return () => { + if (metadataUpdateTimeoutRef.current) { + clearTimeout(metadataUpdateTimeoutRef.current); + } + if (resultsUpdateTimeoutRef.current) { + clearTimeout(resultsUpdateTimeoutRef.current); + } + }; + }, []); + + const enqueueUpdate = ( + key: string, + payload: { questionAnswerId: string; answer: string; sources?: any[] }, + onError: () => void, + ) => { + if (!questionnaireId) { + return; + } + + updateQueueRef.current = updateQueueRef.current + .catch(() => { + // Swallow previous error to keep queue progressing + }) + .then(async () => { + try { + await updateAnswerAction.executeAsync({ + questionnaireId, + questionAnswerId: payload.questionAnswerId, + answer: payload.answer, + sources: payload.sources, + status: 'generated', + }); + console.log('Successfully updated answer in database:', { + questionAnswerId: payload.questionAnswerId, + }); + } catch (error) { + console.error('Failed to update answer in database', error); + onError(); + } + + await new Promise((resolve) => setTimeout(resolve, 200)); + }); + }; + + useEffect(() => { + if (!questionnaireId || !resultsRef.current.length || !autoAnswerRun?.metadata) { + return; + } + + const meta = autoAnswerRun.metadata as Record; + const answerKeys = Object.keys(meta).filter((key) => key.startsWith('answer_')); + + answerKeys.forEach((key) => { + if (processedMetadataAnswersRef.current.has(key)) { + return; + } + + const answerData = meta[key] as { + questionIndex?: number; + answer?: string | null; + sources?: any[]; + }; + + if (!answerData || answerData.questionIndex === undefined || !answerData.answer) { + return; + } + + const resultMatch = resultsRef.current.find((r) => r.originalIndex === answerData.questionIndex); + + if (!resultMatch?.questionAnswerId) { + pendingUpdatesWaitingForIdRef.current.set(answerData.questionIndex, { + answer: answerData.answer || '', + sources: answerData.sources, + }); + return; + } + + processedMetadataAnswersRef.current.add(key); + pendingMetadataUpdatesRef.current.set(key, { + questionAnswerId: resultMatch.questionAnswerId, + answer: answerData.answer || '', + sources: answerData.sources, + }); + }); + + if (metadataUpdateTimeoutRef.current) { + clearTimeout(metadataUpdateTimeoutRef.current); + } + + metadataUpdateTimeoutRef.current = setTimeout(() => { + const updates = Array.from(pendingMetadataUpdatesRef.current.entries()); + pendingMetadataUpdatesRef.current.clear(); + + updates.forEach(([key, update]) => { + enqueueUpdate(key, update, () => { + processedMetadataAnswersRef.current.delete(key); + }); + }); + }, 500); + }, [autoAnswerRun?.metadata, questionnaireId]); + + useEffect(() => { + if (!questionnaireId || !results?.length) { + return; + } + + results.forEach((result) => { + if (result.originalIndex == null) return; + + const pendingForIndex = pendingUpdatesWaitingForIdRef.current.get(result.originalIndex); + + if (pendingForIndex && result.questionAnswerId) { + const answerKey = `${result.questionAnswerId}-${result.originalIndex}`; + if (!processedResultsRef.current.has(answerKey)) { + processedResultsRef.current.add(answerKey); + pendingResultsUpdatesRef.current.set(answerKey, { + questionAnswerId: result.questionAnswerId, + answer: pendingForIndex.answer, + sources: pendingForIndex.sources, + }); + } + pendingUpdatesWaitingForIdRef.current.delete(result.originalIndex); + } + + if (!result.questionAnswerId) return; + + const prevResult = previousResultsRef.current.find((r) => r.originalIndex === result.originalIndex); + const answerKey = `${result.questionAnswerId}-${result.originalIndex}`; + + if (processedResultsRef.current.has(answerKey)) { + return; + } + + if ( + result.answer && + result.answer.trim().length > 0 && + result.status !== 'manual' && + (!prevResult || prevResult.answer !== result.answer) + ) { + processedResultsRef.current.add(answerKey); + pendingResultsUpdatesRef.current.set(answerKey, { + questionAnswerId: result.questionAnswerId, + answer: result.answer, + sources: result.sources, + }); + } + }); + + if (resultsUpdateTimeoutRef.current) { + clearTimeout(resultsUpdateTimeoutRef.current); + } + + resultsUpdateTimeoutRef.current = setTimeout(() => { + const updates = Array.from(pendingResultsUpdatesRef.current.entries()); + pendingResultsUpdatesRef.current.clear(); + + updates.forEach(([answerKey, update]) => { + enqueueUpdate(answerKey, update, () => { + processedResultsRef.current.delete(answerKey); + }); + }); + }, 500); + }, [questionnaireId, results]); + + useEffect(() => { + if (!autoAnswerRun?.metadata || !resultsRef.current.length) { + return; + } + + const meta = autoAnswerRun.metadata as Record; + const answerKeys = Object.keys(meta).filter((key) => key.startsWith('answer_')); + + if (!answerKeys.length) { + return; + } + + const answers = answerKeys + .map((key) => { + const answer = meta[key] as { + questionIndex: number; + question: string; + answer: string | null; + sources?: Array<{ + sourceType: string; + sourceName?: string; + score: number; + }>; + }; + return answer; + }) + .filter((answer): answer is NonNullable => Boolean(answer)) + .sort((a, b) => a.questionIndex - b.questionIndex); + + if (!answers.length) { + return; + } + + setResults((prevResults) => { + if (!prevResults) { + return prevResults; + } + + const updatedResults = [...prevResults]; + let hasChanges = false; + + answers.forEach((answer) => { + const targetIndex = updatedResults.findIndex( + (r) => r.originalIndex === answer.questionIndex, + ); + + if (targetIndex < 0 || targetIndex >= updatedResults.length) { + return; + } + + const currentAnswer = updatedResults[targetIndex]?.answer; + const originalQuestion = updatedResults[targetIndex]?.question; + + if (answer.answer) { + if (currentAnswer !== answer.answer) { + updatedResults[targetIndex] = { + ...updatedResults[targetIndex], + question: originalQuestion || answer.question, + answer: answer.answer, + sources: answer.sources, + failedToGenerate: false, + status: + updatedResults[targetIndex]?.status === 'manual' + ? 'manual' + : 'generated', + }; + hasChanges = true; + + const statusKey = updatedResults[targetIndex]?.originalIndex ?? targetIndex; + setQuestionStatuses((prevStatuses) => { + const newStatuses = new Map(prevStatuses); + if (prevStatuses.get(statusKey) !== 'completed') { + newStatuses.set(statusKey, 'completed'); + return newStatuses; + } + return prevStatuses; + }); + } + } else if (!currentAnswer) { + updatedResults[targetIndex] = { + ...updatedResults[targetIndex], + question: originalQuestion || answer.question, + answer: null, + failedToGenerate: true, + }; + hasChanges = true; + } + }); + + return (hasChanges ? updatedResults : prevResults) as TResults; + }); + }, [autoAnswerRun?.metadata, setQuestionStatuses, setResults]); +} + + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireActions.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireActions.ts index 3ac27bb25..d093b8078 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireActions.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireActions.ts @@ -1,10 +1,11 @@ 'use client'; import { useAction } from 'next-safe-action/hooks'; -import { useCallback } from 'react'; +import { useCallback, useTransition } from 'react'; import type { FileRejection } from 'react-dropzone'; import { toast } from 'sonner'; import { exportQuestionnaire } from '../actions/export-questionnaire'; +import { saveAnswerAction } from '../actions/save-answer'; import type { QuestionAnswer } from '../components/types'; interface UseQuestionnaireActionsProps { @@ -13,6 +14,7 @@ interface UseQuestionnaireActionsProps { results: QuestionAnswer[] | null; editingAnswer: string; expandedSources: Set; + questionnaireId: string | null; setSelectedFile: (file: File | null) => void; setEditingIndex: (index: number | null) => void; setEditingAnswer: (answer: string) => void; @@ -56,6 +58,7 @@ export function useQuestionnaireActions({ results, editingAnswer, expandedSources, + questionnaireId, setSelectedFile, setEditingIndex, setEditingAnswer, @@ -75,6 +78,18 @@ export function useQuestionnaireActions({ triggerAutoAnswer, triggerSingleAnswer, }: UseQuestionnaireActionsProps) { + const saveAnswer = useAction(saveAnswerAction, { + onSuccess: () => { + // Answer saved successfully + }, + onError: ({ error }) => { + console.error('Error saving answer:', error); + // Don't show toast for every save - too noisy + }, + }); + + const [isPending, startTransition] = useTransition(); + const exportAction = useAction(exportQuestionnaire, { onSuccess: ({ data }: { data: any }) => { const responseData = data?.data || data; @@ -215,16 +230,28 @@ export function useQuestionnaireActions({ }; const handleSaveAnswer = (index: number) => { - if (!results) return; + if (!results || !questionnaireId) return; const updated = [...results]; + const answerText = editingAnswer.trim() || null; updated[index] = { ...updated[index], - answer: editingAnswer.trim() || null, + answer: answerText, failedToGenerate: false, }; setResults(updated); setEditingIndex(null); setEditingAnswer(''); + + // Save to database (use startTransition to avoid rendering issues) + startTransition(() => { + saveAnswer.execute({ + questionnaireId, + questionIndex: index, + answer: answerText, + status: 'manual', + }); + }); + toast.success('Answer updated'); }; diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireAutoAnswer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireAutoAnswer.ts index 00e2e4e65..374f9c182 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireAutoAnswer.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireAutoAnswer.ts @@ -2,8 +2,11 @@ import { useRealtimeTaskTrigger } from '@trigger.dev/react-hooks'; import type { vendorQuestionnaireOrchestratorTask } from '@/jobs/tasks/vendors/vendor-questionnaire-orchestrator'; -import { useEffect, useMemo, useRef } from 'react'; +import { useEffect, useMemo, useRef, useTransition } from 'react'; import { toast } from 'sonner'; +import { useAction } from 'next-safe-action/hooks'; +import { saveAnswerAction } from '../actions/save-answer'; +import { saveAnswersBatchAction } from '../actions/save-answers-batch'; import type { QuestionAnswer } from '../components/types'; interface UseQuestionnaireAutoAnswerProps { @@ -18,6 +21,7 @@ interface UseQuestionnaireAutoAnswerProps { React.SetStateAction> >; setAnsweringQuestionIndex: (index: number | null) => void; + questionnaireId: string | null; } export function useQuestionnaireAutoAnswer({ @@ -30,6 +34,7 @@ export function useQuestionnaireAutoAnswer({ setResults, setQuestionStatuses, setAnsweringQuestionIndex, + questionnaireId, }: UseQuestionnaireAutoAnswerProps) { // Use realtime task trigger for auto-answer const { @@ -42,10 +47,46 @@ export function useQuestionnaireAutoAnswer({ enabled: !!autoAnswerToken, }); + // Action for saving answers batch + const saveAnswersBatch = useAction(saveAnswersBatchAction, { + onError: ({ error }) => { + console.error('Error saving answers batch:', error); + }, + }); + + const [isPending, startTransition] = useTransition(); + + // Debug logging for run tracking + useEffect(() => { + console.log('[AutoAnswer] Hook state:', { + hasToken: !!autoAnswerToken, + hasRun: !!autoAnswerRun, + runId: autoAnswerRun?.id, + runStatus: autoAnswerRun?.status, + hasMetadata: !!autoAnswerRun?.metadata, + metadataKeys: autoAnswerRun?.metadata ? Object.keys(autoAnswerRun.metadata as Record).length : 0, + isTriggering: isAutoAnswerTriggering, + hasError: !!autoAnswerError, + }); + + if (autoAnswerRun?.metadata) { + const meta = autoAnswerRun.metadata as Record; + const answerKeys = Object.keys(meta).filter((key) => key.startsWith('answer_')); + const statusKeys = Object.keys(meta).filter((key) => key.startsWith('question_') && key.endsWith('_status')); + console.log('[AutoAnswer] Metadata keys:', { + answerKeys, + statusKeys, + allKeys: Object.keys(meta).slice(0, 20), // First 20 keys for debugging + }); + } + }, [autoAnswerToken, autoAnswerRun?.id, autoAnswerRun?.status, autoAnswerRun?.metadata, isAutoAnswerTriggering, autoAnswerError]); + // Track processed metadata to avoid infinite loops const processedMetadataRef = useRef(''); // Track which run ID we're currently processing for single questions const currentRunIdRef = useRef(null); + // Track which run IDs we've already processed completion for (to prevent infinite loops) + const processedCompletionRef = useRef>(new Set()); // Use ref to access latest results without causing dependency issues const resultsRef = useRef(results); useEffect(() => { @@ -67,7 +108,15 @@ export function useQuestionnaireAutoAnswer({ useEffect(() => { // Read individual answers and statuses from metadata keys // Each answer-question task updates parent metadata when it starts and completes - if (!autoAnswerRun?.metadata || !resultsRef.current) return; + if (!autoAnswerRun?.metadata || !resultsRef.current) { + if (autoAnswerRun && !autoAnswerRun.metadata) { + console.log('[AutoAnswer] Run exists but no metadata yet', { + runId: autoAnswerRun.id, + status: autoAnswerRun.status, + }); + } + return; + } // For single question operations, only process metadata from the current run // This prevents metadata from previous runs (like "Auto Answer All") from interfering @@ -84,10 +133,25 @@ export function useQuestionnaireAutoAnswer({ const answerKeys = Object.keys(meta).filter((key) => key.startsWith('answer_')).sort(); const statusKeys = Object.keys(meta).filter((key) => key.startsWith('question_') && key.endsWith('_status')).sort(); + // Debug logging + if (answerKeys.length > 0) { + console.log('[AutoAnswer] Found answer keys in metadata:', { + answerKeys, + answerCount: answerKeys.length, + runId: autoAnswerRun.id, + status: autoAnswerRun.status, + }); + } + // Build hash from actual values, not just keys const answerValues = answerKeys.map((key) => { const answer = meta[key] as { questionIndex?: number; answer?: string | null } | undefined; - return answer ? `${answer.questionIndex}:${answer.answer ? 'has-answer' : 'no-answer'}` : null; + if (answer) { + const value = `${answer.questionIndex}:${answer.answer ? 'has-answer' : 'no-answer'}`; + console.log('[AutoAnswer] Answer value for hash:', { key, value, answerData: answer }); + return value; + } + return null; }).filter(Boolean); const statusValues = statusKeys.map((key) => { @@ -103,7 +167,15 @@ export function useQuestionnaireAutoAnswer({ }); // Skip if we've already processed this exact metadata state - if (processedMetadataRef.current === metadataHash) return; + if (processedMetadataRef.current === metadataHash) { + console.log('[AutoAnswer] Skipping duplicate metadata hash'); + return; + } + console.log('[AutoAnswer] Processing new metadata:', { + hash: metadataHash.substring(0, 100), + answerKeysCount: answerKeys.length, + statusKeysCount: statusKeys.length, + }); processedMetadataRef.current = metadataHash; const isSingleQuestion = answeringQuestionIndex !== null; @@ -149,24 +221,62 @@ export function useQuestionnaireAutoAnswer({ if (answerKeys.length > 0) { const answers = answerKeys .map((key) => { - const answerData = meta[key] as { - questionIndex: number; - question: string; - answer: string | null; + const rawValue = meta[key]; + + // Handle case where metadata value might be a string (shouldn't happen, but be defensive) + if (typeof rawValue === 'string') { + console.warn('[AutoAnswer] Unexpected string value in metadata:', { key, value: rawValue }); + return undefined; + } + + // Handle case where metadata value is null or undefined + if (!rawValue || typeof rawValue !== 'object') { + console.warn('[AutoAnswer] Invalid metadata value:', { key, value: rawValue, type: typeof rawValue }); + return undefined; + } + + const answerData = rawValue as { + questionIndex?: number; + question?: string; + answer?: string | null; sources?: Array<{ sourceType: string; sourceName?: string; score: number; }>; - } | undefined; - return answerData; + }; + + // Validate that answerData has required fields + if (typeof answerData.questionIndex !== 'number') { + console.warn('[AutoAnswer] Missing questionIndex in answer data:', { key, answerData }); + return undefined; + } + + return { + questionIndex: answerData.questionIndex, + question: answerData.question || '', + answer: answerData.answer ?? null, + sources: answerData.sources || [], + }; }) .filter((answer): answer is NonNullable => answer !== undefined) .sort((a, b) => a.questionIndex - b.questionIndex); if (answers.length > 0) { + console.log('[AutoAnswer] Processing answers:', { + answersCount: answers.length, + answers: answers.map((a) => ({ + questionIndex: a.questionIndex, + hasAnswer: !!a.answer, + answerLength: a.answer?.length || 0, + })), + }); + setResults((prevResults) => { - if (!prevResults) return prevResults; + if (!prevResults) { + console.warn('[AutoAnswer] No previous results to update'); + return prevResults; + } const updatedResults = [...prevResults]; let hasChanges = false; @@ -186,7 +296,7 @@ export function useQuestionnaireAutoAnswer({ // For single question operations, double-check the index matches if (isSingleQuestion && answeringQuestionIndex !== null) { if (targetIndex !== answeringQuestionIndex) { - console.warn('Index mismatch in single question update:', { + console.warn('[AutoAnswer] Index mismatch in single question update:', { targetIndex, answeringQuestionIndex, answerQuestionIndex: answer.questionIndex, @@ -197,7 +307,7 @@ export function useQuestionnaireAutoAnswer({ // Safety check: ensure targetIndex is valid if (targetIndex < 0 || targetIndex >= updatedResults.length) { - console.warn('Invalid questionIndex in answer update:', { + console.warn('[AutoAnswer] Invalid questionIndex in answer update:', { targetIndex, resultsLength: updatedResults.length, answerQuestionIndex: answer.questionIndex, @@ -208,6 +318,13 @@ export function useQuestionnaireAutoAnswer({ const currentAnswer = updatedResults[targetIndex]?.answer; const originalQuestion = updatedResults[targetIndex]?.question; + console.log('[AutoAnswer] Updating answer:', { + targetIndex, + currentAnswer: currentAnswer ? `${currentAnswer.substring(0, 50)}...` : null, + newAnswer: answer.answer ? `${answer.answer.substring(0, 50)}...` : null, + hasAnswer: !!answer.answer, + }); + // Verify we're updating the correct question by checking question text matches // This is an extra safety check to prevent updating wrong questions if (originalQuestion && answer.question) { @@ -215,7 +332,7 @@ export function useQuestionnaireAutoAnswer({ if (isSingleQuestion && answeringQuestionIndex !== null) { const expectedQuestion = resultsRef.current?.[answeringQuestionIndex]?.question; if (expectedQuestion && answer.question.trim() !== expectedQuestion.trim()) { - console.warn('Question text mismatch in single question update:', { + console.warn('[AutoAnswer] Question text mismatch in single question update:', { targetIndex, answeringQuestionIndex, expectedQuestion: expectedQuestion.substring(0, 50), @@ -233,17 +350,22 @@ export function useQuestionnaireAutoAnswer({ if (answer.answer) { if (currentAnswer !== answer.answer) { + console.log('[AutoAnswer] Setting answer for question', targetIndex); updatedResults[targetIndex] = { + ...updatedResults[targetIndex], // Preserve status and other fields question: originalQuestion || answer.question, // Preserve original question text answer: answer.answer, sources: answer.sources, failedToGenerate: false, }; hasChanges = true; + } else { + console.log('[AutoAnswer] Answer unchanged for question', targetIndex); } } else { // Only update if answer is still null (don't overwrite existing answers) if (!currentAnswer) { + console.log('[AutoAnswer] Marking question as failed to generate', targetIndex); updatedResults[targetIndex] = { ...updatedResults[targetIndex], question: originalQuestion || answer.question, // Preserve original question text @@ -255,21 +377,36 @@ export function useQuestionnaireAutoAnswer({ } }); + if (hasChanges) { + console.log('[AutoAnswer] Results updated successfully'); + } else { + console.log('[AutoAnswer] No changes detected in results'); + } + return hasChanges ? updatedResults : prevResults; }); + } else { + console.log('[AutoAnswer] No answers found to process'); } } }, [ autoAnswerRun?.metadata, answeringQuestionIndex, + questionnaireId, + saveAnswersBatch, // Don't include results, setResults, setQuestionStatuses, setAnsweringQuestionIndex in deps // results is only used for existence check, setState functions are stable ]); - // Handle final completion (for toast notifications and cleanup only) - // UI updates are handled by the metadata watcher above + // Handle final completion - read ALL answers from final output + // This is the primary source of truth since metadata may not be reliable useEffect(() => { - if (autoAnswerRun?.status === 'COMPLETED' && autoAnswerRun.output) { + if ( + autoAnswerRun?.status === 'COMPLETED' && + autoAnswerRun.output && + autoAnswerRun.id && + !processedCompletionRef.current.has(autoAnswerRun.id) + ) { const answers = autoAnswerRun.output.answers as | Array<{ questionIndex: number; @@ -284,44 +421,134 @@ export function useQuestionnaireAutoAnswer({ | undefined; if (answers && Array.isArray(answers)) { - const isSingleQuestion = answeringQuestionIndex !== null; + // Mark this run as processed to prevent infinite loops + processedCompletionRef.current.add(autoAnswerRun.id); - // Mark all remaining "processing" questions as "completed" when orchestrator finishes - // This fixes the issue where some questions stay stuck in "Generating answer..." state - // For single question operations, only mark that specific question as completed - if (results) { - setQuestionStatuses((prev) => { - const newStatuses = new Map(prev); - if (isSingleQuestion && answeringQuestionIndex !== null) { - // Single question: only mark that question as completed - const currentStatus = prev.get(answeringQuestionIndex); - if (currentStatus === 'processing') { - newStatuses.set(answeringQuestionIndex, 'completed'); - } - } else { - // Batch operation: mark all processing questions as completed - results.forEach((qa, index) => { - const currentStatus = prev.get(index); - if (currentStatus === 'processing') { - newStatuses.set(index, 'completed'); - } + console.log('[AutoAnswer] Task completed, updating ALL answers from final output:', { + answersCount: answers.length, + answeredCount: answers.filter((a) => a.answer).length, + runId: autoAnswerRun.id, + }); + + // Update results from final output - merge new answers with existing ones + // The orchestrator only returns answers for questions it processed (unanswered ones) + // So we merge them with existing answers + setResults((prevResults) => { + if (!prevResults) return prevResults; + + const updatedResults = [...prevResults]; + let hasChanges = false; + + // Create a map of new answers by questionIndex for quick lookup + const newAnswersMap = new Map( + answers.map((answer) => [answer.questionIndex, answer]) + ); + + // Update only the questions that were processed (have new answers) + newAnswersMap.forEach((answer, targetIndex) => { + // Safety check: ensure targetIndex is valid + if (targetIndex < 0 || targetIndex >= updatedResults.length) { + console.warn('[AutoAnswer] Invalid questionIndex in final output:', { + targetIndex, + resultsLength: updatedResults.length, }); + return; + } + + const originalQuestion = updatedResults[targetIndex]?.question; + const currentAnswer = updatedResults[targetIndex]?.answer; + + // Update with new answer from orchestrator + if (answer.answer) { + // Only update if answer changed + if (currentAnswer !== answer.answer) { + updatedResults[targetIndex] = { + ...updatedResults[targetIndex], // Preserve status and other fields + question: originalQuestion || answer.question, + answer: answer.answer, + sources: answer.sources, + failedToGenerate: false, + }; + hasChanges = true; + } + } else { + // Mark as failed if no answer was generated (only if it wasn't already answered) + if (!currentAnswer) { + updatedResults[targetIndex] = { + ...updatedResults[targetIndex], + question: originalQuestion || answer.question, + answer: null, + failedToGenerate: true, + }; + hasChanges = true; + } } - return newStatuses; }); + + return hasChanges ? updatedResults : prevResults; + }); + + // Save all answers in batch after final output (defer to avoid rendering issues) + if (questionnaireId) { + const answersToSave = answers + .map((answer) => { + if (answer.answer) { + return { + questionIndex: answer.questionIndex, + answer: answer.answer, + sources: answer.sources, + status: 'generated' as const, + }; + } + return null; + }) + .filter((a): a is NonNullable => a !== null); + + if (answersToSave.length > 0) { + // Use startTransition to defer the save call to avoid rendering issues + startTransition(() => { + saveAnswersBatch.execute({ + questionnaireId, + answers: answersToSave, + }); + }); + } } - // Cleanup: mark process as finished + const isSingleQuestion = answeringQuestionIndex !== null; + + // Mark all remaining "processing" questions as "completed" when orchestrator finishes + setQuestionStatuses((prev) => { + const newStatuses = new Map(prev); + if (isSingleQuestion && answeringQuestionIndex !== null) { + // Single question: only mark that question as completed + const currentStatus = prev.get(answeringQuestionIndex); + if (currentStatus === 'processing') { + newStatuses.set(answeringQuestionIndex, 'completed'); + } + } else { + // Batch operation: mark all processing questions as completed + answers.forEach((answer) => { + const currentStatus = prev.get(answer.questionIndex); + if (currentStatus === 'processing') { + newStatuses.set(answer.questionIndex, 'completed'); + } + }); + } + return newStatuses; + }); + + // Cleanup: mark process as finished if (!isSingleQuestion) { isAutoAnswerProcessStartedRef.current = false; setIsAutoAnswerProcessStarted(false); } - // Reset answering index and run ID for single questions - if (isSingleQuestion) { - setAnsweringQuestionIndex(null); - currentRunIdRef.current = null; // Clear run ID when operation completes - } + // Reset answering index and run ID for single questions + if (isSingleQuestion) { + setAnsweringQuestionIndex(null); + currentRunIdRef.current = null; + } // Show final toast notification const totalQuestions = answers.length; @@ -350,8 +577,10 @@ export function useQuestionnaireAutoAnswer({ }, [ autoAnswerRun?.status, autoAnswerRun?.output, + autoAnswerRun?.id, answeringQuestionIndex, - results, + questionnaireId, + saveAnswersBatch, setAnsweringQuestionIndex, setQuestionStatuses, setIsAutoAnswerProcessStarted, diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireDetail.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireDetail.ts new file mode 100644 index 000000000..68927d0f6 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireDetail.ts @@ -0,0 +1,529 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState, type Dispatch, type SetStateAction } from 'react'; +import { useQuestionnaireActions } from './useQuestionnaireActions'; +import { useQuestionnaireAutoAnswer } from './useQuestionnaireAutoAnswer'; +import { useQuestionnaireSingleAnswer } from './useQuestionnaireSingleAnswer'; +import type { QuestionAnswer } from '../components/types'; +import { createTriggerToken } from '../actions/create-trigger-token'; +import { updateQuestionnaireAnswer } from '../actions/update-questionnaire-answer'; +import { deleteQuestionnaireAnswer } from '../actions/delete-questionnaire-answer'; +import { useAction } from 'next-safe-action/hooks'; +import { useRouter } from 'next/navigation'; +import { toast } from 'sonner'; +import { usePersistGeneratedAnswers } from './usePersistGeneratedAnswers'; + +interface QuestionnaireQuestionAnswer { + id: string; + question: string; + answer: string | null; + status: 'untouched' | 'generated' | 'manual'; + questionIndex: number; + sources: any; +} + +type QuestionnaireResult = QuestionAnswer & { + originalIndex: number; + questionAnswerId: string; + status: 'untouched' | 'generated' | 'manual'; +}; + +interface UseQuestionnaireDetailProps { + questionnaireId: string; + organizationId: string; + initialQuestions: QuestionnaireQuestionAnswer[]; +} + +export function useQuestionnaireDetail({ + questionnaireId, + organizationId, + initialQuestions, +}: UseQuestionnaireDetailProps) { + const router = useRouter(); + + // Initialize results from database + const [results, setResults] = useState(() => + initialQuestions.map((q) => ({ + question: q.question, + answer: q.answer ?? null, // Preserve null instead of converting to empty string + originalIndex: q.questionIndex, + sources: q.sources ? (Array.isArray(q.sources) ? q.sources : []) : [], + questionAnswerId: q.id, + status: q.status, + failedToGenerate: false, // Initialize failedToGenerate + })) + ); + + // State management (same as useQuestionnaireState) + const [editingIndex, setEditingIndex] = useState(null); + const [editingAnswer, setEditingAnswer] = useState(''); + const [expandedSources, setExpandedSources] = useState>(new Set()); + const [questionStatuses, setQuestionStatuses] = useState>(new Map()); + const [answeringQuestionIndex, setAnsweringQuestionIndex] = useState(null); + const [isAutoAnswerProcessStarted, setIsAutoAnswerProcessStarted] = useState(false); + const [hasClickedAutoAnswer, setHasClickedAutoAnswer] = useState(false); + const [autoAnswerToken, setAutoAnswerToken] = useState(null); + const [singleAnswerToken, setSingleAnswerToken] = useState(null); + const [searchQuery, setSearchQuery] = useState(''); + const isAutoAnswerProcessStartedRef = useRef(false); + + // Refs to capture values for save callback + const saveIndexRef = useRef(null); + const saveAnswerRef = useRef(''); + + // Actions - use callbacks that reference the refs + const updateAnswerAction = useAction(updateQuestionnaireAnswer, { + onSuccess: () => { + // Only handle manual saves (when saveIndexRef is set) + if (saveIndexRef.current !== null) { + const index = saveIndexRef.current; + const answer = saveAnswerRef.current; + + // Update local state optimistically + setResults((prev) => + prev.map((r, i) => { + if (i === index) { + const trimmedAnswer = answer.trim(); + // If answer is empty, reset failedToGenerate and allow auto-fill + // If answer has content, mark as manual + return { + ...r, + answer: trimmedAnswer || null, + status: trimmedAnswer ? ('manual' as const) : ('untouched' as const), + failedToGenerate: false, // Reset failedToGenerate when user saves (even if empty) + }; + } + return r; + }) + ); + + setEditingIndex(null); + setEditingAnswer(''); + toast.success('Answer saved'); + router.refresh(); + + // Reset refs + saveIndexRef.current = null; + saveAnswerRef.current = ''; + } + }, + onError: ({ error }) => { + console.error('Failed to update answer:', error); + if (saveIndexRef.current !== null) { + toast.error(`Failed to save answer: ${error instanceof Error ? error.message : 'Unknown error'}`); + saveIndexRef.current = null; + saveAnswerRef.current = ''; + } + }, + }); + const deleteAnswerAction = useAction(deleteQuestionnaireAnswer); + // Note: We don't use orchestratorAction here - we use triggerAutoAnswer directly from useQuestionnaireAutoAnswer + // This ensures we get a trackable run via useRealtimeTaskTrigger + + // Create trigger tokens + useEffect(() => { + const fetchTokens = async () => { + const [autoTokenResult, singleTokenResult] = await Promise.all([ + createTriggerToken('vendor-questionnaire-orchestrator'), + createTriggerToken('answer-question'), + ]); + + if (autoTokenResult.success && autoTokenResult.token) { + setAutoAnswerToken(autoTokenResult.token); + } + if (singleTokenResult.success && singleTokenResult.token) { + setSingleAnswerToken(singleTokenResult.token); + } + }; + + fetchTokens(); + }, []); + + // Auto-answer hook (same as useQuestionnaireParser) + const autoAnswer = useQuestionnaireAutoAnswer({ + autoAnswerToken, + results: results as QuestionAnswer[] | null, + answeringQuestionIndex, + isAutoAnswerProcessStarted, + isAutoAnswerProcessStartedRef, + setIsAutoAnswerProcessStarted, + setResults: setResults as Dispatch>, + setQuestionStatuses, + setAnsweringQuestionIndex, + questionnaireId, + }); + + // Wrapper for setResults that handles QuestionnaireResult[] with originalIndex + const setResultsWrapper = useCallback((updater: React.SetStateAction) => { + setResults((prevResults) => { + if (!prevResults) { + const newResults = typeof updater === 'function' ? updater(null) : updater; + if (!newResults) return prevResults; // Return empty array instead of null + // Convert QuestionAnswer[] to QuestionnaireResult[] + return newResults.map((r, index) => ({ + question: r.question, + answer: r.answer ?? null, // Preserve null instead of converting to empty string + originalIndex: index, + sources: r.sources || [], + questionAnswerId: '', + status: 'untouched' as const, + failedToGenerate: r.failedToGenerate ?? false, // Preserve failedToGenerate + })); + } + + const questionAnswerResults = prevResults.map((r) => ({ + question: r.question, + answer: r.answer, + sources: r.sources, + failedToGenerate: (r as any).failedToGenerate ?? false, // Preserve failedToGenerate from result + _originalIndex: r.originalIndex, // Preserve originalIndex + })); + + const newResults = typeof updater === 'function' + ? updater(questionAnswerResults) + : updater; + + if (!newResults) return prevResults; // Return previous results instead of null + + // Map back to QuestionnaireResult[] preserving originalIndex + return newResults.map((newR, index) => { + const originalIndex = (newR as any)._originalIndex !== undefined + ? (newR as any)._originalIndex + : index; + const existingResult = prevResults.find((r) => r.originalIndex === originalIndex); + if (existingResult) { + return { + ...existingResult, + question: newR.question, + answer: newR.answer ?? null, // Preserve null instead of converting to empty string + sources: newR.sources, + failedToGenerate: newR.failedToGenerate ?? false, // Preserve failedToGenerate + }; + } + // Fallback: create new result (shouldn't happen) + return { + question: newR.question, + answer: newR.answer ?? null, // Preserve null instead of converting to empty string + originalIndex, + sources: newR.sources || [], + questionAnswerId: '', + status: 'untouched' as const, + failedToGenerate: newR.failedToGenerate ?? false, // Preserve failedToGenerate + }; + }); + }); + }, []); + + // Single answer hook (same as useQuestionnaireParser) + const singleAnswer = useQuestionnaireSingleAnswer({ + singleAnswerToken, + results: results.map((r) => ({ + question: r.question, + answer: r.answer, + sources: r.sources, + failedToGenerate: (r as any).failedToGenerate ?? false, // Preserve failedToGenerate from result + _originalIndex: r.originalIndex, // Pass originalIndex for reference + })) as QuestionAnswer[], + answeringQuestionIndex, + setResults: setResultsWrapper, + setQuestionStatuses, + setAnsweringQuestionIndex, + questionnaireId, + }); + + // Expose isSingleAnswerTriggering for isLoading calculation + const isSingleAnswerTriggering = singleAnswer.isSingleAnswerTriggering; + + // Reuse the same actions hook (but adapt it for detail page) + const actions = useQuestionnaireActions({ + orgId: organizationId, + selectedFile: null, + results: results as QuestionAnswer[] | null, + editingAnswer, + expandedSources, + setSelectedFile: () => {}, + setEditingIndex, + setEditingAnswer, + setResults: setResults as Dispatch>, + setExpandedSources, + setIsParseProcessStarted: () => {}, + setIsAutoAnswerProcessStarted, + isAutoAnswerProcessStartedRef, + setHasClickedAutoAnswer, + answeringQuestionIndex, + setAnsweringQuestionIndex, + setQuestionStatuses, + questionnaireId, + setParseTaskId: () => {}, + setParseToken: () => {}, + uploadFileAction: { execute: async () => {}, status: 'idle' as const }, + parseAction: { execute: async () => {}, status: 'idle' as const }, + triggerAutoAnswer: autoAnswer.triggerAutoAnswer, + triggerSingleAnswer: singleAnswer.triggerSingleAnswer, + } as Parameters[0]); + + const persistenceAction = { + execute: () => {}, + executeAsync: (input: Parameters[0]) => + updateAnswerAction.executeAsync(input), + }; + + usePersistGeneratedAnswers({ + questionnaireId, + results: results as QuestionAnswer[] | null, + setResults: setResults as Dispatch>, + autoAnswerRun: autoAnswer.autoAnswerRun ?? null, + updateAnswerAction: persistenceAction as any, + setQuestionStatuses, + }); + + // Override handleAutoAnswer to include orchestrator call + const handleAutoAnswer = () => { + if (answeringQuestionIndex !== null) { + toast.warning('Please wait for the current question to finish before answering all questions'); + return; + } + + setHasClickedAutoAnswer(true); + setIsAutoAnswerProcessStarted(true); + isAutoAnswerProcessStartedRef.current = true; + + // Filter out questions with manual answers + const questionsToAnswer = results + .filter((r) => r.status !== 'manual' && (!r.answer || r.answer.trim().length === 0)) + .map((r) => ({ + question: r.question, + answer: r.answer || null, + _originalIndex: r.originalIndex, + })); + + if (questionsToAnswer.length === 0) { + toast.info('All questions already have answers'); + setIsAutoAnswerProcessStarted(false); + isAutoAnswerProcessStartedRef.current = false; + return; + } + + // Optimistic UI update: immediately show spinners for all unanswered questions + setQuestionStatuses((prev) => { + const newStatuses = new Map(prev); + questionsToAnswer.forEach((q) => { + if (q._originalIndex !== undefined) { + newStatuses.set(q._originalIndex, 'processing'); + } + }); + return newStatuses; + }); + + // Call triggerAutoAnswer directly - this will trigger the task AND give us a trackable run + // The useRealtimeTaskTrigger hook will handle the actual task triggering + try { + autoAnswer.triggerAutoAnswer({ + vendorId: `org_${organizationId}`, + organizationId, + questionsAndAnswers: questionsToAnswer.map((q) => ({ + question: q.question, + answer: q.answer, + _originalIndex: q._originalIndex, // Pass _originalIndex for orchestrator to use + })) as any, + }); + console.log('Triggered auto-answer, run should be available soon'); + } catch (error) { + console.error('Failed to trigger auto-answer:', error); + toast.error('Failed to start auto-answer process'); + setIsAutoAnswerProcessStarted(false); + isAutoAnswerProcessStartedRef.current = false; + // Reset question statuses + setQuestionStatuses((prev) => { + const newStatuses = new Map(prev); + questionsToAnswer.forEach((q) => { + if (q._originalIndex !== undefined) { + newStatuses.delete(q._originalIndex); + } + }); + return newStatuses; + }); + } + }; + + // Override handleAnswerSingleQuestion to set processing status immediately + const handleAnswerSingleQuestion = (index: number) => { + if (isAutoAnswerProcessStarted || answeringQuestionIndex !== null) { + return; + } + + const result = results.find((r) => r.originalIndex === index); + if (!result) { + return; + } + + // Allow auto-fill even if status is 'manual' - user may have cleared the answer + // Only prevent if there's already an answer (non-empty) + if (result.status === 'manual' && result.answer && result.answer.trim().length > 0) { + return; + } + + setAnsweringQuestionIndex(index); + + // Set status to processing immediately to show spinner + setQuestionStatuses((prev) => { + const newStatuses = new Map(prev); + newStatuses.set(index, 'processing'); + return newStatuses; + }); + + singleAnswer.triggerSingleAnswer({ + question: result.question, + organizationId, + questionIndex: index, + totalQuestions: results.length, + }); + }; + + // Handle delete manual answer + const handleDeleteAnswer = async (questionAnswerId: string, questionIndex: number) => { + try { + await deleteAnswerAction.execute({ + questionnaireId, + questionAnswerId, + }); + + setResults((prev) => + prev.map((r) => + r.originalIndex === questionIndex + ? { ...r, answer: '', status: 'untouched' as const } + : r + ) + ); + + toast.success('Answer deleted. You can now generate a new answer.'); + router.refresh(); + } catch (error) { + console.error('Failed to delete answer:', error); + toast.error('Failed to delete answer'); + } + }; + + // Handle save answer (override to save to database) + const handleSaveAnswer = (index: number) => { + // The index passed is the array index from QuestionnaireResultsTable + // Find the result at that array position + const result = results[index]; + + if (!result) { + console.error('Cannot save answer: result not found at index', { + index, + resultsLength: results.length, + results: results.map((r, i) => ({ i, originalIndex: r.originalIndex, questionAnswerId: r.questionAnswerId })) + }); + toast.error('Cannot find question to save'); + return; + } + + if (!result.questionAnswerId) { + console.error('Cannot save answer: questionAnswerId not found', { index, result }); + toast.error('Cannot save answer: missing question ID'); + return; + } + + // Store values in refs for the callback + saveIndexRef.current = index; + saveAnswerRef.current = editingAnswer; + + // Execute the save - callbacks will use the refs + updateAnswerAction.execute({ + questionnaireId, + questionAnswerId: result.questionAnswerId, + answer: editingAnswer.trim(), + }); + }; + + const filteredResults = useMemo(() => { + if (!searchQuery.trim()) return results; + const query = searchQuery.toLowerCase(); + return results.filter( + (r) => + r.question.toLowerCase().includes(query) || + (r.answer && r.answer.toLowerCase().includes(query)) + ); + }, [results, searchQuery]); + + const answeredCount = useMemo(() => { + return results.filter((r) => r.answer && r.answer.trim().length > 0).length; + }, [results]); + + const progressPercentage = useMemo(() => { + if (results.length === 0) return 0; + return Math.round((answeredCount / results.length) * 100); + }, [answeredCount, results.length]); + + const isAutoAnswering = useMemo(() => { + return ( + isAutoAnswerProcessStarted && + hasClickedAutoAnswer && + (autoAnswer.autoAnswerRun?.status === 'EXECUTING' || + autoAnswer.autoAnswerRun?.status === 'QUEUED' || + autoAnswer.autoAnswerRun?.status === 'WAITING') + ); + }, [isAutoAnswerProcessStarted, hasClickedAutoAnswer, autoAnswer.autoAnswerRun?.status]); + + // Calculate isLoading based on answer generation status + const isLoading = useMemo(() => { + // Check if any question is being processed + const hasProcessingQuestions = Array.from(questionStatuses.values()).some( + (status) => status === 'processing' + ); + + // Check if single answer is being triggered + const isSingleAnswerTriggering = singleAnswer.isSingleAnswerTriggering; + + // Check if auto answer is being triggered or running + const isAutoAnswerTriggering = autoAnswer.isAutoAnswerTriggering; + const isAutoAnswerRunActive = + autoAnswer.autoAnswerRun?.status === 'EXECUTING' || + autoAnswer.autoAnswerRun?.status === 'QUEUED' || + autoAnswer.autoAnswerRun?.status === 'WAITING'; + + return hasProcessingQuestions || isSingleAnswerTriggering || isAutoAnswerTriggering || isAutoAnswerRunActive; + }, [ + questionStatuses, + isSingleAnswerTriggering, + autoAnswer.isAutoAnswerTriggering, + autoAnswer.autoAnswerRun?.status, + ]); + + // Check if saving is in progress + const isSaving = updateAnswerAction.status === 'executing'; + const savingIndex = isSaving && saveIndexRef.current !== null ? saveIndexRef.current : null; + + return { + orgId: organizationId, + results, + searchQuery, + setSearchQuery, + editingIndex, + editingAnswer, + setEditingAnswer, + expandedSources, + questionStatuses, + answeringQuestionIndex, + hasClickedAutoAnswer, + isLoading, + isAutoAnswering, + isExporting: actions.exportAction.status === 'executing', + isSaving, + savingIndex, + filteredResults, + answeredCount, + totalCount: results.length, + progressPercentage, + handleAutoAnswer, + handleAnswerSingleQuestion, + handleEditAnswer: actions.handleEditAnswer, + handleSaveAnswer, + handleCancelEdit: actions.handleCancelEdit, + handleExport: actions.handleExport, + handleToggleSource: actions.handleToggleSource, + }; +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParse.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParse.ts index 580cfde90..6941778d4 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParse.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParse.ts @@ -3,6 +3,7 @@ import type { parseQuestionnaireTask } from '@/jobs/tasks/vendors/parse-questionnaire'; import { useRealtimeRun } from '@trigger.dev/react-hooks'; import { useAction } from 'next-safe-action/hooks'; +import { useRouter } from 'next/navigation'; import { useEffect } from 'react'; import { toast } from 'sonner'; import { createRunReadToken, createTriggerToken } from '../actions/create-trigger-token'; @@ -26,6 +27,8 @@ interface UseQuestionnaireParseProps { React.SetStateAction> >; setHasClickedAutoAnswer: (clicked: boolean) => void; + setQuestionnaireId: (id: string | null) => void; + orgId: string; } export function useQuestionnaireParse({ @@ -42,7 +45,10 @@ export function useQuestionnaireParse({ setExtractedContent, setQuestionStatuses, setHasClickedAutoAnswer, + setQuestionnaireId, + orgId, }: UseQuestionnaireParseProps) { + const router = useRouter(); // Get trigger token for auto-answer (can trigger and read) useEffect(() => { async function getAutoAnswerToken() { @@ -86,6 +92,7 @@ export function useQuestionnaireParse({ }> | undefined; const extractedContent = run.output.extractedContent as string | undefined; + const questionnaireId = run.output.questionnaireId as string | undefined; if (questionsAndAnswers && Array.isArray(questionsAndAnswers)) { const initializedResults = questionsAndAnswers.map((qa) => ({ @@ -96,6 +103,13 @@ export function useQuestionnaireParse({ setExtractedContent(extractedContent || null); setQuestionStatuses(new Map()); setHasClickedAutoAnswer(false); + if (questionnaireId) { + setQuestionnaireId(questionnaireId); + // Redirect to questionnaire detail page after successful parse + setTimeout(() => { + router.push(`/${orgId}/security-questionnaire/${questionnaireId}`); + }, 500); // Small delay to show success toast + } toast.success( `Successfully parsed ${questionsAndAnswers.length} question-answer pairs`, ); diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParser.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParser.ts index c63c36974..ebe54cb8a 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParser.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireParser.ts @@ -24,6 +24,8 @@ export function useQuestionnaireParser() { setExtractedContent: state.setExtractedContent, setQuestionStatuses: state.setQuestionStatuses, setHasClickedAutoAnswer: state.setHasClickedAutoAnswer, + setQuestionnaireId: state.setQuestionnaireId, + orgId: state.orgId, }); const autoAnswer = useQuestionnaireAutoAnswer({ @@ -36,6 +38,7 @@ export function useQuestionnaireParser() { setResults: state.setResults, setQuestionStatuses: state.setQuestionStatuses, setAnsweringQuestionIndex: state.setAnsweringQuestionIndex, + questionnaireId: state.questionnaireId, }); const singleAnswer = useQuestionnaireSingleAnswer({ @@ -45,6 +48,7 @@ export function useQuestionnaireParser() { setResults: state.setResults, setQuestionStatuses: state.setQuestionStatuses, setAnsweringQuestionIndex: state.setAnsweringQuestionIndex, + questionnaireId: state.questionnaireId, }); const actions = useQuestionnaireActions({ @@ -53,6 +57,7 @@ export function useQuestionnaireParser() { results: state.results, editingAnswer: state.editingAnswer, expandedSources: state.expandedSources, + questionnaireId: state.questionnaireId, setSelectedFile: state.setSelectedFile, setEditingIndex: state.setEditingIndex, setEditingAnswer: state.setEditingAnswer, diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireSingleAnswer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireSingleAnswer.ts index 4df51913c..2189ba06e 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireSingleAnswer.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireSingleAnswer.ts @@ -2,8 +2,10 @@ import { useRealtimeTaskTrigger } from '@trigger.dev/react-hooks'; import type { answerQuestion } from '@/jobs/tasks/vendors/answer-question'; -import { useEffect } from 'react'; +import { useEffect, useTransition } from 'react'; import { toast } from 'sonner'; +import { useAction } from 'next-safe-action/hooks'; +import { saveAnswerAction } from '../actions/save-answer'; import type { QuestionAnswer } from '../components/types'; interface UseQuestionnaireSingleAnswerProps { @@ -15,6 +17,7 @@ interface UseQuestionnaireSingleAnswerProps { React.SetStateAction> >; setAnsweringQuestionIndex: (index: number | null) => void; + questionnaireId: string | null; } export function useQuestionnaireSingleAnswer({ @@ -24,6 +27,7 @@ export function useQuestionnaireSingleAnswer({ setResults, setQuestionStatuses, setAnsweringQuestionIndex, + questionnaireId, }: UseQuestionnaireSingleAnswerProps) { // Use realtime task trigger for single question answer const { @@ -36,6 +40,39 @@ export function useQuestionnaireSingleAnswer({ enabled: !!singleAnswerToken, }); + // Action for saving answer + const saveAnswer = useAction(saveAnswerAction, { + onError: ({ error }) => { + console.error('Error saving answer:', error); + }, + }); + + const [isPending, startTransition] = useTransition(); + + // Set status to processing when task starts or is triggering + useEffect(() => { + if (answeringQuestionIndex !== null) { + const shouldBeProcessing = + isSingleAnswerTriggering || + singleAnswerRun?.status === 'EXECUTING' || + singleAnswerRun?.status === 'QUEUED' || + singleAnswerRun?.status === 'WAITING'; + + if (shouldBeProcessing) { + setQuestionStatuses((prev) => { + const newStatuses = new Map(prev); + // Ensure status is set to processing when task is running or triggering + const currentStatus = prev.get(answeringQuestionIndex); + if (currentStatus !== 'processing') { + newStatuses.set(answeringQuestionIndex, 'processing'); + return newStatuses; + } + return prev; + }); + } + } + }, [singleAnswerRun?.status, answeringQuestionIndex, isSingleAnswerTriggering, setQuestionStatuses]); + // Handle single answer completion useEffect(() => { if (singleAnswerRun?.status === 'COMPLETED' && singleAnswerRun.output && answeringQuestionIndex !== null) { @@ -57,26 +94,61 @@ export function useQuestionnaireSingleAnswer({ } if (output.success && output.answer) { - // Update the results with the answer - setResults((prevResults) => { - if (!prevResults) return prevResults; - - const updatedResults = [...prevResults]; - const targetIndex = output.questionIndex; - - // Verify we're updating the correct question - if (targetIndex === answeringQuestionIndex && targetIndex >= 0 && targetIndex < updatedResults.length) { - updatedResults[targetIndex] = { - question: updatedResults[targetIndex].question, // Preserve original question text - answer: output.answer, - sources: output.sources, - failedToGenerate: false, - }; - return updatedResults; + const targetIndex = output.questionIndex; + + // Verify we're updating the correct question + if (targetIndex === answeringQuestionIndex && targetIndex >= 0) { + // Update the results with the answer + setResults((prevResults) => { + if (!prevResults) return prevResults; + + const updatedResults = [...prevResults]; + + // Try to find by questionIndex first (for QuestionnaireResult with originalIndex) + // Otherwise use array index + let resultIndex = -1; + for (let i = 0; i < updatedResults.length; i++) { + const result = updatedResults[i] as any; + if (result.originalIndex === targetIndex || result._originalIndex === targetIndex) { + resultIndex = i; + break; + } + } + + // Fallback to array index if not found by originalIndex + if (resultIndex === -1 && targetIndex < updatedResults.length) { + resultIndex = targetIndex; + } + + if (resultIndex >= 0 && resultIndex < updatedResults.length) { + const existingResult = updatedResults[resultIndex]; + updatedResults[resultIndex] = { + ...existingResult, + question: existingResult.question, // Preserve original question text + answer: output.answer, + sources: output.sources, + failedToGenerate: false, + }; + return updatedResults; + } + + return prevResults; + }); + + // Save answer to database (outside of setState) + if (questionnaireId && output.answer) { + // Use startTransition to defer the save call to avoid rendering issues + startTransition(() => { + saveAnswer.execute({ + questionnaireId, + questionIndex: targetIndex, + answer: output.answer!, + sources: output.sources, + status: 'generated', + }); + }); } - - return prevResults; - }); + } // Mark question as completed setQuestionStatuses((prev) => { @@ -121,7 +193,7 @@ export function useQuestionnaireSingleAnswer({ // Reset answering index setAnsweringQuestionIndex(null); } - }, [singleAnswerRun?.status, singleAnswerRun?.output, answeringQuestionIndex, setResults, setQuestionStatuses, setAnsweringQuestionIndex]); + }, [singleAnswerRun?.status, singleAnswerRun?.output, answeringQuestionIndex, questionnaireId, saveAnswer, setResults, setQuestionStatuses, setAnsweringQuestionIndex]); // Handle single answer errors useEffect(() => { diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireState.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireState.ts index fdd8c1aa3..327049809 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireState.ts +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/hooks/useQuestionnaireState.ts @@ -27,6 +27,7 @@ export function useQuestionnaireState() { const [singleAnswerToken, setSingleAnswerToken] = useState(null); const [isParseProcessStarted, setIsParseProcessStarted] = useState(false); const [isAutoAnswerProcessStarted, setIsAutoAnswerProcessStarted] = useState(false); + const [questionnaireId, setQuestionnaireId] = useState(null); const isAutoAnswerProcessStartedRef = useRef(false); const resetState = () => { @@ -42,6 +43,7 @@ export function useQuestionnaireState() { setIsParseProcessStarted(false); isAutoAnswerProcessStartedRef.current = false; setIsAutoAnswerProcessStarted(false); + setQuestionnaireId(null); }; return { @@ -81,6 +83,8 @@ export function useQuestionnaireState() { isAutoAnswerProcessStarted, setIsAutoAnswerProcessStarted, isAutoAnswerProcessStartedRef, + questionnaireId, + setQuestionnaireId, resetState, }; } diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/delete-document.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/delete-document.ts new file mode 100644 index 000000000..2fbda7066 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/delete-document.ts @@ -0,0 +1,116 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; +import { db } from '@db'; +import { DeleteObjectCommand } from '@aws-sdk/client-s3'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; +import { tasks } from '@trigger.dev/sdk'; +import { deleteKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/delete-knowledge-base-document'; + +const deleteDocumentSchema = z.object({ + documentId: z.string(), +}); + +export const deleteKnowledgeBaseDocumentAction = authActionClient + .inputSchema(deleteDocumentSchema) + .metadata({ + name: 'delete-knowledge-base-document', + track: { + event: 'delete-knowledge-base-document', + description: 'Delete Knowledge Base Document', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { documentId } = parsedInput; + const { activeOrganizationId } = ctx.session; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + if (!s3Client) { + return { + success: false, + error: 'S3 client not configured', + }; + } + + if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { + return { + success: false, + error: 'Knowledge base bucket is not configured', + }; + } + + try { + // Find the document + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: documentId, + organizationId: activeOrganizationId, + }, + }); + + if (!document) { + return { + success: false, + error: 'Document not found', + }; + } + + // Delete embeddings from vector database first (async, non-blocking) + let vectorDeletionRunId: string | undefined; + try { + const handle = await tasks.trigger( + 'delete-knowledge-base-document-from-vector', + { + documentId: document.id, + organizationId: activeOrganizationId, + }, + ); + vectorDeletionRunId = handle.id; + } catch (triggerError) { + // Log error but continue with deletion + console.error('Failed to trigger vector deletion task:', triggerError); + } + + // Delete from S3 + try { + const deleteCommand = new DeleteObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, + Key: document.s3Key, + }); + await s3Client.send(deleteCommand); + } catch (s3Error) { + // Log error but continue with database deletion + console.error('Error deleting file from S3:', s3Error); + } + + // Delete from database + await db.knowledgeBaseDocument.delete({ + where: { + id: documentId, + }, + }); + + revalidatePath(`/${activeOrganizationId}/security-questionnaire/knowledge-base`); + + return { + success: true, + vectorDeletionRunId, // Return run ID for tracking deletion progress + }; + } catch (error) { + console.error('Error deleting knowledge base document:', error); + return { + success: false, + error: 'Failed to delete document', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/download-document.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/download-document.ts new file mode 100644 index 000000000..31e1dbd26 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/download-document.ts @@ -0,0 +1,95 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; +import { db } from '@db'; +import { GetObjectCommand } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { z } from 'zod'; + +const downloadDocumentSchema = z.object({ + documentId: z.string(), +}); + +export const downloadKnowledgeBaseDocumentAction = authActionClient + .inputSchema(downloadDocumentSchema) + .metadata({ + name: 'download-knowledge-base-document', + track: { + event: 'download-knowledge-base-document', + description: 'Download Knowledge Base Document', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { documentId } = parsedInput; + const { session } = ctx; + + if (!session?.activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + if (!s3Client) { + return { + success: false, + error: 'S3 client not configured', + }; + } + + if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { + return { + success: false, + error: 'Knowledge base bucket is not configured', + }; + } + + try { + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: documentId, + organizationId: session.activeOrganizationId, + }, + select: { + s3Key: true, + name: true, + fileType: true, + }, + }); + + if (!document) { + return { + success: false, + error: 'Document not found', + }; + } + + // Generate signed URL for download + const command = new GetObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, + Key: document.s3Key, + ResponseContentDisposition: `attachment; filename="${encodeURIComponent(document.name)}"`, + }); + + const signedUrl = await getSignedUrl(s3Client, command, { + expiresIn: 3600, // URL expires in 1 hour + }); + + return { + success: true, + data: { + signedUrl, + fileName: document.name, + }, + }; + } catch (error) { + console.error('Error generating download URL:', error); + return { + success: false, + error: 'Failed to generate download URL', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts new file mode 100644 index 000000000..b991568d9 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/get-document-view-url.ts @@ -0,0 +1,119 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; +import { db } from '@db'; +import { GetObjectCommand } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { z } from 'zod'; + +const getDocumentViewUrlSchema = z.object({ + documentId: z.string(), +}); + +/** + * Gets a signed URL for viewing a knowledge base document (opens in browser, doesn't force download) + */ +export const getKnowledgeBaseDocumentViewUrlAction = authActionClient + .inputSchema(getDocumentViewUrlSchema) + .metadata({ + name: 'get-knowledge-base-document-view-url', + track: { + event: 'get-knowledge-base-document-view-url', + description: 'Get Knowledge Base Document View URL', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { documentId } = parsedInput; + const { session } = ctx; + + if (!session?.activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + if (!s3Client) { + return { + success: false, + error: 'S3 client not configured', + }; + } + + if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { + return { + success: false, + error: 'Knowledge base bucket is not configured', + }; + } + + try { + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: documentId, + organizationId: session.activeOrganizationId, + }, + select: { + s3Key: true, + name: true, + fileType: true, + }, + }); + + if (!document) { + return { + success: false, + error: 'Document not found', + }; + } + + // Generate signed URL for viewing in browser + // Set Content-Type header so browser knows how to handle the file + // For PDFs, images, and text files: browser will display inline + // For DOCX, XLSX, etc.: browser may download or try to open with external app + const command = new GetObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, + Key: document.s3Key, + ResponseContentDisposition: `inline; filename="${encodeURIComponent(document.name)}"`, + ResponseContentType: document.fileType || 'application/octet-stream', // Set Content-Type header + }); + + const signedUrl = await getSignedUrl(s3Client, command, { + expiresIn: 3600, // URL expires in 1 hour + }); + + // Determine if file can be viewed inline in browser + const viewableInBrowser = [ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/gif', + 'image/webp', + 'image/svg+xml', + 'text/plain', + 'text/html', + 'text/csv', + 'text/markdown', + ].includes(document.fileType); + + return { + success: true, + data: { + signedUrl, + fileName: document.name, + fileType: document.fileType, + viewableInBrowser, + }, + }; + } catch (error) { + console.error('Error generating view URL:', error); + return { + success: false, + error: 'Failed to generate view URL', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/process-documents.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/process-documents.ts new file mode 100644 index 000000000..f66ceed12 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/process-documents.ts @@ -0,0 +1,78 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { tasks } from '@trigger.dev/sdk'; +import { processKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/process-knowledge-base-document'; +import { processKnowledgeBaseDocumentsOrchestratorTask } from '@/jobs/tasks/vector/process-knowledge-base-documents-orchestrator'; +import { z } from 'zod'; + +const processDocumentsSchema = z.object({ + documentIds: z.array(z.string()).min(1), + organizationId: z.string(), +}); + +/** + * Server action to trigger document processing + * Uses orchestrator for multiple documents, individual task for single document + */ +export const processKnowledgeBaseDocumentsAction = authActionClient + .inputSchema(processDocumentsSchema) + .metadata({ + name: 'process-knowledge-base-documents', + track: { + event: 'process-knowledge-base-documents', + description: 'Process Knowledge Base Documents', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { documentIds, organizationId } = parsedInput; + const { session } = ctx; + + if (!session?.activeOrganizationId || session.activeOrganizationId !== organizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + let runId: string | undefined; + + // Use orchestrator for multiple documents, individual task for single document + if (documentIds.length > 1) { + const handle = await tasks.trigger( + 'process-knowledge-base-documents-orchestrator', + { + documentIds, + organizationId, + }, + ); + runId = handle.id; + } else { + const handle = await tasks.trigger( + 'process-knowledge-base-document', + { + documentId: documentIds[0]!, + organizationId, + }, + ); + runId = handle.id; + } + + return { + success: true, + runId, + message: documentIds.length > 1 + ? `Processing ${documentIds.length} documents in parallel...` + : 'Processing document...', + }; + } catch (error) { + console.error('Failed to trigger document processing:', error); + return { + success: false, + error: 'Failed to trigger document processing', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/upload-document.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/upload-document.ts new file mode 100644 index 000000000..66108a77a --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/actions/upload-document.ts @@ -0,0 +1,132 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { APP_AWS_KNOWLEDGE_BASE_BUCKET, s3Client } from '@/app/s3'; +import { db } from '@db'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { randomBytes } from 'crypto'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; + +const uploadDocumentSchema = z.object({ + fileName: z.string(), + fileType: z.string(), + fileData: z.string(), // base64 encoded file + description: z.string().optional(), + organizationId: z.string(), +}); + +export const uploadKnowledgeBaseDocumentAction = authActionClient + .inputSchema(uploadDocumentSchema) + .metadata({ + name: 'upload-knowledge-base-document', + track: { + event: 'upload-knowledge-base-document', + description: 'Upload Knowledge Base Document', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { fileName, fileType, fileData, description, organizationId } = parsedInput; + const { session } = ctx; + + if (!session?.activeOrganizationId || session.activeOrganizationId !== organizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + if (!s3Client) { + return { + success: false, + error: 'S3 client not configured', + }; + } + + if (!APP_AWS_KNOWLEDGE_BASE_BUCKET) { + return { + success: false, + error: 'Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable.', + }; + } + + try { + // Convert base64 to buffer + const fileBuffer = Buffer.from(fileData, 'base64'); + + // Validate file size (10MB limit) + const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; + if (fileBuffer.length > MAX_FILE_SIZE_BYTES) { + return { + success: false, + error: `File exceeds the ${MAX_FILE_SIZE_BYTES / (1024 * 1024)}MB limit`, + }; + } + + // Generate unique file key + const fileId = randomBytes(16).toString('hex'); + const sanitizedFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_'); + const timestamp = Date.now(); + const s3Key = `${organizationId}/knowledge-base-documents/${timestamp}-${fileId}-${sanitizedFileName}`; + + // Sanitize filename for S3 metadata + // S3 metadata values must be valid HTTP header values + // To be absolutely safe, we'll encode the filename using a safe character set + // Remove control characters and non-ASCII characters, keep only safe printable ASCII + const sanitizedMetadataFileName = Buffer.from(fileName, 'utf8') + .toString('ascii') // Convert to ASCII, replacing non-ASCII with '?' + .replace(/[\x00-\x1F\x7F]/g, '') // Remove control characters + .replace(/\?/g, '_') // Replace '?' (from non-ASCII conversion) with '_' + .trim() + .substring(0, 1024); // S3 metadata values have a 2KB limit per value + + // Upload to S3 + const putCommand = new PutObjectCommand({ + Bucket: APP_AWS_KNOWLEDGE_BASE_BUCKET, + Key: s3Key, + Body: fileBuffer, + ContentType: fileType, + Metadata: { + originalFileName: sanitizedMetadataFileName, + organizationId, + }, + }); + + await s3Client.send(putCommand); + + // Create database record + const document = await db.knowledgeBaseDocument.create({ + data: { + name: fileName, + description: description || null, + s3Key, + fileType, + fileSize: fileBuffer.length, + organizationId, + processingStatus: 'pending', + }, + }); + + // Note: Processing is triggered by orchestrator in the component + // when multiple files are uploaded, or individually for single files + + revalidatePath(`/${organizationId}/security-questionnaire/knowledge-base`); + + return { + success: true, + data: { + id: document.id, + name: document.name, + s3Key: document.s3Key, + }, + }; + } catch (error) { + console.error('Error uploading knowledge base document:', error); + return { + success: false, + error: 'Failed to upload document', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx new file mode 100644 index 000000000..05ec51f4c --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/AdditionalDocumentsSection.tsx @@ -0,0 +1,436 @@ +'use client'; + +import { FileUploader } from '@/components/file-uploader'; +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@comp/ui/accordion'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@comp/ui/alert-dialog'; +import { Button } from '@comp/ui/button'; +import { Card } from '@comp/ui'; +import { ChevronLeft, ChevronRight, Download, FileText, Trash2, Upload } from 'lucide-react'; +import { useState, useRef } from 'react'; +import { toast } from 'sonner'; +import { uploadKnowledgeBaseDocumentAction } from '../actions/upload-document'; +import { downloadKnowledgeBaseDocumentAction } from '../actions/download-document'; +import { deleteKnowledgeBaseDocumentAction } from '../actions/delete-document'; +import { processKnowledgeBaseDocumentsAction } from '../actions/process-documents'; +import { useRouter } from 'next/navigation'; +import { usePagination } from '../../hooks/usePagination'; +import { format } from 'date-fns'; +import { useDocumentProcessing } from '../hooks/useDocumentProcessing'; +import { Loader2 } from 'lucide-react'; + +type KnowledgeBaseDocument = Awaited< + ReturnType +>[number]; + +interface AdditionalDocumentsSectionProps { + organizationId: string; + documents: Awaited>; +} + +export function AdditionalDocumentsSection({ + organizationId, + documents, +}: AdditionalDocumentsSectionProps) { + const router = useRouter(); + const sectionRef = useRef(null); + const [isUploading, setIsUploading] = useState(false); + const [uploadProgress, setUploadProgress] = useState>({}); + const [downloadingIds, setDownloadingIds] = useState>(new Set()); + const [deletingId, setDeletingId] = useState(null); + const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false); + const [documentToDelete, setDocumentToDelete] = useState<{ id: string; name: string } | null>( + null, + ); + + // Track processing and deletion run IDs + const [processingRunIds, setProcessingRunIds] = useState>(new Map()); // documentId -> runId + const [deletionRunIds, setDeletionRunIds] = useState>(new Map()); // documentId -> runId + + // Track processing/deletion progress for current document + const currentProcessingRunId = Array.from(processingRunIds.values())[0] || null; + const currentDeletionRunId = deletionRunIds.get(deletingId || '') || null; + + const { isProcessing, isDeleting, processingStatus, deletionStatus } = useDocumentProcessing({ + processingRunId: currentProcessingRunId, + deletionRunId: currentDeletionRunId, + onProcessingComplete: () => { + // Clear processing run ID and refresh + setProcessingRunIds(new Map()); + router.refresh(); + toast.success('Document processed successfully'); + }, + onDeletionComplete: () => { + // Clear deletion run ID + const newDeletionRunIds = new Map(deletionRunIds); + if (deletingId) { + newDeletionRunIds.delete(deletingId); + } + setDeletionRunIds(newDeletionRunIds); + }, + }); + + const { currentPage, totalPages, paginatedItems, handlePageChange } = usePagination({ + items: documents, + itemsPerPage: 10, + }); + + const handleAccordionChange = (value: string) => { + // If opening (value is set), scroll to section + if (value === 'additional-documents' && sectionRef.current) { + setTimeout(() => { + sectionRef.current?.scrollIntoView({ + behavior: 'smooth', + block: 'start', + }); + }, 100); // Small delay to allow accordion animation to start + } + }; + + const handleFileUpload = async (files: File[]) => { + setIsUploading(true); + const newProgress: Record = {}; + + try { + // Initialize progress for all files + files.forEach((file) => { + newProgress[file.name] = 0; + }); + setUploadProgress(newProgress); + + const uploadedDocumentIds: string[] = []; + + // Upload files sequentially + for (const file of files) { + try { + // Convert file to base64 + const fileData = await fileToBase64(file); + + // Update progress + newProgress[file.name] = 50; + setUploadProgress({ ...newProgress }); + + // Upload file + const result = await uploadKnowledgeBaseDocumentAction({ + fileName: file.name, + fileType: file.type, + fileData, + organizationId, + }); + + if (result?.data?.success && result.data.data?.id) { + uploadedDocumentIds.push(result.data.data.id); + newProgress[file.name] = 100; + setUploadProgress({ ...newProgress }); + toast.success(`Successfully uploaded ${file.name}`); + } else { + throw new Error(result?.data?.error || 'Failed to upload file'); + } + } catch (error) { + console.error(`Error uploading ${file.name}:`, error); + toast.error( + `Failed to upload ${file.name}: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + delete newProgress[file.name]; + setUploadProgress({ ...newProgress }); + } + } + + // Trigger processing for uploaded documents (orchestrator for multiple, individual for single) + if (uploadedDocumentIds.length > 0) { + try { + const result = await processKnowledgeBaseDocumentsAction({ + documentIds: uploadedDocumentIds, + organizationId, + }); + + if (result?.data?.success) { + // Store run ID for tracking progress + const runId = result.data.runId; + if (runId) { + const newProcessingRunIds = new Map(processingRunIds); + // For orchestrator, track all documents with the same run ID + uploadedDocumentIds.forEach((docId) => { + newProcessingRunIds.set(docId, runId); + }); + setProcessingRunIds(newProcessingRunIds); + } + toast.success(result.data.message || 'Processing documents...'); + } else { + console.error('Failed to trigger document processing:', result?.data?.error); + } + } catch (error) { + console.error('Failed to trigger document processing:', error); + } + } + + // Refresh the page to show new documents + router.refresh(); + } catch (error) { + console.error('Error during file upload:', error); + toast.error('An error occurred during file upload'); + } finally { + setIsUploading(false); + setUploadProgress({}); + } + }; + + const handleDownload = async (documentId: string, fileName: string, e?: React.MouseEvent) => { + if (e) { + e.stopPropagation(); + } + + if (downloadingIds.has(documentId)) { + return; + } + + setDownloadingIds((prev) => new Set(prev).add(documentId)); + + try { + const result = await downloadKnowledgeBaseDocumentAction({ documentId }); + + if (result?.data?.success && result.data.data?.signedUrl) { + // Create a temporary link and trigger download + const link = document.createElement('a'); + link.href = result.data.data.signedUrl; + link.download = fileName; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + toast.success(`Downloading ${fileName}...`); + } else { + toast.error(result?.data?.error || 'Failed to download file'); + } + } catch (error) { + console.error('Error downloading file:', error); + toast.error('An error occurred while downloading the file'); + } finally { + setDownloadingIds((prev) => { + const newSet = new Set(prev); + newSet.delete(documentId); + return newSet; + }); + } + }; + + const handleDeleteClick = (documentId: string, fileName: string, e: React.MouseEvent) => { + e.stopPropagation(); + setDocumentToDelete({ id: documentId, name: fileName }); + setIsDeleteDialogOpen(true); + }; + + const handleDeleteConfirm = async () => { + if (!documentToDelete) return; + + setDeletingId(documentToDelete.id); + setIsDeleteDialogOpen(false); + + try { + const result = await deleteKnowledgeBaseDocumentAction({ + documentId: documentToDelete.id, + }); + + if (result?.data?.success) { + // Store deletion run ID for tracking progress + const vectorDeletionRunId = result.data.vectorDeletionRunId; + if (vectorDeletionRunId) { + const newDeletionRunIds = new Map(deletionRunIds); + newDeletionRunIds.set(documentToDelete.id, vectorDeletionRunId); + setDeletionRunIds(newDeletionRunIds); + } + + toast.success(`Successfully deleted ${documentToDelete.name}`); + router.refresh(); + } else { + toast.error(result?.data?.error || 'Failed to delete document'); + } + } catch (error) { + console.error('Error deleting document:', error); + toast.error('An error occurred while deleting the document'); + } finally { + setDeletingId(null); + setDocumentToDelete(null); + } + }; + + const fileToBase64 = (file: File): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => { + const result = reader.result as string; + // Remove data URL prefix (e.g., "data:image/png;base64,") + const base64 = result.split(',')[1]; + resolve(base64); + }; + reader.onerror = (error) => reject(error); + }); + }; + + return ( + <> + + + + +
+ + Additional Documents + ({documents.length}) +
+ + +
+

+ Upload documents or images to enhance your knowledge base. Supported formats: PDF, Word (.docx), Excel, CSV, text files, and images (PNG, JPG, GIF, WebP, SVG). Click on a document to download it. +

+
+ + {/* Documents List */} + {documents.length > 0 && ( +
+ {paginatedItems.map((document: KnowledgeBaseDocument) => { + const isDownloading = downloadingIds.has(document.id); + const isDeleting = deletingId === document.id; + const isProcessingDocument = processingRunIds.has(document.id); + const isDeletingVector = deletionRunIds.has(document.id); + const formattedDate = format(new Date(document.createdAt), 'MMM dd, yyyy'); + + return ( +
+
+
!isDownloading && !isDeleting && handleDownload(document.id, document.name)} + className="flex flex-1 cursor-pointer items-center gap-3 min-w-0" + > +
+ +
+
+
+

+ {document.name} +

+ +
+
+ {formattedDate} +
+
+
+ {(isProcessingDocument || isDeletingVector) ? ( +
+ +
+ ) : ( + + )} +
+
+ ); + })} +
+ )} + + {/* File Uploader */} + + + {/* Pagination */} + {totalPages > 1 && ( +
+
+ + + {currentPage} of {totalPages} + + +
+
+ )} + + + + + + {/* Delete Confirmation Dialog */} + + e.stopPropagation()}> + + Delete Document + + Are you sure you want to delete "{documentToDelete?.name}"? This action cannot be undone. + + + + Cancel + + {deletingId ? 'Deleting...' : 'Delete'} + + + + + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/index.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/index.ts new file mode 100644 index 000000000..757fd201b --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/components/index.ts @@ -0,0 +1,2 @@ +export { AdditionalDocumentsSection } from './AdditionalDocumentsSection'; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts new file mode 100644 index 000000000..0ab979a77 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/additional-documents/hooks/useDocumentProcessing.ts @@ -0,0 +1,84 @@ +'use client'; + +import { useRealtimeRun } from '@trigger.dev/react-hooks'; +import { useEffect, useState } from 'react'; +import { createRunReadToken } from '../../../actions/create-trigger-token'; +import type { processKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/process-knowledge-base-document'; +import type { processKnowledgeBaseDocumentsOrchestratorTask } from '@/jobs/tasks/vector/process-knowledge-base-documents-orchestrator'; +import type { deleteKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/delete-knowledge-base-document'; + +interface UseDocumentProcessingOptions { + processingRunId?: string | null; + deletionRunId?: string | null; + onProcessingComplete?: () => void; + onDeletionComplete?: () => void; +} + +export function useDocumentProcessing({ + processingRunId, + deletionRunId, + onProcessingComplete, + onDeletionComplete, +}: UseDocumentProcessingOptions) { + const [processingToken, setProcessingToken] = useState(null); + const [deletionToken, setDeletionToken] = useState(null); + + // Get read token for processing run + useEffect(() => { + async function getProcessingToken() { + if (processingRunId) { + const result = await createRunReadToken(processingRunId); + if (result.success && result.token) { + setProcessingToken(result.token); + } + } + } + getProcessingToken(); + }, [processingRunId]); + + // Get read token for deletion run + useEffect(() => { + async function getDeletionToken() { + if (deletionRunId) { + const result = await createRunReadToken(deletionRunId); + if (result.success && result.token) { + setDeletionToken(result.token); + } + } + } + getDeletionToken(); + }, [deletionRunId]); + + // Track processing run + const { run: processingRun } = useRealtimeRun< + typeof processKnowledgeBaseDocumentTask | typeof processKnowledgeBaseDocumentsOrchestratorTask + >(processingRunId || '', { + accessToken: processingToken || undefined, + enabled: !!processingRunId && !!processingToken, + onComplete: () => { + onProcessingComplete?.(); + }, + }); + + // Track deletion run + const { run: deletionRun } = useRealtimeRun( + deletionRunId || '', + { + accessToken: deletionToken || undefined, + enabled: !!deletionRunId && !!deletionToken, + onComplete: () => { + onDeletionComplete?.(); + }, + }, + ); + + return { + processingRun, + deletionRun, + isProcessing: processingRun?.status === 'EXECUTING' || processingRun?.status === 'QUEUED', + isDeleting: deletionRun?.status === 'EXECUTING' || deletionRun?.status === 'QUEUED', + processingStatus: processingRun?.status, + deletionStatus: deletionRun?.status, + }; +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/BackButton.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/BackButton.tsx new file mode 100644 index 000000000..547bafb43 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/BackButton.tsx @@ -0,0 +1,22 @@ +'use client'; + +import { Button } from '@comp/ui/button'; +import { ArrowLeft } from 'lucide-react'; +import { useRouter } from 'next/navigation'; + +export function BackButton() { + const router = useRouter(); + + return ( + + ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx new file mode 100644 index 000000000..59b15e216 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseBreadcrumb.tsx @@ -0,0 +1,19 @@ +'use client'; + +import { BookOpen } from 'lucide-react'; + +export function KnowledgeBaseBreadcrumb() { + return ( + + ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseHeader.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseHeader.tsx new file mode 100644 index 000000000..9e0c2c422 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/KnowledgeBaseHeader.tsx @@ -0,0 +1,18 @@ +'use client'; + +interface KnowledgeBaseHeaderProps { + organizationId: string; +} + +export function KnowledgeBaseHeader({ organizationId }: KnowledgeBaseHeaderProps) { + return ( +
+

Knowledge Base

+

+ Manage your organization's knowledge base including published policies, context entries, + manual answers, and additional documents. +

+
+ ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/index.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/index.ts new file mode 100644 index 000000000..ad5444742 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/components/index.ts @@ -0,0 +1,3 @@ +export { BackButton } from './BackButton'; +export { KnowledgeBaseBreadcrumb } from './KnowledgeBaseBreadcrumb'; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/ContextSection.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/ContextSection.tsx new file mode 100644 index 000000000..79090b867 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/ContextSection.tsx @@ -0,0 +1,178 @@ +'use client'; + +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@comp/ui/accordion'; +import { Button } from '@comp/ui/button'; +import { Card } from '@comp/ui'; +import { isJSON } from '@/lib/utils'; +import { ChevronLeft, ChevronRight, ExternalLink, MessageSquare } from 'lucide-react'; +import Link from 'next/link'; +import { useParams } from 'next/navigation'; +import { useRef } from 'react'; +import { usePagination } from '../../hooks/usePagination'; + +interface ContextSectionProps { + contextEntries: Awaited>; +} + +function hasValidAnswer(answer: string): boolean { + if (!answer || answer.trim() === '') { + return false; + } + + if (isJSON(answer)) { + try { + const parsed = JSON.parse(answer); + // Check if there are any non-empty values + const hasNonEmptyValue = Object.values(parsed).some((value) => { + if (typeof value === 'string') { + return value.trim() !== ''; + } + return !!value; + }); + return hasNonEmptyValue; + } catch { + return false; + } + } + + return true; +} + +function formatAnswer(answer: string): string { + if (isJSON(answer)) { + try { + const parsed = JSON.parse(answer); + const entries = Object.entries(parsed) + .filter(([key, value]) => { + if (typeof value === 'string') { + return value.trim() !== ''; + } + return !!value; + }) + .map(([key, value]) => `${key}: ${value}`) + .join(', '); + + return entries || ''; + } catch { + return answer; + } + } + + return answer; +} + +export function ContextSection({ contextEntries }: ContextSectionProps) { + const params = useParams(); + const orgId = params.orgId as string; + const sectionRef = useRef(null); + + const validEntries = contextEntries.filter((entry) => hasValidAnswer(entry.answer)); + + const { currentPage, totalPages, paginatedItems, handlePageChange } = usePagination({ + items: validEntries, + itemsPerPage: 5, + }); + + const handleAccordionChange = (value: string) => { + // If opening (value is set), scroll to section + if (value === 'context' && sectionRef.current) { + setTimeout(() => { + sectionRef.current?.scrollIntoView({ + behavior: 'smooth', + block: 'start', + }); + }, 100); // Small delay to allow accordion animation to start + } + }; + + return ( + + + + +
+ + Context + + ({validEntries.length}) + +
+ + + {validEntries.length === 0 ? ( +
+ No context entries found +
+ ) : ( + <> +
+ {paginatedItems.map((entry) => { + const formattedAnswer = formatAnswer(entry.answer); + return ( + +
+
+

+ {entry.question} +

+ {formattedAnswer && ( +

+ {formattedAnswer} +

+ )} +
+ +
+ + ); + })} +
+ + {/* Pagination */} + {totalPages > 1 && ( +
+
+ + + {currentPage} of {totalPages} + + +
+
+ )} + + )} + + + + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/index.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/index.ts new file mode 100644 index 000000000..f7f736347 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/context/components/index.ts @@ -0,0 +1,2 @@ +export { ContextSection } from './ContextSection'; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/data/queries.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/data/queries.ts new file mode 100644 index 000000000..7e84f76eb --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/data/queries.ts @@ -0,0 +1,131 @@ +'use server'; + +import { auth } from '@/utils/auth'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import 'server-only'; + +export const getPublishedPolicies = async (organizationId: string) => { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.session?.activeOrganizationId || session.session.activeOrganizationId !== organizationId) { + return []; + } + + const policies = await db.policy.findMany({ + where: { + organizationId, + status: 'published', + isArchived: false, + }, + select: { + id: true, + name: true, + description: true, + createdAt: true, + updatedAt: true, + }, + orderBy: { + name: 'asc', + }, + }); + + return policies; +}; + +export const getContextEntries = async (organizationId: string) => { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.session?.activeOrganizationId || session.session.activeOrganizationId !== organizationId) { + return []; + } + + const contextEntries = await db.context.findMany({ + where: { + organizationId, + answer: { + not: '', + }, + }, + select: { + id: true, + question: true, + answer: true, + tags: true, + createdAt: true, + updatedAt: true, + }, + orderBy: { + createdAt: 'desc', + }, + }); + + return contextEntries; +}; + +export const getKnowledgeBaseDocuments = async (organizationId: string) => { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.session?.activeOrganizationId || session.session.activeOrganizationId !== organizationId) { + return []; + } + + const documents = await db.knowledgeBaseDocument.findMany({ + where: { + organizationId, + }, + select: { + id: true, + name: true, + description: true, + s3Key: true, + fileType: true, + fileSize: true, + processingStatus: true, + createdAt: true, + updatedAt: true, + }, + orderBy: { + createdAt: 'desc', + }, + }); + + return documents; +}; + +export const getManualAnswers = async (organizationId: string) => { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.session?.activeOrganizationId || session.session.activeOrganizationId !== organizationId) { + return []; + } + + const manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ + where: { + organizationId, + }, + select: { + id: true, + question: true, + answer: true, + tags: true, + sourceQuestionnaireId: true, + createdAt: true, + updatedAt: true, + }, + orderBy: { + updatedAt: 'desc', + }, + }); + + return manualAnswers; +}; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/hooks/usePagination.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/hooks/usePagination.ts new file mode 100644 index 000000000..17da9148e --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/hooks/usePagination.ts @@ -0,0 +1,39 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; + +interface UsePaginationProps { + items: T[]; + itemsPerPage?: number; +} + +export function usePagination({ items, itemsPerPage = 10 }: UsePaginationProps) { + const [currentPage, setCurrentPage] = useState(1); + + // Calculate pagination + const totalPages = Math.max(1, Math.ceil(items.length / itemsPerPage)); + const startIndex = (currentPage - 1) * itemsPerPage; + const endIndex = startIndex + itemsPerPage; + const paginatedItems = useMemo( + () => items.slice(startIndex, endIndex), + [items, startIndex, endIndex], + ); + + const handlePageChange = (page: number) => { + setCurrentPage(page); + }; + + // Reset to page 1 when items change + useEffect(() => { + setCurrentPage(1); + }, [items.length]); + + return { + currentPage, + totalPages, + paginatedItems, + totalItems: items.length, + handlePageChange, + }; +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts new file mode 100644 index 000000000..c992b61b5 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-all-manual-answers.ts @@ -0,0 +1,105 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { tasks } from '@trigger.dev/sdk'; +import { logger } from '@/utils/logger'; +import { z } from 'zod'; + +// Empty schema since this action doesn't need input +const deleteAllManualAnswersSchema = z.object({}); + +export const deleteAllManualAnswers = authActionClient + .inputSchema(deleteAllManualAnswersSchema) + .metadata({ + name: 'delete-all-manual-answers', + track: { + event: 'delete-all-manual-answers', + description: 'Delete All Manual Answers', + channel: 'server', + }, + }) + .action(async ({ ctx }) => { + const { activeOrganizationId } = ctx.session; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // First, get all manual answer IDs BEFORE deletion + // This ensures the orchestrator has the IDs to delete from vector DB + const manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ + where: { + organizationId: activeOrganizationId, + }, + select: { + id: true, + }, + }); + + logger.info('Found manual answers to delete', { + organizationId: activeOrganizationId, + count: manualAnswers.length, + ids: manualAnswers.map((ma) => ma.id), + }); + + // Trigger orchestrator task to delete all manual answers from vector DB in parallel + // Pass the IDs directly to avoid race condition + // This runs in the background and processes deletions efficiently + if (manualAnswers.length > 0) { + try { + await tasks.trigger('delete-all-manual-answers-orchestrator', { + organizationId: activeOrganizationId, + manualAnswerIds: manualAnswers.map((ma) => ma.id), // Pass IDs directly + }); + logger.info('Triggered delete all manual answers orchestrator task', { + organizationId: activeOrganizationId, + count: manualAnswers.length, + }); + } catch (error) { + // Log error but continue with DB deletion + logger.warn('Failed to trigger delete all manual answers orchestrator', { + organizationId: activeOrganizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Continue with DB deletion even if orchestrator trigger fails + } + } else { + logger.info('No manual answers to delete', { + organizationId: activeOrganizationId, + }); + } + + // Delete all manual answers from main DB + // Vector DB deletion happens in background via orchestrator + await db.securityQuestionnaireManualAnswer.deleteMany({ + where: { + organizationId: activeOrganizationId, + }, + }); + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + revalidatePath(`/${activeOrganizationId}/security-questionnaire/knowledge-base`); + + return { + success: true, + }; + } catch (error) { + console.error('Error deleting all manual answers:', error); + return { + success: false, + error: 'Failed to delete all manual answers', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts new file mode 100644 index 000000000..87e24c27d --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/delete-manual-answer.ts @@ -0,0 +1,98 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; +import { tasks } from '@trigger.dev/sdk'; +import { logger } from '@/utils/logger'; + +const deleteManualAnswerSchema = z.object({ + manualAnswerId: z.string(), +}); + +export const deleteManualAnswer = authActionClient + .inputSchema(deleteManualAnswerSchema) + .metadata({ + name: 'delete-manual-answer', + track: { + event: 'delete-manual-answer', + description: 'Delete Manual Answer', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { manualAnswerId } = parsedInput; + const { activeOrganizationId } = ctx.session; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // Verify manual answer exists and belongs to organization + const manualAnswer = await db.securityQuestionnaireManualAnswer.findUnique({ + where: { + id: manualAnswerId, + organizationId: activeOrganizationId, + }, + }); + + if (!manualAnswer) { + return { + success: false, + error: 'Manual answer not found', + }; + } + + // Trigger Trigger.dev task to delete from vector DB in background + // This runs asynchronously and doesn't block the main DB deletion + try { + await tasks.trigger('delete-manual-answer-from-vector', { + manualAnswerId, + organizationId: activeOrganizationId, + }); + logger.info('Triggered delete manual answer from vector DB task', { + manualAnswerId, + organizationId: activeOrganizationId, + }); + } catch (error) { + // Log error but continue with DB deletion + logger.warn('Failed to trigger delete manual answer from vector DB task', { + manualAnswerId, + organizationId: activeOrganizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Continue with DB deletion even if task trigger fails + } + + // Delete the manual answer from main DB + await db.securityQuestionnaireManualAnswer.delete({ + where: { + id: manualAnswerId, + }, + }); + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + revalidatePath(`/${activeOrganizationId}/security-questionnaire/knowledge-base`); + + return { + success: true, + }; + } catch (error) { + console.error('Error deleting manual answer:', error); + return { + success: false, + error: 'Failed to delete manual answer', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/save-manual-answer.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/save-manual-answer.ts new file mode 100644 index 000000000..6611fd2a7 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/actions/save-manual-answer.ts @@ -0,0 +1,142 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; +import { syncManualAnswerToVector } from '@/lib/vector/sync/sync-manual-answer'; +import { countEmbeddings, listManualAnswerEmbeddings } from '@/lib/vector'; +import { logger } from '@/utils/logger'; + +const saveManualAnswerSchema = z.object({ + question: z.string().min(1), + answer: z.string().min(1), + questionnaireId: z.string().optional(), + tags: z.array(z.string()).optional().default([]), +}); + +export const saveManualAnswer = authActionClient + .inputSchema(saveManualAnswerSchema) + .metadata({ + name: 'save-manual-answer', + track: { + event: 'save-manual-answer', + description: 'Save Manual Answer', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { question, answer, questionnaireId, tags } = parsedInput; + const { activeOrganizationId } = ctx.session; + const userId = ctx.user.id; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + // Upsert manual answer (create or update if question already exists) + const manualAnswer = await db.securityQuestionnaireManualAnswer.upsert({ + where: { + organizationId_question: { + organizationId: activeOrganizationId, + question: question.trim(), + }, + }, + create: { + question: question.trim(), + answer: answer.trim(), + tags: tags || [], + organizationId: activeOrganizationId, + sourceQuestionnaireId: questionnaireId || null, + createdBy: userId || null, + updatedBy: userId || null, + }, + update: { + answer: answer.trim(), + tags: tags || [], + sourceQuestionnaireId: questionnaireId || null, + updatedBy: userId || null, + updatedAt: new Date(), + }, + }); + + // Sync to vector DB SYNCHRONOUSLY (fast ~1-2 sec) + // This ensures manual answers are immediately available for answer generation + + // Count embeddings BEFORE sync + const countBefore = await countEmbeddings(activeOrganizationId, 'manual_answer'); + logger.info('📊 Manual answer embeddings count BEFORE sync', { + organizationId: activeOrganizationId, + count: countBefore.total, + bySourceType: countBefore.bySourceType, + }); + + const syncResult = await syncManualAnswerToVector( + manualAnswer.id, + activeOrganizationId, + ); + + if (!syncResult.success) { + // Log error but don't fail the operation + logger.error('❌ Failed to sync manual answer to vector DB', { + manualAnswerId: manualAnswer.id, + organizationId: activeOrganizationId, + error: syncResult.error, + }); + // Still return success - manual answer is saved in DB + } else { + // Count embeddings AFTER sync to verify it was added + const countAfter = await countEmbeddings(activeOrganizationId, 'manual_answer'); + logger.info('📊 Manual answer embeddings count AFTER sync', { + organizationId: activeOrganizationId, + count: countAfter.total, + bySourceType: countAfter.bySourceType, + increased: countAfter.total > countBefore.total, + difference: countAfter.total - countBefore.total, + }); + + // Also list all manual answer embeddings for debugging + const allManualAnswers = await listManualAnswerEmbeddings(activeOrganizationId); + logger.info('📋 All manual answer embeddings in vector DB', { + organizationId: activeOrganizationId, + count: allManualAnswers.length, + embeddings: allManualAnswers.map((e) => ({ + id: e.id, + sourceId: e.sourceId, + contentPreview: e.content.substring(0, 100), + })), + }); + } + + const headersList = await headers(); + let path = headersList.get('x-pathname') || headersList.get('referer') || ''; + path = path.replace(/\/[a-z]{2}\//, '/'); + + revalidatePath(path); + // Also revalidate knowledge base page + revalidatePath(`/${activeOrganizationId}/security-questionnaire/knowledge-base`); + + // Return embedding ID for verification + // Use embeddingId from syncResult if available, otherwise construct it + const embeddingId = syncResult.embeddingId || `manual_answer_${manualAnswer.id}`; + + return { + success: true, + syncedToVector: syncResult.success, + manualAnswerId: manualAnswer.id, + embeddingId, // Embedding ID for verification in Upstash Vector (e.g., "manual_answer_sqma_xxx") + }; + } catch (error) { + console.error('Error saving manual answer:', error); + return { + success: false, + error: 'Failed to save manual answer', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx new file mode 100644 index 000000000..0b0d64782 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/ManualAnswersSection.tsx @@ -0,0 +1,289 @@ +'use client'; + +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@comp/ui/accordion'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@comp/ui/alert-dialog'; +import { Button } from '@comp/ui/button'; +import { Card } from '@comp/ui'; +import { ChevronLeft, ChevronRight, ExternalLink, PenTool, Trash2 } from 'lucide-react'; +import Link from 'next/link'; +import { useParams, useRouter } from 'next/navigation'; +import { useRef, useState } from 'react'; +import { usePagination } from '../../hooks/usePagination'; +import { format } from 'date-fns'; +import { useAction } from 'next-safe-action/hooks'; +import { toast } from 'sonner'; +import { deleteManualAnswer } from '../actions/delete-manual-answer'; +import { deleteAllManualAnswers } from '../actions/delete-all-manual-answers'; + +interface ManualAnswersSectionProps { + manualAnswers: Awaited>; +} + +export function ManualAnswersSection({ manualAnswers }: ManualAnswersSectionProps) { + const params = useParams(); + const orgId = params.orgId as string; + const router = useRouter(); + const sectionRef = useRef(null); + + const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); + const [deleteAllDialogOpen, setDeleteAllDialogOpen] = useState(false); + const [answerIdToDelete, setAnswerIdToDelete] = useState(null); + const [isDeleting, setIsDeleting] = useState(false); + + const deleteAction = useAction(deleteManualAnswer, { + onSuccess: ({ data }) => { + if (data?.success) { + toast.success('Manual answer deleted successfully'); + setDeleteDialogOpen(false); + setAnswerIdToDelete(null); + setIsDeleting(false); + router.refresh(); + } else { + toast.error(data?.error || 'Failed to delete manual answer'); + setIsDeleting(false); + } + }, + onError: ({ error }) => { + toast.error(error.serverError || 'Failed to delete manual answer'); + setIsDeleting(false); + }, + }); + + const deleteAllAction = useAction(deleteAllManualAnswers, { + onSuccess: ({ data }) => { + if (data?.success) { + toast.success('All manual answers deleted successfully'); + setDeleteAllDialogOpen(false); + router.refresh(); + } else { + toast.error(data?.error || 'Failed to delete all manual answers'); + } + }, + onError: ({ error }) => { + toast.error(error.serverError || 'Failed to delete all manual answers'); + }, + }); + + const { currentPage, totalPages, paginatedItems, handlePageChange } = usePagination({ + items: manualAnswers, + itemsPerPage: 10, + }); + + const handleDelete = (answerId: string) => { + setAnswerIdToDelete(answerId); + setDeleteDialogOpen(true); + }; + + const handleConfirmDelete = () => { + if (answerIdToDelete) { + setIsDeleting(true); + deleteAction.execute({ manualAnswerId: answerIdToDelete }); + } + }; + + const handleDeleteAll = () => { + setDeleteAllDialogOpen(true); + }; + + const handleConfirmDeleteAll = () => { + deleteAllAction.execute({}); + }; + + const handleAccordionChange = (value: string) => { + // If opening (value is set), scroll to section + if (value === 'manual-answers' && sectionRef.current) { + setTimeout(() => { + sectionRef.current?.scrollIntoView({ + behavior: 'smooth', + block: 'start', + }); + }, 100); // Small delay to allow accordion animation to start + } + }; + + return ( + + + + +
+ + Manual Answers + + ({manualAnswers.length}) + +
+ + + {manualAnswers.length === 0 ? ( +
+ No manual answers yet. Answers you write manually in questionnaires will appear here. +
+ ) : ( +
+
+ {paginatedItems.map((answer) => { + const isItemDeleting = isDeleting && answerIdToDelete === answer.id; + return ( +
+
+
+

+ {answer.question} +

+
+ {answer.sourceQuestionnaireId && ( + e.stopPropagation()} + > + + + )} + +
+
+

+ {answer.answer} +

+
+ + Updated {format(new Date(answer.updatedAt), 'MMM dd, yyyy')} + + {answer.tags && answer.tags.length > 0 && ( + + Tags: + {answer.tags.join(', ')} + + )} +
+
+
+ ); + })} +
+ + {/* Pagination and Delete All */} +
+
+
+ Showing {paginatedItems.length} of {manualAnswers.length} answers +
+ {manualAnswers.length > 0 && ( + + )} +
+ {totalPages > 1 && ( +
+ +
+ Page {currentPage} of {totalPages} +
+ +
+ )} +
+
+ )} + + + + + {/* Delete Single Answer Dialog */} + + e.stopPropagation()}> + + Delete Manual Answer + + Are you sure you want to delete this manual answer? This action cannot be undone. + + + + Cancel + + {isDeleting ? 'Deleting...' : 'Delete'} + + + + + + {/* Delete All Answers Dialog */} + + e.stopPropagation()}> + + Delete All Manual Answers + + Are you sure you want to delete all {manualAnswers.length} manual answers? This action cannot be undone. + + + + Cancel + + {deleteAllAction.status === 'executing' ? 'Deleting...' : 'Delete All'} + + + + + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/index.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/index.ts new file mode 100644 index 000000000..99ecb887b --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/manual-answers/components/index.ts @@ -0,0 +1,2 @@ +export { ManualAnswersSection } from './ManualAnswersSection'; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/page.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/page.tsx new file mode 100644 index 000000000..b3516afc3 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/page.tsx @@ -0,0 +1,62 @@ +import PageWithBreadcrumb from '@/components/pages/PageWithBreadcrumb'; +import { auth } from '@/utils/auth'; +import { headers } from 'next/headers'; +import { notFound } from 'next/navigation'; +import { AdditionalDocumentsSection } from './additional-documents/components'; +import { ContextSection } from './context/components'; +import { ManualAnswersSection } from './manual-answers/components'; +import { PublishedPoliciesSection } from './published-policies/components'; +import { KnowledgeBaseHeader } from './components/KnowledgeBaseHeader'; +import { + getContextEntries, + getKnowledgeBaseDocuments, + getManualAnswers, + getPublishedPolicies, +} from './data/queries'; + +export default async function KnowledgeBasePage() { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.user?.id || !session?.session?.activeOrganizationId) { + return notFound(); + } + + const organizationId = session.session.activeOrganizationId; + + // Fetch all data in parallel + const [policies, contextEntries, manualAnswers, documents] = await Promise.all([ + getPublishedPolicies(organizationId), + getContextEntries(organizationId), + getManualAnswers(organizationId), + getKnowledgeBaseDocuments(organizationId), + ]); + + return ( + + + +
+ {/* Published Policies and Context Sections - Side by Side */} +
+ + +
+ + {/* Manual Answers Section */} + + + {/* Additional Documents Section */} + +
+ + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/PublishedPoliciesSection.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/PublishedPoliciesSection.tsx new file mode 100644 index 000000000..a5679268a --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/PublishedPoliciesSection.tsx @@ -0,0 +1,123 @@ +'use client'; + +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@comp/ui/accordion'; +import { Button } from '@comp/ui/button'; +import { Card } from '@comp/ui'; +import { ChevronLeft, ChevronRight, ExternalLink, FileText } from 'lucide-react'; +import Link from 'next/link'; +import { useParams } from 'next/navigation'; +import { useRef } from 'react'; +import { usePagination } from '../../hooks/usePagination'; + +interface PublishedPoliciesSectionProps { + policies: Awaited>; +} + +export function PublishedPoliciesSection({ policies }: PublishedPoliciesSectionProps) { + const params = useParams(); + const orgId = params.orgId as string; + const sectionRef = useRef(null); + + const { currentPage, totalPages, paginatedItems, handlePageChange } = usePagination({ + items: policies, + itemsPerPage: 5, + }); + + const handleAccordionChange = (value: string) => { + // If opening (value is set), scroll to section + if (value === 'published-policies' && sectionRef.current) { + setTimeout(() => { + sectionRef.current?.scrollIntoView({ + behavior: 'smooth', + block: 'start', + }); + }, 100); // Small delay to allow accordion animation to start + } + }; + + return ( + + + + +
+ + Published Policies + ({policies.length}) +
+ + + {policies.length === 0 ? ( +
+ No published policies found +
+ ) : ( + <> +
+ {paginatedItems.map((policy) => ( + +
+
+

+ {policy.name} +

+ {policy.description && ( +

+ {policy.description} +

+ )} +
+ +
+ + ))} +
+ + {/* Pagination */} + {totalPages > 1 && ( +
+
+ + + {currentPage} of {totalPages} + + +
+
+ )} + + )} + + + + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/index.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/index.ts new file mode 100644 index 000000000..b604dc0b7 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/knowledge-base/published-policies/components/index.ts @@ -0,0 +1,2 @@ +export { PublishedPoliciesSection } from './PublishedPoliciesSection'; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/layout.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/layout.tsx new file mode 100644 index 000000000..fe856c89d --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/layout.tsx @@ -0,0 +1,29 @@ +import { SecondaryMenu } from '@comp/ui/secondary-menu'; + +interface LayoutProps { + children: React.ReactNode; + params: Promise<{ orgId: string }>; +} + +export default async function Layout({ children, params }: LayoutProps) { + const { orgId } = await params; + + return ( +
+ +
{children}
+
+ ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/new_questionnaire/page.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/new_questionnaire/page.tsx new file mode 100644 index 000000000..0342b7d7f --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/new_questionnaire/page.tsx @@ -0,0 +1,98 @@ +import { getFeatureFlags } from '@/app/posthog'; +import { AppOnboarding } from '@/components/app-onboarding'; +import PageWithBreadcrumb from '@/components/pages/PageWithBreadcrumb'; +import { auth } from '@/utils/auth'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import { notFound } from 'next/navigation'; +import { QuestionnaireParser } from '../components/QuestionnaireParser'; + +export default async function NewQuestionnairePage() { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.user?.id || !session?.session?.activeOrganizationId) { + return notFound(); + } + + // Check feature flag on server + const flags = await getFeatureFlags(session.user.id); + const isFeatureEnabled = flags['ai-vendor-questionnaire'] === true; + + if (!isFeatureEnabled) { + return notFound(); + } + + const organizationId = session.session.activeOrganizationId; + + // Check if organization has published policies + const hasPublishedPolicies = await checkPublishedPolicies(organizationId); + + // Show onboarding if no published policies exist + if (!hasPublishedPolicies) { + return ( + + + + ); + } + + return ( + + + + ); +} + +const checkPublishedPolicies = async (organizationId: string): Promise => { + const count = await db.policy.count({ + where: { + organizationId, + status: 'published', + isArchived: false, + }, + }); + + return count > 0; +}; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/page.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/page.tsx index f960ef3ca..351ba687b 100644 --- a/apps/app/src/app/(app)/[orgId]/security-questionnaire/page.tsx +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/page.tsx @@ -1,11 +1,12 @@ import { getFeatureFlags } from '@/app/posthog'; import { AppOnboarding } from '@/components/app-onboarding'; +import PageWithBreadcrumb from '@/components/pages/PageWithBreadcrumb'; import { auth } from '@/utils/auth'; import { db } from '@db'; import { headers } from 'next/headers'; import { notFound } from 'next/navigation'; -import { cache } from 'react'; -import { QuestionnaireParser } from './components/QuestionnaireParser'; +import { QuestionnaireOverview } from './start_page/components'; +import { getQuestionnaires } from './start_page/data/queries'; export default async function SecurityQuestionnairePage() { const session = await auth.api.getSession({ @@ -32,7 +33,7 @@ export default async function SecurityQuestionnairePage() { // Show onboarding if no published policies exist if (!hasPublishedPolicies) { return ( -
+
- -
+ + + ); } -const checkPublishedPolicies = cache(async (organizationId: string): Promise => { +const checkPublishedPolicies = async (organizationId: string): Promise => { const count = await db.policy.count({ where: { organizationId, @@ -84,4 +92,4 @@ const checkPublishedPolicies = cache(async (organizationId: string): Promise 0; -}); +}; diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/README.md b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/README.md new file mode 100644 index 000000000..a5238a241 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/README.md @@ -0,0 +1,28 @@ +# Start Page (First Step) + +This folder contains components and logic for the **first step** of the security questionnaire flow - the start page. + +## Structure + +``` +start_page/ +├── components/ +│ ├── QuestionnaireOverview.tsx # Main overview component +│ ├── QuestionnaireHistory.tsx # History/list component +│ └── index.ts # Component exports +└── README.md # This file +``` + +## Purpose + +The start page (`/security-questionnaire`) displays: +- Header with navigation buttons (Questionnaires, Knowledge Base) +- "New Questionnaire" card with button to create a new questionnaire +- History of previously parsed questionnaires + +## Flow + +1. **First Step (Start Page)**: `/security-questionnaire` - Shows overview and history +2. **Second Step (Create)**: `/security-questionnaire/new_questionnaire` - File upload and parsing +3. **View Details**: `/security-questionnaire/[questionnaireId]` - View individual questionnaire + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/actions/delete-questionnaire.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/actions/delete-questionnaire.ts new file mode 100644 index 000000000..88cf20f56 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/actions/delete-questionnaire.ts @@ -0,0 +1,65 @@ +'use server'; + +import { authActionClient } from '@/actions/safe-action'; +import { db } from '@db'; +import { revalidatePath } from 'next/cache'; +import { z } from 'zod'; + +const deleteQuestionnaireSchema = z.object({ + questionnaireId: z.string(), +}); + +export const deleteQuestionnaireAction = authActionClient + .inputSchema(deleteQuestionnaireSchema) + .metadata({ + name: 'delete-questionnaire', + track: { + event: 'delete-questionnaire', + description: 'Delete Questionnaire', + channel: 'server', + }, + }) + .action(async ({ parsedInput, ctx }) => { + const { questionnaireId } = parsedInput; + const { activeOrganizationId } = ctx.session; + + if (!activeOrganizationId) { + return { + success: false, + error: 'Not authorized', + }; + } + + try { + const questionnaire = await db.questionnaire.findUnique({ + where: { + id: questionnaireId, + organizationId: activeOrganizationId, + }, + }); + + if (!questionnaire) { + return { + success: false, + error: 'Questionnaire not found', + }; + } + + await db.questionnaire.delete({ + where: { id: questionnaireId }, + }); + + revalidatePath(`/${activeOrganizationId}/security-questionnaire`); + + return { + success: true, + }; + } catch (error) { + console.error(error); + return { + success: false, + error: 'Failed to delete questionnaire', + }; + } + }); + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireHistory.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireHistory.tsx new file mode 100644 index 000000000..108e38e02 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireHistory.tsx @@ -0,0 +1,330 @@ +'use client'; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@comp/ui/alert-dialog'; +import { Button } from '@comp/ui/button'; +import { Card } from '@comp/ui'; +import { Input } from '@comp/ui/input'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@comp/ui/select'; +import { formatDistanceToNow } from 'date-fns'; +import { CheckCircle2, ChevronLeft, ChevronRight, FileSpreadsheet, FileText, Loader2, Trash2, X } from 'lucide-react'; +import { useRouter } from 'next/navigation'; +import { useState } from 'react'; +import { toast } from 'sonner'; +import { deleteQuestionnaireAction } from '../actions/delete-questionnaire'; +import { useQuestionnaireHistory } from '../hooks/useQuestionnaireHistory'; + +function getFileIcon(filename: string) { + const extension = filename.split('.').pop()?.toLowerCase() || ''; + + if (extension === 'pdf') { + return FileText; + } + + if (['xls', 'xlsx', 'csv'].includes(extension)) { + return FileSpreadsheet; + } + + return FileText; +} + +interface QuestionnaireHistoryProps { + questionnaires: Awaited>; + orgId: string; +} + +export function QuestionnaireHistory({ questionnaires, orgId }: QuestionnaireHistoryProps) { + const router = useRouter(); + const { + searchQuery, + setSearchQuery, + currentPage, + itemsPerPage, + totalPages, + paginatedQuestionnaires, + totalFiltered, + handlePageChange, + handleItemsPerPageChange, + } = useQuestionnaireHistory({ questionnaires }); + + if (questionnaires.length === 0) { + return ( + +
+
+ +
+
+

No questionnaires yet

+

+ Create your first questionnaire to see it here +

+
+
+ + ); + } + + return ( +
+ {/* Search Input and Items Per Page */} +
+
+ setSearchQuery(e.target.value)} + /> + {searchQuery && ( + + )} +
+
+ Items per page: + +
+
+ + {/* Results Count */} + {searchQuery && ( +

+ {totalFiltered} {totalFiltered === 1 ? 'result' : 'results'} found +

+ )} + + {/* Questionnaire List */} + {paginatedQuestionnaires.length === 0 ? ( + +
+
+ +
+
+

No questionnaires found

+

+ {searchQuery ? 'Try a different search term' : 'Create your first questionnaire to see it here'} +

+
+
+ + ) : ( +
+ {paginatedQuestionnaires.map((questionnaire: Awaited>[number]) => ( + + ))} +
+ )} + + {/* Pagination */} + {totalPages > 1 && ( +
+
+ + + {currentPage} of {totalPages} + + +
+
+ )} +
+ ); +} + +interface QuestionnaireHistoryItemProps { + questionnaire: Awaited>[number]; + orgId: string; + router: ReturnType; +} + +function QuestionnaireHistoryItem({ + questionnaire, + orgId, + router, +}: QuestionnaireHistoryItemProps) { + const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); + + const answeredCount = questionnaire.questions.filter((q: { answer: string | null }) => q.answer).length; + const totalQuestions = questionnaire.questions.length; + const isParsing = questionnaire.status === 'parsing'; + const FileIcon = getFileIcon(questionnaire.filename); + + const handleItemClick = () => { + if (!isParsing) { + router.push(`/${orgId}/security-questionnaire/${questionnaire.id}`); + } + }; + + const handleDelete = async (e: React.MouseEvent) => { + e.stopPropagation(); + setIsDeleting(true); + + try { + const result = await deleteQuestionnaireAction({ questionnaireId: questionnaire.id }); + + if (result?.data?.success) { + toast.success('Questionnaire deleted successfully'); + setIsDeleteDialogOpen(false); + router.refresh(); + } else { + toast.error(result?.data?.error || 'Failed to delete questionnaire'); + } + } catch (error) { + toast.error('An error occurred while deleting the questionnaire'); + } finally { + setIsDeleting(false); + } + }; + + return ( + <> + +
+ {/* Icon */} +
+ {isParsing ? ( + + ) : ( + + )} +
+ + {/* Content */} +
+
+
+

+ {questionnaire.filename} +

+
+ {isParsing ? ( + Parsing... + ) : ( + <> +
+ + + {formatDistanceToNow(new Date(questionnaire.createdAt), { + addSuffix: true, + })} + +
+ {totalQuestions > 0 && ( +
+ + Answered + + + {answeredCount}/{totalQuestions} + +
+ )} + + )} +
+
+
+
+ + {/* Delete Button */} + +
+ + + {/* Delete Confirmation Dialog */} + + e.stopPropagation()}> + + Delete Questionnaire + + Are you sure you want to delete {questionnaire.filename}? This action + cannot be undone and will permanently delete all questions and answers. + + + + Cancel + + {isDeleting ? ( + <> + + Deleting... + + ) : ( + 'Delete' + )} + + + + + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireOverview.tsx b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireOverview.tsx new file mode 100644 index 000000000..e41c4d684 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/QuestionnaireOverview.tsx @@ -0,0 +1,70 @@ +'use client'; + +import { Button } from '@comp/ui/button'; +import { Card, CardContent } from '@comp/ui'; +import { FileText, Plus } from 'lucide-react'; +import Link from 'next/link'; +import { useParams } from 'next/navigation'; +import { QuestionnaireHistory } from './QuestionnaireHistory'; + +interface QuestionnaireOverviewProps { + questionnaires: Awaited>; +} + +export function QuestionnaireOverview({ questionnaires }: QuestionnaireOverviewProps) { + const params = useParams(); + const orgId = params.orgId as string; + + return ( +
+ {/* Header */} +
+

+ Security Questionnaire +

+

+ Automatically analyze and answer questionnaires using AI. Upload questionnaires from + vendors, and our system will extract questions and generate answers based on your + organization's policies and documentation. +

+
+ + {/* New Questionnaire Card */} + + +
+
+
+ +
+
+

Create New Questionnaire

+

+ Upload a questionnaire file to extract questions and generate answers +

+
+
+ +
+ + + + {/* History Section */} +
+
+

History

+

+ View and manage your previously parsed questionnaires +

+
+ +
+
+ ); +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/index.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/index.ts new file mode 100644 index 000000000..b0f7bdfd0 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/components/index.ts @@ -0,0 +1,7 @@ +// Start Page (First Step) Components +// This folder contains components for the questionnaire start page +// which displays the list of questionnaires and allows creating new ones. + +export { QuestionnaireOverview } from './QuestionnaireOverview'; +export { QuestionnaireHistory } from './QuestionnaireHistory'; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/data/queries.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/data/queries.ts new file mode 100644 index 000000000..e0ed85fbd --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/data/queries.ts @@ -0,0 +1,45 @@ +'use server'; + +import { auth } from '@/utils/auth'; +import { db } from '@db'; +import { headers } from 'next/headers'; +import 'server-only'; + +export const getQuestionnaires = async (organizationId: string) => { + const session = await auth.api.getSession({ + headers: await headers(), + }); + + if (!session?.session?.activeOrganizationId || session.session.activeOrganizationId !== organizationId) { + return []; + } + + const questionnaires = await db.questionnaire.findMany({ + where: { + organizationId, + status: { + in: ['completed', 'parsing'], + }, + }, + include: { + questions: { + orderBy: { + questionIndex: 'asc', + }, + select: { + id: true, + question: true, + answer: true, + status: true, + questionIndex: true, + }, + }, + }, + orderBy: { + createdAt: 'desc', + }, + }); + + return questionnaires; +}; + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/hooks/useQuestionnaireHistory.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/hooks/useQuestionnaireHistory.ts new file mode 100644 index 000000000..55871e1e5 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/start_page/hooks/useQuestionnaireHistory.ts @@ -0,0 +1,59 @@ +'use client'; + +import { useMemo, useState } from 'react'; + +interface UseQuestionnaireHistoryProps { + questionnaires: Awaited>; +} + +export function useQuestionnaireHistory({ questionnaires }: UseQuestionnaireHistoryProps) { + const [searchQuery, setSearchQuery] = useState(''); + const [currentPage, setCurrentPage] = useState(1); + const [itemsPerPage, setItemsPerPage] = useState(5); + + // Filter questionnaires by filename + const filteredQuestionnaires = useMemo(() => { + if (!searchQuery.trim()) { + return questionnaires; + } + + const query = searchQuery.toLowerCase(); + return questionnaires.filter((questionnaire: Awaited>[number]) => + questionnaire.filename.toLowerCase().includes(query), + ); + }, [questionnaires, searchQuery]); + + // Calculate pagination + const totalPages = Math.max(1, Math.ceil(filteredQuestionnaires.length / itemsPerPage)); + const startIndex = (currentPage - 1) * itemsPerPage; + const endIndex = startIndex + itemsPerPage; + const paginatedQuestionnaires = filteredQuestionnaires.slice(startIndex, endIndex); + + // Reset to page 1 when search changes or items per page changes + const handleSearchChange = (value: string) => { + setSearchQuery(value); + setCurrentPage(1); + }; + + const handleItemsPerPageChange = (value: number) => { + setItemsPerPage(value); + setCurrentPage(1); + }; + + const handlePageChange = (page: number) => { + setCurrentPage(page); + }; + + return { + searchQuery, + setSearchQuery: handleSearchChange, + currentPage, + itemsPerPage, + totalPages, + paginatedQuestionnaires, + totalFiltered: filteredQuestionnaires.length, + handlePageChange, + handleItemsPerPageChange, + }; +} + diff --git a/apps/app/src/app/(app)/[orgId]/security-questionnaire/utils/deduplicate-sources.ts b/apps/app/src/app/(app)/[orgId]/security-questionnaire/utils/deduplicate-sources.ts new file mode 100644 index 000000000..014975db7 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/security-questionnaire/utils/deduplicate-sources.ts @@ -0,0 +1,111 @@ +/** + * Universal utility function to deduplicate sources for questionnaire answers. + * + * Deduplication rules: + * - Policies: Same policy name = same source (deduplicated by policyName) + * - Context: All context entries = single "Context Q&A" source + * - Manual Answers: All manual answers = single "Manual Answer" source + * - Knowledge Base Documents: Deduplicated by sourceId (each document is a separate source) + * - Other sources: Deduplicated by sourceId + */ + +export interface Source { + sourceType: string; + sourceName?: string; + sourceId?: string; + policyName?: string; + documentName?: string; + score: number; +} + +/** + * Deduplicates an array of sources based on source type and content. + * For each source type, uses appropriate deduplication key: + * - Policies: policyName + * - Context: "Context Q&A" (all grouped together) + * - Manual Answers: "Manual Answer" (all grouped together) + * - Knowledge Base Documents: sourceId (each document is separate) + * - Others: sourceId + * + * When duplicates are found, keeps the one with the highest score. + */ +export function deduplicateSources(sources: Source[]): Source[] { + const sourceMap = new Map(); + + for (const source of sources) { + let deduplicationKey: string; + + // Determine deduplication key based on source type + if (source.sourceType === 'policy' && source.policyName) { + // Policies: deduplicate by policy name + deduplicationKey = `policy:${source.policyName}`; + } else if (source.sourceType === 'context') { + // Context: all context entries are grouped as one source + deduplicationKey = 'context:all'; + } else if (source.sourceType === 'manual_answer') { + // Manual Answers: all manual answers are grouped as one source + deduplicationKey = 'manual_answer:all'; + } else if (source.sourceType === 'knowledge_base_document') { + // Knowledge Base Documents: deduplicate by sourceId (each document is separate) + deduplicationKey = `knowledge_base_document:${source.sourceId || 'unknown'}`; + } else { + // Other sources: deduplicate by sourceId + deduplicationKey = source.sourceId || `unknown:${source.sourceType}`; + } + + // If we haven't seen this source, or this chunk has a higher score, use it + const existing = sourceMap.get(deduplicationKey); + if (!existing || source.score > existing.score) { + // Create a normalized source with appropriate sourceName + // Preserve documentName if available (it might be missing in some chunks) + const normalizedSource: Source = { + ...source, + documentName: source.documentName || existing?.documentName, + sourceName: getSourceDisplayName({ + ...source, + documentName: source.documentName || existing?.documentName, + }), + }; + sourceMap.set(deduplicationKey, normalizedSource); + } else if (existing && source.documentName && !existing.documentName) { + // If existing source doesn't have documentName but new one does, update it + existing.documentName = source.documentName; + existing.sourceName = getSourceDisplayName(existing); + } + } + + // Convert map to array and sort by score (highest first) + return Array.from(sourceMap.values()).sort((a, b) => b.score - a.score); +} + +/** + * Generates a display name for a source based on its type and properties. + */ +function getSourceDisplayName(source: Source): string { + if (source.sourceType === 'policy' && source.policyName) { + return `Policy: ${source.policyName}`; + } + + if (source.sourceType === 'context') { + return 'Context Q&A'; + } + + if (source.sourceType === 'manual_answer') { + return 'Manual Answer'; + } + + if (source.sourceType === 'knowledge_base_document') { + // Show filename if available, otherwise just "Knowledge Base Document" + if (source.documentName) { + return `Knowledge Base Document (${source.documentName})`; + } + return 'Knowledge Base Document'; + } + + if (source.sourceName) { + return source.sourceName; + } + + return source.sourceType || 'Unknown Source'; +} + diff --git a/apps/app/src/app/s3.ts b/apps/app/src/app/s3.ts index 6dc732e3c..f9907505f 100644 --- a/apps/app/src/app/s3.ts +++ b/apps/app/src/app/s3.ts @@ -6,6 +6,7 @@ const APP_AWS_SECRET_ACCESS_KEY = process.env.APP_AWS_SECRET_ACCESS_KEY; export const BUCKET_NAME = process.env.APP_AWS_BUCKET_NAME; export const APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET = process.env.APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET; +export const APP_AWS_KNOWLEDGE_BASE_BUCKET = process.env.APP_AWS_KNOWLEDGE_BASE_BUCKET; let s3ClientInstance: S3Client; diff --git a/apps/app/src/components/app-onboarding.tsx b/apps/app/src/components/app-onboarding.tsx index 6d7920829..f56465406 100644 --- a/apps/app/src/components/app-onboarding.tsx +++ b/apps/app/src/components/app-onboarding.tsx @@ -108,20 +108,20 @@ export function AppOnboarding({ {cta && (
- {href ? ( + {href ? ( - + - + + {cta} + + {ctaDisabled && ctaTooltip && ( @@ -134,15 +134,15 @@ export function AppOnboarding({ - + > + + {cta} + {ctaDisabled && ctaTooltip && ( @@ -151,7 +151,7 @@ export function AppOnboarding({ )} - )} + )}
)} diff --git a/apps/app/src/env.mjs b/apps/app/src/env.mjs index 25e1e707d..423f6315f 100644 --- a/apps/app/src/env.mjs +++ b/apps/app/src/env.mjs @@ -30,6 +30,7 @@ export const env = createEnv({ APP_AWS_REGION: z.string().optional(), APP_AWS_BUCKET_NAME: z.string().optional(), APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET: z.string().optional(), + APP_AWS_KNOWLEDGE_BASE_BUCKET: z.string().optional(), NEXT_PUBLIC_PORTAL_URL: z.string(), FIRECRAWL_API_KEY: z.string().optional(), FLEET_URL: z.string().optional(), @@ -85,6 +86,7 @@ export const env = createEnv({ APP_AWS_REGION: process.env.APP_AWS_REGION, APP_AWS_BUCKET_NAME: process.env.APP_AWS_BUCKET_NAME, APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET: process.env.APP_AWS_QUESTIONNAIRE_UPLOAD_BUCKET, + APP_AWS_KNOWLEDGE_BASE_BUCKET: process.env.APP_AWS_KNOWLEDGE_BASE_BUCKET, NEXT_PUBLIC_PORTAL_URL: process.env.NEXT_PUBLIC_PORTAL_URL, FIRECRAWL_API_KEY: process.env.FIRECRAWL_API_KEY, FLEET_URL: process.env.FLEET_URL, diff --git a/apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts b/apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts new file mode 100644 index 000000000..40d9d1c94 --- /dev/null +++ b/apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts @@ -0,0 +1,167 @@ +import { logger, metadata, task } from '@trigger.dev/sdk'; +import { db } from '@db'; +import { deleteManualAnswerTask } from './delete-manual-answer'; + +const BATCH_SIZE = 50; // Process 50 deletions at a time in parallel + +/** + * Orchestrator task to delete all manual answers from vector database + * Processes deletions in parallel batches for better performance + */ +export const deleteAllManualAnswersOrchestratorTask = task({ + id: 'delete-all-manual-answers-orchestrator', + retry: { + maxAttempts: 3, + }, + run: async (payload: { + organizationId: string; + manualAnswerIds?: string[]; // Optional: IDs passed directly to avoid race condition + }) => { + logger.info('Starting delete all manual answers from vector DB', { + organizationId: payload.organizationId, + manualAnswerIdsProvided: !!payload.manualAnswerIds, + manualAnswerIdsCount: payload.manualAnswerIds?.length || 0, + }); + + try { + // Use provided IDs if available, otherwise fetch from DB + let manualAnswers: Array<{ id: string }>; + + if (payload.manualAnswerIds && payload.manualAnswerIds.length > 0) { + // Use IDs passed directly (avoids race condition with DB deletion) + manualAnswers = payload.manualAnswerIds.map((id) => ({ id })); + logger.info('Using provided manual answer IDs', { + organizationId: payload.organizationId, + count: manualAnswers.length, + }); + } else { + // Fallback: get all manual answers for the organization + // This might return empty if DB records were already deleted + manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ + where: { + organizationId: payload.organizationId, + }, + select: { + id: true, + }, + }); + + logger.info('Fetched manual answers from DB', { + organizationId: payload.organizationId, + count: manualAnswers.length, + }); + } + + if (manualAnswers.length === 0) { + logger.info('No manual answers to delete', { + organizationId: payload.organizationId, + }); + return { + success: true, + deletedCount: 0, + }; + } + + // Initialize metadata for tracking progress + metadata.set('totalManualAnswers', manualAnswers.length); + metadata.set('deletedCount', 0); + metadata.set('failedCount', 0); + metadata.set('currentBatch', 0); + metadata.set('totalBatches', Math.ceil(manualAnswers.length / BATCH_SIZE)); + + let deletedCount = 0; + let failedCount = 0; + + // Process deletions in batches + for (let i = 0; i < manualAnswers.length; i += BATCH_SIZE) { + const batch = manualAnswers.slice(i, i + BATCH_SIZE); + const batchNumber = Math.floor(i / BATCH_SIZE) + 1; + const totalBatches = Math.ceil(manualAnswers.length / BATCH_SIZE); + + logger.info(`Processing deletion batch ${batchNumber}/${totalBatches}`, { + batchSize: batch.length, + manualAnswerIds: batch.map((ma) => ma.id), + }); + + // Update metadata + metadata.set('currentBatch', batchNumber); + + // Trigger batch deletions in parallel + const batchItems = batch.map((ma) => ({ + payload: { + manualAnswerId: ma.id, + organizationId: payload.organizationId, + }, + })); + + const batchHandle = await deleteManualAnswerTask.batchTriggerAndWait(batchItems); + + // Process batch results + batchHandle.runs.forEach((run, batchIdx) => { + const ma = batch[batchIdx]; + + if (run.ok && run.output) { + const taskResult = run.output; + if (taskResult.success) { + deletedCount++; + } else { + failedCount++; + logger.warn('Failed to delete manual answer from vector DB', { + manualAnswerId: ma.id, + error: taskResult.error, + }); + } + } else { + failedCount++; + const errorMessage = + run.ok === false && run.error + ? run.error instanceof Error + ? run.error.message + : String(run.error) + : 'Unknown error'; + logger.error('Task failed to delete manual answer', { + manualAnswerId: ma.id, + error: errorMessage, + }); + } + }); + + // Update metadata + metadata.set('deletedCount', deletedCount); + metadata.set('failedCount', failedCount); + + logger.info(`Batch ${batchNumber}/${totalBatches} completed`, { + batchSize: batch.length, + deletedSoFar: deletedCount, + failedSoFar: failedCount, + remaining: manualAnswers.length - deletedCount - failedCount, + }); + } + + logger.info('Delete all manual answers from vector DB completed', { + organizationId: payload.organizationId, + total: manualAnswers.length, + deleted: deletedCount, + failed: failedCount, + }); + + // Mark as completed + metadata.set('completed', true); + + return { + success: true, + deletedCount, + failedCount, + total: manualAnswers.length, + }; + } catch (error) { + logger.error('Failed to delete all manual answers from vector DB', { + organizationId: payload.organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + errorStack: error instanceof Error ? error.stack : undefined, + }); + throw error; + } + }, +}); + diff --git a/apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts b/apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts new file mode 100644 index 000000000..d752af191 --- /dev/null +++ b/apps/app/src/jobs/tasks/vector/delete-knowledge-base-document.ts @@ -0,0 +1,264 @@ +import { logger, task } from '@trigger.dev/sdk'; +import { findEmbeddingsForSource } from '@/lib/vector/core/find-existing-embeddings'; +import { vectorIndex } from '@/lib/vector/core/client'; +import { db } from '@db'; + +/** + * Task to delete all embeddings for a Knowledge Base document from vector database + */ +export const deleteKnowledgeBaseDocumentTask = task({ + id: 'delete-knowledge-base-document-from-vector', + retry: { + maxAttempts: 3, + }, + run: async (payload: { + documentId: string; + organizationId: string; + }) => { + logger.info('Deleting Knowledge Base document from vector DB', { + documentId: payload.documentId, + organizationId: payload.organizationId, + }); + + try { + // Fetch document info to use document name in query (helps find all chunks) + let documentName: string | undefined; + try { + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: payload.documentId, + organizationId: payload.organizationId, + }, + select: { + name: true, + }, + }); + documentName = document?.name; + } catch (dbError) { + logger.warn('Could not fetch document name, proceeding without it', { + documentId: payload.documentId, + error: dbError instanceof Error ? dbError.message : 'Unknown error', + }); + } + + // Find all embeddings for this document + // Pass documentName to help find all chunks (used in query strategies) + const existingEmbeddings = await findEmbeddingsForSource( + payload.documentId, + 'knowledge_base_document', + payload.organizationId, + documentName, // Optional: helps find chunks semantically similar to document name + ); + + if (existingEmbeddings.length === 0) { + logger.info('No embeddings found for document', { + documentId: payload.documentId, + }); + return { + success: true, + documentId: payload.documentId, + deletedCount: 0, + }; + } + + // Delete all embeddings + if (!vectorIndex) { + logger.error('Vector index not configured'); + return { + success: false, + documentId: payload.documentId, + error: 'Vector index not configured', + }; + } + + const idsToDelete = existingEmbeddings.map((e) => e.id); + + if (idsToDelete.length === 0) { + logger.info('No embeddings to delete for document', { + documentId: payload.documentId, + }); + return { + success: true, + documentId: payload.documentId, + deletedCount: 0, + }; + } + + // Delete all embeddings in batches (Upstash Vector supports batch delete) + const batchSize = 100; + let deletedCount = 0; + + for (let i = 0; i < idsToDelete.length; i += batchSize) { + const batch = idsToDelete.slice(i, i + batchSize); + try { + await vectorIndex.delete(batch); + deletedCount += batch.length; + logger.info('Deleted batch of embeddings', { + documentId: payload.documentId, + batchSize: batch.length, + totalDeleted: deletedCount, + totalToDelete: idsToDelete.length, + }); + } catch (batchError) { + logger.error('Error deleting batch of embeddings', { + documentId: payload.documentId, + batchSize: batch.length, + error: batchError instanceof Error ? batchError.message : 'Unknown error', + }); + // Continue with next batch even if one fails + } + } + + // Verify deletion with retry logic (with delays to allow propagation) + // This helps catch cases where some chunks might have been missed or not found initially + // Use the enhanced findEmbeddingsForSource which now includes chunk content queries + let remainingEmbeddings = await findEmbeddingsForSource( + payload.documentId, + 'knowledge_base_document', + payload.organizationId, + documentName, // Use document name in verification queries too + ); + + logger.info('Initial verification after deletion', { + documentId: payload.documentId, + remainingCount: remainingEmbeddings.length, + remainingIds: remainingEmbeddings.map((e) => e.id), + }); + + // Retry deletion up to 3 times if chunks remain + let retryAttempt = 0; + const maxRetries = 3; + + while (remainingEmbeddings.length > 0 && retryAttempt < maxRetries) { + retryAttempt++; + logger.warn('Some embeddings were not deleted, attempting retry deletion', { + documentId: payload.documentId, + remainingCount: remainingEmbeddings.length, + remainingIds: remainingEmbeddings.map((e) => e.id), + retryAttempt, + maxRetries, + }); + + // Wait before retry to allow propagation + await new Promise((resolve) => setTimeout(resolve, 2000 * retryAttempt)); // Increasing delay + + // Try deleting remaining chunks + const remainingIds = remainingEmbeddings.map((e) => e.id); + try { + // Delete in batches + const batchSize = 100; + for (let i = 0; i < remainingIds.length; i += batchSize) { + const batch = remainingIds.slice(i, i + batchSize); + await vectorIndex.delete(batch); + deletedCount += batch.length; + } + + logger.info('Deleted remaining embeddings in retry attempt', { + documentId: payload.documentId, + deletedCount: remainingIds.length, + retryAttempt, + }); + } catch (retryError) { + logger.error('Error deleting remaining embeddings in retry attempt', { + documentId: payload.documentId, + retryAttempt, + error: retryError instanceof Error ? retryError.message : 'Unknown error', + }); + } + + // Query again to check if deletion was successful + await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for propagation + remainingEmbeddings = await findEmbeddingsForSource( + payload.documentId, + 'knowledge_base_document', + payload.organizationId, + documentName, // Use document name in retry queries too + ); + } + + // Final verification - if chunks still remain, try one more aggressive search + if (remainingEmbeddings.length > 0) { + logger.warn('Chunks still remain after retries, attempting final aggressive search', { + documentId: payload.documentId, + remainingCount: remainingEmbeddings.length, + remainingIds: remainingEmbeddings.map((e) => e.id), + }); + + // Wait a bit longer for final attempt + await new Promise((resolve) => setTimeout(resolve, 3000)); + + // Try one more time with enhanced search (now includes chunk content queries) + const finalRemainingEmbeddings = await findEmbeddingsForSource( + payload.documentId, + 'knowledge_base_document', + payload.organizationId, + documentName, + ); + + if (finalRemainingEmbeddings.length > 0) { + // Try deleting these final chunks + const finalIds = finalRemainingEmbeddings.map((e) => e.id); + try { + await vectorIndex.delete(finalIds); + deletedCount += finalIds.length; + logger.info('Deleted chunks in final aggressive attempt', { + documentId: payload.documentId, + deletedCount: finalIds.length, + }); + } catch (finalError) { + logger.error('Error in final deletion attempt', { + documentId: payload.documentId, + error: finalError instanceof Error ? finalError.message : 'Unknown error', + }); + } + + // Final check + await new Promise((resolve) => setTimeout(resolve, 2000)); + const trulyRemaining = await findEmbeddingsForSource( + payload.documentId, + 'knowledge_base_document', + payload.organizationId, + documentName, + ); + + if (trulyRemaining.length > 0) { + logger.error('CRITICAL: Some embeddings still remain after all deletion attempts', { + documentId: payload.documentId, + remainingCount: trulyRemaining.length, + remainingIds: trulyRemaining.map((e) => e.id), + remainingChunks: trulyRemaining.map((e) => ({ + id: e.id, + sourceId: e.sourceId, + updatedAt: e.updatedAt, + })), + note: 'These chunks may need manual deletion or there may be a synchronization issue with Upstash Vector', + }); + } + } + } + + logger.info('Successfully deleted Knowledge Base document embeddings from vector DB', { + documentId: payload.documentId, + deletedCount, + totalFound: idsToDelete.length, + }); + + return { + success: true, + documentId: payload.documentId, + deletedCount, + }; + } catch (error) { + logger.error('Error deleting Knowledge Base document from vector DB', { + documentId: payload.documentId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + success: false, + documentId: payload.documentId, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } + }, +}); + diff --git a/apps/app/src/jobs/tasks/vector/delete-manual-answer.ts b/apps/app/src/jobs/tasks/vector/delete-manual-answer.ts new file mode 100644 index 000000000..85cc52380 --- /dev/null +++ b/apps/app/src/jobs/tasks/vector/delete-manual-answer.ts @@ -0,0 +1,61 @@ +import { logger, task } from '@trigger.dev/sdk'; +import { deleteManualAnswerFromVector } from '@/lib/vector/sync/sync-manual-answer'; + +/** + * Task to delete a single manual answer from vector database + * Used by orchestrator for parallel deletion + */ +export const deleteManualAnswerTask = task({ + id: 'delete-manual-answer-from-vector', + retry: { + maxAttempts: 3, + }, + run: async (payload: { + manualAnswerId: string; + organizationId: string; + }) => { + logger.info('Deleting manual answer from vector DB', { + manualAnswerId: payload.manualAnswerId, + organizationId: payload.organizationId, + }); + + try { + const result = await deleteManualAnswerFromVector( + payload.manualAnswerId, + payload.organizationId, + ); + + if (!result.success) { + logger.warn('Failed to delete manual answer from vector DB', { + manualAnswerId: payload.manualAnswerId, + error: result.error, + }); + return { + success: false, + manualAnswerId: payload.manualAnswerId, + error: result.error, + }; + } + + logger.info('Successfully deleted manual answer from vector DB', { + manualAnswerId: payload.manualAnswerId, + }); + + return { + success: true, + manualAnswerId: payload.manualAnswerId, + }; + } catch (error) { + logger.error('Error deleting manual answer from vector DB', { + manualAnswerId: payload.manualAnswerId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + success: false, + manualAnswerId: payload.manualAnswerId, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } + }, +}); + diff --git a/apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts b/apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts new file mode 100644 index 000000000..371e0f716 --- /dev/null +++ b/apps/app/src/jobs/tasks/vector/helpers/extract-content-from-file.ts @@ -0,0 +1,231 @@ +import { logger } from '@/utils/logger'; +import { openai } from '@ai-sdk/openai'; +import { generateText } from 'ai'; +import * as XLSX from 'xlsx'; +import mammoth from 'mammoth'; + +const htmlEntityMap = { + ' ': ' ', + '&': '&', + '<': '<', + '>': '>', + '"': '"', +} as const; + +const decodeBasicHtmlEntities = (input: string) => { + const entityPattern = /&(nbsp|amp|lt|gt|quot);/g; + let decoded = input; + let previousValue: string; + + do { + previousValue = decoded; + decoded = decoded.replace(entityPattern, (entity) => htmlEntityMap[entity as keyof typeof htmlEntityMap] ?? entity); + } while (decoded !== previousValue); + + return decoded; +}; + +/** + * Extracts content from a file using various methods based on file type + * Supports: PDF, Excel (.xlsx, .xls), CSV, text files (.txt, .md), Word documents (.doc, .docx), images + */ +export async function extractContentFromFile( + fileData: string, + fileType: string, +): Promise { + const fileBuffer = Buffer.from(fileData, 'base64'); + + // Handle Excel files (.xlsx, .xls) + if ( + fileType === 'application/vnd.ms-excel' || + fileType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || + fileType === 'application/vnd.ms-excel.sheet.macroEnabled.12' + ) { + try { + const excelStartTime = Date.now(); + const fileSizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2); + + logger.info('Processing Excel file', { + fileType, + fileSizeMB, + }); + + const workbook = XLSX.read(fileBuffer, { type: 'buffer' }); + + // Process sheets sequentially + const sheets: string[] = []; + + for (const sheetName of workbook.SheetNames) { + const worksheet = workbook.Sheets[sheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: '' }); + + // Convert to readable text format + const sheetText = jsonData + .map((row: any) => { + if (Array.isArray(row)) { + return row.filter((cell) => cell !== null && cell !== undefined && cell !== '').join(' | '); + } + return String(row); + }) + .filter((line: string) => line.trim() !== '') + .join('\n'); + + if (sheetText.trim()) { + sheets.push(`Sheet: ${sheetName}\n${sheetText}`); + } + } + + const extractionTime = ((Date.now() - excelStartTime) / 1000).toFixed(2); + logger.info('Excel file processed', { + fileSizeMB, + totalSheets: workbook.SheetNames.length, + extractedLength: sheets.join('\n\n').length, + extractionTimeSeconds: extractionTime, + }); + + return sheets.join('\n\n'); + } catch (error) { + throw new Error(`Failed to parse Excel file: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // Handle CSV files + if (fileType === 'text/csv' || fileType === 'text/comma-separated-values') { + try { + const text = fileBuffer.toString('utf-8'); + // Convert CSV to readable format + const lines = text.split('\n').filter((line) => line.trim() !== ''); + return lines.join('\n'); + } catch (error) { + throw new Error(`Failed to parse CSV file: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // Handle plain text files + if (fileType === 'text/plain' || fileType.startsWith('text/')) { + try { + return fileBuffer.toString('utf-8'); + } catch (error) { + throw new Error(`Failed to read text file: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // Handle Word documents (.docx) - extract text using mammoth library + if (fileType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { + try { + const docxStartTime = Date.now(); + const fileSizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2); + + logger.info('Processing DOCX file', { + fileType, + fileSizeMB, + }); + + // Extract text from DOCX using mammoth + const result = await mammoth.extractRawText({ buffer: fileBuffer }); + const text = result.value; + + // Also extract formatted text (includes formatting information) + const formattedResult = await mammoth.convertToHtml({ buffer: fileBuffer }); + + // Use formatted HTML if available, otherwise use plain text + const extractedText = formattedResult.value || text; + + const extractionTime = ((Date.now() - docxStartTime) / 1000).toFixed(2); + logger.info('DOCX file processed', { + fileSizeMB, + extractedLength: extractedText.length, + extractionTimeSeconds: extractionTime, + }); + + // Convert HTML to plain text if needed (remove HTML tags) + if (formattedResult.value) { + // Simple HTML tag removal - keep text content and decode entities safely + const plainText = decodeBasicHtmlEntities( + extractedText.replace(/<[^>]*>/g, ' '), + ) + .replace(/\s+/g, ' ') // Replace multiple spaces with single space + .trim(); + + return plainText || text; + } + + return text; + } catch (error) { + logger.error('Failed to parse DOCX file', { + fileType, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw new Error(`Failed to parse DOCX file: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // Handle legacy Word documents (.doc) - not supported, suggest conversion + if (fileType === 'application/msword') { + throw new Error( + 'Legacy Word documents (.doc) are not supported. Please convert to .docx or PDF format before uploading.', + ); + } + + // For images and PDFs, use OpenAI vision API + const isImage = fileType.startsWith('image/'); + const isPdf = fileType === 'application/pdf'; + + if (isImage || isPdf) { + const base64Data = fileData; + const mimeType = fileType; + const fileSizeMB = (Buffer.from(fileData, 'base64').length / (1024 * 1024)).toFixed(2); + + logger.info('Extracting content from PDF/image using vision API', { + fileType: mimeType, + fileSizeMB, + }); + + const startTime = Date.now(); + + try { + const { text } = await generateText({ + model: openai('gpt-4o-mini'), // Using gpt-4o-mini for better text extraction + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Extract all text content from this document. Preserve the structure, formatting, and order of the content. Include all paragraphs, headings, lists, tables, and any other text elements. Return the extracted text in a clear, readable format.`, + }, + { + type: 'image', + image: `data:${mimeType};base64,${base64Data}`, + }, + ], + }, + ], + }); + + const extractionTime = ((Date.now() - startTime) / 1000).toFixed(2); + logger.info('Content extracted from PDF/image', { + fileType: mimeType, + extractedLength: text.length, + extractionTimeSeconds: extractionTime, + }); + + return text; + } catch (error) { + const extractionTime = ((Date.now() - startTime) / 1000).toFixed(2); + logger.error('Failed to extract content from PDF/image', { + fileType: mimeType, + fileSizeMB, + extractionTimeSeconds: extractionTime, + error: error instanceof Error ? error.message : 'Unknown error', + }); + throw new Error(`Failed to extract content: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + // For other file types that might be binary formats, provide helpful error message + throw new Error( + `Unsupported file type: ${fileType}. Supported formats: PDF, images (PNG, JPG, etc.), Excel (.xlsx, .xls), CSV, text files (.txt, .md), Word documents (.docx). Legacy Word documents (.doc) should be converted to .docx or PDF.`, + ); +} + diff --git a/apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts b/apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts new file mode 100644 index 000000000..571516fee --- /dev/null +++ b/apps/app/src/jobs/tasks/vector/process-knowledge-base-document.ts @@ -0,0 +1,283 @@ +import { logger, task } from '@trigger.dev/sdk'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { db } from '@db'; +import { batchUpsertEmbeddings } from '@/lib/vector/core/upsert-embedding'; +import { chunkText } from '@/lib/vector/utils/chunk-text'; +import { extractContentFromFile } from './helpers/extract-content-from-file'; + +/** + * Creates an S3 client instance for Trigger.dev tasks + */ +function createS3Client(): S3Client { + const region = process.env.APP_AWS_REGION || 'us-east-1'; + const accessKeyId = process.env.APP_AWS_ACCESS_KEY_ID; + const secretAccessKey = process.env.APP_AWS_SECRET_ACCESS_KEY; + + if (!accessKeyId || !secretAccessKey) { + throw new Error( + 'AWS S3 credentials are missing. Please set APP_AWS_ACCESS_KEY_ID and APP_AWS_SECRET_ACCESS_KEY environment variables in Trigger.dev.', + ); + } + + return new S3Client({ + region, + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +/** + * Extracts content from a Knowledge Base document stored in S3 + */ +async function extractContentFromKnowledgeBaseDocument( + s3Key: string, + fileType: string, +): Promise { + const knowledgeBaseBucket = process.env.APP_AWS_KNOWLEDGE_BASE_BUCKET; + + if (!knowledgeBaseBucket) { + throw new Error('Knowledge base bucket is not configured. Please set APP_AWS_KNOWLEDGE_BASE_BUCKET environment variable in Trigger.dev.'); + } + + const s3Client = createS3Client(); + + const getCommand = new GetObjectCommand({ + Bucket: knowledgeBaseBucket, + Key: s3Key, + }); + + const response = await s3Client.send(getCommand); + + if (!response.Body) { + throw new Error('Failed to retrieve file from S3'); + } + + // Convert stream to buffer + const chunks: Uint8Array[] = []; + for await (const chunk of response.Body as any) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + const base64Data = buffer.toString('base64'); + + // Use provided fileType or determine from content type + const detectedFileType = response.ContentType || fileType || 'application/octet-stream'; + + const content = await extractContentFromFile(base64Data, detectedFileType); + + return content; +} + +/** + * Task to process a Knowledge Base document and add it to the vector database + * Supports: PDF, Excel (.xlsx, .xls), CSV, text files (.txt, .md), Word documents (.docx), images (PNG, JPG, GIF, WebP, SVG) + */ +export const processKnowledgeBaseDocumentTask = task({ + id: 'process-knowledge-base-document', + retry: { + maxAttempts: 3, + }, + maxDuration: 1000 * 60 * 30, // 30 minutes for large files + run: async (payload: { + documentId: string; + organizationId: string; + }) => { + logger.info('Processing Knowledge Base document', { + documentId: payload.documentId, + organizationId: payload.organizationId, + }); + + try { + // Fetch document from database + const document = await db.knowledgeBaseDocument.findUnique({ + where: { + id: payload.documentId, + organizationId: payload.organizationId, + }, + }); + + if (!document) { + logger.error('Document not found', { + documentId: payload.documentId, + organizationId: payload.organizationId, + }); + return { + success: false, + documentId: payload.documentId, + error: 'Document not found', + }; + } + + // Update status to processing + await db.knowledgeBaseDocument.update({ + where: { id: document.id }, + data: { processingStatus: 'processing' }, + }); + + // Extract content from file in S3 + logger.info('Extracting content from file', { + documentId: document.id, + s3Key: document.s3Key, + fileType: document.fileType, + }); + + const content = await extractContentFromKnowledgeBaseDocument( + document.s3Key, + document.fileType, + ); + + if (!content || content.trim().length === 0) { + logger.warn('No content extracted from document', { + documentId: document.id, + }); + await db.knowledgeBaseDocument.update({ + where: { id: document.id }, + data: { + processingStatus: 'failed', + processedAt: new Date(), + }, + }); + return { + success: false, + documentId: document.id, + error: 'No content extracted from document', + }; + } + + logger.info('Content extracted successfully', { + documentId: document.id, + contentLength: content.length, + }); + + // Delete existing embeddings for this document (if any) + const { findEmbeddingsForSource } = await import('@/lib/vector/core/find-existing-embeddings'); + const existingEmbeddings = await findEmbeddingsForSource( + document.id, + 'knowledge_base_document', + payload.organizationId, + ); + + if (existingEmbeddings.length > 0) { + const { vectorIndex } = await import('@/lib/vector/core/client'); + if (vectorIndex) { + const idsToDelete = existingEmbeddings.map((e) => e.id); + try { + await vectorIndex.delete(idsToDelete); + logger.info('Deleted existing embeddings', { + documentId: document.id, + deletedCount: idsToDelete.length, + }); + } catch (error) { + logger.warn('Failed to delete existing embeddings', { + documentId: document.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + } + + // Chunk content for embedding + const chunks = chunkText(content, 500, 50); + + if (chunks.length === 0) { + logger.warn('No chunks created from content', { + documentId: document.id, + }); + await db.knowledgeBaseDocument.update({ + where: { id: document.id }, + data: { + processingStatus: 'failed', + processedAt: new Date(), + }, + }); + return { + success: false, + documentId: document.id, + error: 'No chunks created from content', + }; + } + + logger.info('Created chunks for embedding', { + documentId: document.id, + chunkCount: chunks.length, + }); + + // Create embeddings for each chunk + const updatedAt = document.updatedAt.toISOString(); + const chunkItems = chunks + .map((chunk, chunkIndex) => ({ + id: `knowledge_base_document_${document.id}_chunk${chunkIndex}`, + text: chunk, + metadata: { + organizationId: payload.organizationId, + sourceType: 'knowledge_base_document' as const, + sourceId: document.id, + content: chunk, + documentName: document.name, + updatedAt, + }, + })) + .filter((item) => item.text && item.text.trim().length > 0); + + if (chunkItems.length > 0) { + await batchUpsertEmbeddings(chunkItems); + logger.info('Successfully created embeddings', { + documentId: document.id, + embeddingCount: chunkItems.length, + }); + } + + // Update status to completed + await db.knowledgeBaseDocument.update({ + where: { id: document.id }, + data: { + processingStatus: 'completed', + processedAt: new Date(), + }, + }); + + logger.info('Successfully processed Knowledge Base document', { + documentId: document.id, + organizationId: payload.organizationId, + chunkCount: chunkItems.length, + }); + + return { + success: true, + documentId: document.id, + chunkCount: chunkItems.length, + }; + } catch (error) { + logger.error('Error processing Knowledge Base document', { + documentId: payload.documentId, + error: error instanceof Error ? error.message : 'Unknown error', + errorStack: error instanceof Error ? error.stack : undefined, + }); + + // Update status to failed + try { + await db.knowledgeBaseDocument.update({ + where: { id: payload.documentId }, + data: { + processingStatus: 'failed', + processedAt: new Date(), + }, + }); + } catch (updateError) { + logger.error('Failed to update document status to failed', { + documentId: payload.documentId, + error: updateError instanceof Error ? updateError.message : 'Unknown error', + }); + } + + return { + success: false, + documentId: payload.documentId, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } + }, +}); + diff --git a/apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts b/apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts new file mode 100644 index 000000000..f84395ccf --- /dev/null +++ b/apps/app/src/jobs/tasks/vector/process-knowledge-base-documents-orchestrator.ts @@ -0,0 +1,160 @@ +import { logger, metadata, task } from '@trigger.dev/sdk'; +import { processKnowledgeBaseDocumentTask } from './process-knowledge-base-document'; + +const BATCH_SIZE = 10; // Process 10 documents at a time + +/** + * Orchestrator task to process multiple Knowledge Base documents in parallel batches + * Similar to vendor-questionnaire-orchestrator, this manages the processing of multiple documents + */ +export const processKnowledgeBaseDocumentsOrchestratorTask = task({ + id: 'process-knowledge-base-documents-orchestrator', + retry: { + maxAttempts: 3, + }, + run: async (payload: { + documentIds: string[]; + organizationId: string; + }) => { + logger.info('Starting Knowledge Base documents processing orchestrator', { + organizationId: payload.organizationId, + documentCount: payload.documentIds.length, + }); + + if (payload.documentIds.length === 0) { + logger.info('No documents to process'); + return { + success: true, + processed: 0, + failed: 0, + }; + } + + // Initialize metadata for tracking progress + metadata.set('documentsTotal', payload.documentIds.length); + metadata.set('documentsCompleted', 0); + metadata.set('documentsFailed', 0); + metadata.set('documentsRemaining', payload.documentIds.length); + metadata.set('currentBatch', 0); + metadata.set('totalBatches', Math.ceil(payload.documentIds.length / BATCH_SIZE)); + + // Initialize individual document statuses - all start as 'pending' + payload.documentIds.forEach((documentId, index) => { + metadata.set(`document_${documentId}_status`, 'pending'); + }); + + const results: Array<{ + documentId: string; + success: boolean; + chunkCount?: number; + error?: string; + }> = []; + + // Process documents in batches + for (let i = 0; i < payload.documentIds.length; i += BATCH_SIZE) { + const batch = payload.documentIds.slice(i, i + BATCH_SIZE); + const batchNumber = Math.floor(i / BATCH_SIZE) + 1; + const totalBatches = Math.ceil(payload.documentIds.length / BATCH_SIZE); + + logger.info(`Processing batch ${batchNumber}/${totalBatches}`, { + batchSize: batch.length, + documentIds: batch, + }); + + // Update metadata + metadata.set('currentBatch', batchNumber); + + // Mark documents as processing + batch.forEach((documentId) => { + metadata.set(`document_${documentId}_status`, 'processing'); + }); + + // Use batchTriggerAndWait - this runs tasks in parallel and waits for all to complete + const batchItems = batch.map((documentId) => ({ + payload: { + documentId, + organizationId: payload.organizationId, + }, + })); + + const batchHandle = await processKnowledgeBaseDocumentTask.batchTriggerAndWait(batchItems); + + // Process batch results + batchHandle.runs.forEach((run, batchIdx) => { + const documentId = batch[batchIdx]; + + if (run.ok && run.output) { + const taskResult = run.output; + if (taskResult.success) { + results.push({ + documentId, + success: true, + chunkCount: taskResult.chunkCount, + }); + metadata.set(`document_${documentId}_status`, 'completed'); + metadata.increment('documentsCompleted'); + } else { + results.push({ + documentId, + success: false, + error: taskResult.error, + }); + metadata.set(`document_${documentId}_status`, 'failed'); + metadata.increment('documentsFailed'); + } + } else { + // Task failed + const errorMessage = + run.ok === false && run.error + ? run.error instanceof Error + ? run.error.message + : String(run.error) + : 'Unknown error'; + + logger.error('Document processing task failed', { + documentId, + error: errorMessage, + }); + results.push({ + documentId, + success: false, + error: errorMessage, + }); + metadata.set(`document_${documentId}_status`, 'failed'); + metadata.increment('documentsFailed'); + } + }); + + // Update remaining count + const completed = results.filter((r) => r.success).length + results.filter((r) => !r.success).length; + metadata.set('documentsRemaining', payload.documentIds.length - completed); + + logger.info(`Batch ${batchNumber}/${totalBatches} completed`, { + batchSize: batch.length, + successful: results.filter((r) => r.success).length, + failed: results.filter((r) => !r.success).length, + }); + } + + const successful = results.filter((r) => r.success).length; + const failed = results.filter((r) => !r.success).length; + + logger.info('Knowledge Base documents processing orchestrator completed', { + organizationId: payload.organizationId, + total: payload.documentIds.length, + successful, + failed, + }); + + // Mark as completed + metadata.set('completed', true); + + return { + success: true, + processed: successful, + failed, + results, + }; + }, +}); + diff --git a/apps/app/src/jobs/tasks/vendors/answer-question-helpers.ts b/apps/app/src/jobs/tasks/vendors/answer-question-helpers.ts index f616ce4b5..d28520e5f 100644 --- a/apps/app/src/jobs/tasks/vendors/answer-question-helpers.ts +++ b/apps/app/src/jobs/tasks/vendors/answer-question-helpers.ts @@ -1,7 +1,9 @@ import { findSimilarContent } from '@/lib/vector'; +import type { SimilarContentResult } from '@/lib/vector'; import { openai } from '@ai-sdk/openai'; import { logger } from '@trigger.dev/sdk'; import { generateText } from 'ai'; +import { deduplicateSources } from '@/app/(app)/[orgId]/security-questionnaire/utils/deduplicate-sources'; export interface AnswerWithSources { answer: string | null; @@ -22,7 +24,7 @@ export async function generateAnswerWithRAG( ): Promise { try { // Find similar content from vector database - const similarContent = await findSimilarContent(question, organizationId, 5); + const similarContent = await findSimilarContent(question, organizationId, 5) as SimilarContentResult[]; logger.info('Vector search results', { question: question.substring(0, 100), @@ -35,50 +37,35 @@ export async function generateAnswerWithRAG( })), }); - // Extract sources information and deduplicate by sourceName - // Multiple chunks from the same source (same policy/context) should appear as a single source - const sourceMap = new Map< - string, - { - sourceType: string; - sourceName?: string; - sourceId: string; - policyName?: string; - score: number; - } - >(); - - for (const result of similarContent) { - // Generate sourceName first to use as deduplication key - let sourceName: string | undefined; - if (result.policyName) { - sourceName = `Policy: ${result.policyName}`; - } else if (result.vendorName && result.questionnaireQuestion) { - sourceName = `Questionnaire: ${result.vendorName}`; - } else if (result.contextQuestion) { - sourceName = 'Context Q&A'; - } - - // Use sourceName as the unique key to prevent duplicates - // For policies: same policy name = same source - // For context: all context entries = single "Context Q&A" source - const key = sourceName || result.sourceId; - - // If we haven't seen this source, or this chunk has a higher score, use it - const existing = sourceMap.get(key); - if (!existing || result.score > existing.score) { - sourceMap.set(key, { - sourceType: result.sourceType, + // Extract sources information and deduplicate using universal utility + // Multiple chunks from the same source (same policy/context/manual answer/knowledge base document) should appear as a single source + // Note: sourceName is set for some types, but knowledge_base_document will be handled by deduplication function + const sources = deduplicateSources( + similarContent.map((result) => { + // Use any to avoid TypeScript narrowing issues, then assert correct type + const r = result as any as SimilarContentResult; + let sourceName: string | undefined; + if (r.policyName) { + sourceName = `Policy: ${r.policyName}`; + } else if (r.vendorName && r.questionnaireQuestion) { + sourceName = `Questionnaire: ${r.vendorName}`; + } else if (r.contextQuestion) { + sourceName = 'Context Q&A'; + } else if ((r.sourceType as string) === 'manual_answer') { + sourceName = 'Manual Answer'; + } + // Don't set sourceName for knowledge_base_document - let deduplication function handle it with filename + + return { + sourceType: r.sourceType, sourceName, - sourceId: result.sourceId, - policyName: result.policyName, - score: result.score, - }); - } - } - - // Convert map to array and sort by score (highest first) - const sources = Array.from(sourceMap.values()).sort((a, b) => b.score - a.score); + sourceId: r.sourceId, + policyName: r.policyName, + documentName: r.documentName, + score: r.score, + }; + }), + ); // If no relevant content found, return null if (similarContent.length === 0) { @@ -91,18 +78,31 @@ export async function generateAnswerWithRAG( // Build context from retrieved content const contextParts = similarContent.map((result, index) => { + // Use any to avoid TypeScript narrowing issues, then assert correct type + const r = result as any as SimilarContentResult; let sourceInfo = ''; - if (result.policyName) { - sourceInfo = `Source: Policy "${result.policyName}"`; - } else if (result.vendorName && result.questionnaireQuestion) { - sourceInfo = `Source: Questionnaire from "${result.vendorName}"`; - } else if (result.contextQuestion) { + if (r.policyName) { + sourceInfo = `Source: Policy "${r.policyName}"`; + } else if (r.vendorName && r.questionnaireQuestion) { + sourceInfo = `Source: Questionnaire from "${r.vendorName}"`; + } else if (r.contextQuestion) { sourceInfo = `Source: Context Q&A`; + } else if ((r.sourceType as string) === 'knowledge_base_document') { + const docName = r.documentName; + if (docName) { + sourceInfo = `Source: Knowledge Base Document "${docName}"`; + } else { + sourceInfo = `Source: Knowledge Base Document`; + } + } else if ((r.sourceType as string) === 'knowledge_base_document') { + sourceInfo = `Source: Knowledge Base Document`; + } else if ((r.sourceType as string) === 'manual_answer') { + sourceInfo = `Source: Manual Answer`; } else { - sourceInfo = `Source: ${result.sourceType}`; + sourceInfo = `Source: ${r.sourceType}`; } - return `[${index + 1}] ${sourceInfo}\n${result.content}`; + return `[${index + 1}] ${sourceInfo}\n${r.content}`; }); const context = contextParts.join('\n\n'); diff --git a/apps/app/src/jobs/tasks/vendors/parse-questionnaire.ts b/apps/app/src/jobs/tasks/vendors/parse-questionnaire.ts index 0facf6291..64118bc9d 100644 --- a/apps/app/src/jobs/tasks/vendors/parse-questionnaire.ts +++ b/apps/app/src/jobs/tasks/vendors/parse-questionnaire.ts @@ -635,6 +635,53 @@ export const parseQuestionnaireTask = task({ parseTimeSeconds: parseTime, totalTimeSeconds: totalTime, }); + + // Create questionnaire record in database + let questionnaireId: string; + try { + const fileName = payload.fileName || payload.url || payload.attachmentId || 'questionnaire'; + const s3Key = payload.s3Key || ''; + const fileType = payload.fileType || 'application/octet-stream'; + // For s3 input, we don't have fileData, so estimate size or use 0 + // The actual file size isn't critical for questionnaire records + const fileSize = payload.fileData ? Buffer.from(payload.fileData, 'base64').length : 0; + + const questionnaire = await db.questionnaire.create({ + data: { + filename: fileName, + s3Key: s3Key || '', + fileType, + fileSize, + organizationId: payload.organizationId, + status: 'completed', + parsedAt: new Date(), + totalQuestions: questionsAndAnswers.length, + answeredQuestions: 0, + questions: { + create: questionsAndAnswers.map((qa, index) => ({ + question: qa.question, + answer: qa.answer || null, + questionIndex: index, + status: qa.answer ? 'generated' : 'untouched', + })), + }, + }, + }); + + questionnaireId = questionnaire.id; + + logger.info('Questionnaire record created', { + questionnaireId, + questionCount: questionsAndAnswers.length, + }); + } catch (error) { + logger.error('Failed to create questionnaire record', { + error: error instanceof Error ? error.message : 'Unknown error', + }); + // Don't fail parsing if DB creation fails - we can still return results + // Frontend can handle saving later + questionnaireId = ''; + } // NOTE: We no longer add questionnaire Q&A pairs to the vector database // They are not used as a source for generating answers (only Policy and Context are used) @@ -693,6 +740,7 @@ export const parseQuestionnaireTask = task({ return { success: true, + questionnaireId, questionsAndAnswers, extractedContent: extractedContent.substring(0, 1000), // Return first 1000 chars for preview }; diff --git a/apps/app/src/lib/vector/README-MANUAL-ANSWERS.md b/apps/app/src/lib/vector/README-MANUAL-ANSWERS.md new file mode 100644 index 000000000..c787b5903 --- /dev/null +++ b/apps/app/src/lib/vector/README-MANUAL-ANSWERS.md @@ -0,0 +1,110 @@ +# Manual Answers Vector Database Integration + +## Overview + +Manual answers are automatically synced to the Upstash Vector database to improve AI answer generation quality. This document explains how to verify embeddings and troubleshoot sync issues. + +## Embedding ID Format + +When a manual answer is saved, it gets an embedding ID in the format: +``` +manual_answer_{manualAnswerId} +``` + +For example: +- Manual Answer ID: `sqma_abc123xyz` +- Embedding ID: `manual_answer_sqma_abc123xyz` + +## Verifying Embeddings + +### Method 1: Check Embedding ID in Response + +When you save a manual answer, the response includes the `embeddingId`: + +```typescript +const result = await saveManualAnswer.execute({ + question: "What is your data retention policy?", + answer: "We retain data for 7 years as per GDPR requirements.", +}); + +if (result.data?.success) { + console.log('Embedding ID:', result.data.embeddingId); + // Output: "manual_answer_sqma_abc123xyz" +} +``` + +### Method 2: Search in Upstash Vector Dashboard + +1. Go to your Upstash Vector dashboard +2. Use the search/filter functionality +3. Search for the embedding ID: `manual_answer_sqma_abc123xyz` +4. Or filter by metadata: + - `sourceType`: `manual_answer` + - `sourceId`: `sqma_abc123xyz` (the manual answer ID) + - `organizationId`: `org_123` + +## Sync Behavior + +### Synchronous Sync (Single Manual Answer) + +When a user saves a manual answer: +1. Manual answer is saved to the database +2. **Immediately** synced to vector DB (~1-2 seconds) +3. Embedding ID is returned in the response +4. Manual answer is **immediately available** for answer generation + +### Automatic Sync (Before Answer Generation) + +Before generating answers for questionnaires: +1. `syncOrganizationEmbeddings()` is called automatically +2. This ensures all manual answers are up-to-date +3. Manual answers are included in the RAG search + +### Background Sync (Delete All) + +When deleting all manual answers: +1. Orchestrator task is triggered in the background +2. Deletions happen in parallel batches (50 at a time) +3. Progress can be tracked via Trigger.dev dashboard + +## Troubleshooting + +### Embedding Not Found + +If an embedding is not found: + +1. **Check if sync succeeded**: Look at the `embeddingId` field in the save response - if present, sync was successful +2. **Check logs**: Look for errors in the server logs +3. **Manual sync**: The embedding will be synced automatically on the next `syncOrganizationEmbeddings()` call +4. **Check Upstash Vector Dashboard**: Use the dashboard to search for the embedding ID or filter by metadata +5. **Check Upstash Vector**: Verify the vector database is configured correctly + +### Sync Failed + +If sync fails: +- The manual answer is still saved in the database +- It will be synced automatically on the next organization sync +- Check server logs for detailed error messages + +## Testing + +To verify that an embedding was created: + +```typescript +// After saving a manual answer +const saveResult = await saveManualAnswer.execute({...}); + +if (saveResult.data?.embeddingId) { + console.log('Embedding ID:', saveResult.data.embeddingId); + // The embedding ID confirms that sync was successful + // You can verify it exists in the Upstash Vector Dashboard +} +``` + +## Related Files + +- `apps/app/src/lib/vector/sync/sync-manual-answer.ts` - Sync functions +- `apps/app/src/lib/vector/core/find-existing-embeddings.ts` - Functions to find embeddings by source +- `apps/app/src/jobs/tasks/vector/delete-manual-answer.ts` - Single deletion task +- `apps/app/src/jobs/tasks/vector/delete-all-manual-answers-orchestrator.ts` - Batch deletion orchestrator + diff --git a/apps/app/src/lib/vector/core/count-embeddings.ts b/apps/app/src/lib/vector/core/count-embeddings.ts new file mode 100644 index 000000000..821129b8f --- /dev/null +++ b/apps/app/src/lib/vector/core/count-embeddings.ts @@ -0,0 +1,142 @@ +import 'server-only'; + +import { vectorIndex } from './client'; +import { generateEmbedding } from './generate-embedding'; +import { logger } from '@/utils/logger'; + +/** + * Counts embeddings for a specific organization and source type + * Useful for debugging and verification + */ +export async function countEmbeddings( + organizationId: string, + sourceType?: 'policy' | 'context' | 'manual_answer', +): Promise<{ + total: number; + bySourceType: Record; + error?: string; +}> { + if (!vectorIndex) { + return { + total: 0, + bySourceType: {}, + error: 'Vector DB not configured', + }; + } + + try { + // Use organizationId as query to find all embeddings + const queryEmbedding = await generateEmbedding(organizationId); + + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 1000, // Max allowed by Upstash Vector + includeMetadata: true, + }); + + // Filter by organizationId + const orgResults = results.filter((result) => { + const metadata = result.metadata as any; + return metadata?.organizationId === organizationId; + }); + + // Count by sourceType + const bySourceType: Record = {}; + let total = 0; + + for (const result of orgResults) { + const metadata = result.metadata as any; + const st = metadata?.sourceType || 'unknown'; + + if (!sourceType || st === sourceType) { + bySourceType[st] = (bySourceType[st] || 0) + 1; + total++; + } + } + + logger.info('Counted embeddings', { + organizationId, + sourceType: sourceType || 'all', + total, + bySourceType, + }); + + return { + total, + bySourceType, + }; + } catch (error) { + logger.error('Failed to count embeddings', { + organizationId, + sourceType, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + total: 0, + bySourceType: {}, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} + +/** + * Lists all manual answer embeddings for an organization + * Useful for debugging + */ +export async function listManualAnswerEmbeddings( + organizationId: string, +): Promise> { + if (!vectorIndex) { + return []; + } + + try { + // Use organizationId as query + const queryEmbedding = await generateEmbedding(organizationId); + + const results = await vectorIndex.query({ + vector: queryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + // Filter for manual_answer type + const manualAnswerEmbeddings = results + .filter((result) => { + const metadata = result.metadata as any; + return ( + metadata?.organizationId === organizationId && + metadata?.sourceType === 'manual_answer' + ); + }) + .map((result) => { + const metadata = result.metadata as any; + return { + id: String(result.id), + sourceId: metadata?.sourceId || '', + content: metadata?.content || '', + updatedAt: metadata?.updatedAt, + }; + }); + + logger.info('Listed manual answer embeddings', { + organizationId, + count: manualAnswerEmbeddings.length, + ids: manualAnswerEmbeddings.map((e) => e.id), + }); + + return manualAnswerEmbeddings; + } catch (error) { + logger.error('Failed to list manual answer embeddings', { + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return []; + } +} + diff --git a/apps/app/src/lib/vector/core/find-existing-embeddings.ts b/apps/app/src/lib/vector/core/find-existing-embeddings.ts index 97f973b7a..ff1284010 100644 --- a/apps/app/src/lib/vector/core/find-existing-embeddings.ts +++ b/apps/app/src/lib/vector/core/find-existing-embeddings.ts @@ -7,19 +7,30 @@ import { logger } from '@/utils/logger'; export interface ExistingEmbedding { id: string; sourceId: string; - sourceType: 'policy' | 'context'; + sourceType: 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document'; updatedAt?: string; } /** - * Finds existing embeddings for a specific policy or context - * On-demand approach: checks only what we need, avoids Upstash Vector 1000 limit - * More efficient and performant than fetching all embeddings upfront + * Finds existing embeddings for a specific policy, context, manual answer, or knowledge base document + * Uses multiple query strategies to ensure we find ALL chunks: + * 1. Query with organizationId (finds org-wide embeddings) + * 2. Query with sourceId (finds source-specific embeddings) + * 3. Query with combined query (organizationId + sourceId) + * 4. Query with documentName (for knowledge_base_document, finds chunks semantically similar to filename) + * 5. Query with content from already-found chunks (uses both chunk content AND filename from metadata) + * 6. Query with generic terms (for knowledge_base_document) + * + * Note: We store `documentName` (filename) in metadata for all knowledge_base_document chunks. + * This allows us to use the filename as a query vector to find related chunks. + * + * This approach ensures we find all chunks even if org has >1000 total embeddings. */ export async function findEmbeddingsForSource( sourceId: string, - sourceType: 'policy' | 'context', + sourceType: 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', organizationId: string, + documentName?: string, // Optional: for knowledge_base_document, helps find chunks semantically similar to filename ): Promise { if (!vectorIndex) { return []; @@ -30,41 +41,314 @@ export async function findEmbeddingsForSource( } try { - // Create a specific query that will match this source - // Using sourceId in the query helps find exact matches - const queryText = sourceType === 'policy' - ? `policy ${sourceId} security compliance` - : `context ${sourceId} question answer`; + const allResults = new Map(); - const queryEmbedding = await generateEmbedding(queryText); - - // Use smaller topK since we're looking for specific source - // Upstash Vector limit is 1000, but we only need a few results - const results = await vectorIndex.query({ - vector: queryEmbedding, - topK: 100, // Small number - we're looking for specific source - includeMetadata: true, - }); + // Strategy 1: Query with organizationId + try { + const orgQueryEmbedding = await generateEmbedding(organizationId); + const orgResults = await vectorIndex.query({ + vector: orgQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); - // Filter by exact sourceId match and organizationId - const matchingEmbeddings = results - .filter((result) => { + for (const result of orgResults) { const metadata = result.metadata as any; - return ( + if ( metadata?.organizationId === organizationId && metadata?.sourceType === sourceType && metadata?.sourceId === sourceId - ); - }) - .map((result) => { + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: metadata?.sourceId || '', + sourceType: metadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: metadata?.updatedAt, + }); + } + } + } + } catch (error) { + logger.warn('Error in organizationId query strategy', { + sourceId, + sourceType, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + + // Strategy 2: Query with sourceId (more specific, likely to find document chunks) + try { + const sourceQueryEmbedding = await generateEmbedding(sourceId); + const sourceResults = await vectorIndex.query({ + vector: sourceQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + for (const result of sourceResults) { const metadata = result.metadata as any; - return { - id: String(result.id), - sourceId: metadata?.sourceId || '', - sourceType: metadata?.sourceType as 'policy' | 'context', - updatedAt: metadata?.updatedAt, - }; + if ( + metadata?.organizationId === organizationId && + metadata?.sourceType === sourceType && + metadata?.sourceId === sourceId + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: metadata?.sourceId || '', + sourceType: metadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: metadata?.updatedAt, + }); + } + } + } + } catch (error) { + logger.warn('Error in sourceId query strategy', { + sourceId, + sourceType, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', }); + } + + // Strategy 3: Query with combined query (organizationId + sourceId) + // This helps find chunks that might be semantically closer to the combination + try { + const combinedQuery = `${organizationId} ${sourceId}`; + const combinedQueryEmbedding = await generateEmbedding(combinedQuery); + const combinedResults = await vectorIndex.query({ + vector: combinedQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + for (const result of combinedResults) { + const metadata = result.metadata as any; + if ( + metadata?.organizationId === organizationId && + metadata?.sourceType === sourceType && + metadata?.sourceId === sourceId + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: metadata?.sourceId || '', + sourceType: metadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: metadata?.updatedAt, + }); + } + } + } + } catch (error) { + logger.warn('Error in combined query strategy', { + sourceId, + sourceType, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + + // Strategy 4: Query with documentName (for knowledge_base_document only) + // This helps find chunks that are semantically similar to the document filename + if (sourceType === 'knowledge_base_document' && documentName) { + try { + const docNameQueryEmbedding = await generateEmbedding(documentName); + const docNameResults = await vectorIndex.query({ + vector: docNameQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + for (const result of docNameResults) { + const metadata = result.metadata as any; + if ( + metadata?.organizationId === organizationId && + metadata?.sourceType === sourceType && + metadata?.sourceId === sourceId + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: metadata?.sourceId || '', + sourceType: metadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: metadata?.updatedAt, + }); + } + } + } + } catch (error) { + logger.warn('Error in documentName query strategy', { + sourceId, + sourceType, + organizationId, + documentName, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + + // Strategy 5: Query with content from already-found chunks (for knowledge_base_document) + // This helps find chunks that are semantically similar to chunks we've already found + // This is especially useful for finding the "last remaining" chunks + if (sourceType === 'knowledge_base_document' && allResults.size > 0) { + try { + // Get a few chunks we've already found and use their content/metadata as query vectors + const foundChunkIds = Array.from(allResults.keys()).slice(0, 3); // Use first 3 chunks + + // Query Upstash Vector to get the actual content/metadata of these chunks + // Then use that content AND filename to find more chunks + for (const chunkId of foundChunkIds) { + try { + // Fetch the chunk by ID to get its content and metadata + const chunkResult = await vectorIndex.fetch([chunkId]); + if (chunkResult && chunkResult.length > 0) { + const chunk = chunkResult[0]; + if (!chunk) continue; + const metadata = chunk.metadata as any; + const chunkContent = metadata?.content as string; + const chunkDocumentName = metadata?.documentName as string; + + // Strategy 5a: Query with chunk content + if (chunkContent && chunkContent.length > 50) { + // Use a portion of the chunk content as query (first 200 chars) + const contentQuery = chunkContent.substring(0, 200); + const contentQueryEmbedding = await generateEmbedding(contentQuery); + const contentResults = await vectorIndex.query({ + vector: contentQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + for (const result of contentResults) { + const resultMetadata = result.metadata as any; + if ( + resultMetadata?.organizationId === organizationId && + resultMetadata?.sourceType === sourceType && + resultMetadata?.sourceId === sourceId + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: resultMetadata?.sourceId || '', + sourceType: resultMetadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: resultMetadata?.updatedAt, + }); + } + } + } + } + + // Strategy 5b: Query with filename from chunk metadata (if available) + // This helps find chunks that might be semantically related to the filename + if (chunkDocumentName && chunkDocumentName.length > 0) { + const filenameQueryEmbedding = await generateEmbedding(chunkDocumentName); + const filenameResults = await vectorIndex.query({ + vector: filenameQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + for (const result of filenameResults) { + const resultMetadata = result.metadata as any; + if ( + resultMetadata?.organizationId === organizationId && + resultMetadata?.sourceType === sourceType && + resultMetadata?.sourceId === sourceId + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: resultMetadata?.sourceId || '', + sourceType: resultMetadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: resultMetadata?.updatedAt, + }); + } + } + } + } + } + } catch (chunkError) { + logger.warn('Error querying with chunk content/filename', { + chunkId, + error: chunkError instanceof Error ? chunkError.message : 'Unknown error', + }); + } + } + } catch (error) { + logger.warn('Error in chunk content/filename query strategy', { + sourceId, + sourceType, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + + // Strategy 6: Query with generic terms (for knowledge_base_document) + // Use generic terms that are likely to match document content + if (sourceType === 'knowledge_base_document') { + const genericQueries = [ + 'document information content', + 'knowledge base document', + 'file content text', + ]; + + for (const genericQuery of genericQueries) { + try { + const genericQueryEmbedding = await generateEmbedding(genericQuery); + const genericResults = await vectorIndex.query({ + vector: genericQueryEmbedding, + topK: 1000, + includeMetadata: true, + }); + + for (const result of genericResults) { + const metadata = result.metadata as any; + if ( + metadata?.organizationId === organizationId && + metadata?.sourceType === sourceType && + metadata?.sourceId === sourceId + ) { + const id = String(result.id); + if (!allResults.has(id)) { + allResults.set(id, { + id, + sourceId: metadata?.sourceId || '', + sourceType: metadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', + updatedAt: metadata?.updatedAt, + }); + } + } + } + } catch (error) { + logger.warn('Error in generic query strategy', { + genericQuery, + sourceId, + sourceType, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } + } + + const matchingEmbeddings = Array.from(allResults.values()); + + logger.info('Found embeddings for source', { + sourceId, + sourceType, + organizationId, + count: matchingEmbeddings.length, + uniqueIds: matchingEmbeddings.map((e) => e.id), + }); return matchingEmbeddings; } catch (error) { @@ -116,7 +400,10 @@ export async function findAllOrganizationEmbeddings( return ( metadata?.organizationId === organizationId && metadata?.sourceType !== 'questionnaire' && - (metadata?.sourceType === 'policy' || metadata?.sourceType === 'context') + (metadata?.sourceType === 'policy' || + metadata?.sourceType === 'context' || + metadata?.sourceType === 'manual_answer' || + metadata?.sourceType === 'knowledge_base_document') ); }) .map((result) => { @@ -124,7 +411,7 @@ export async function findAllOrganizationEmbeddings( return { id: String(result.id), sourceId: metadata?.sourceId || '', - sourceType: metadata?.sourceType as 'policy' | 'context', + sourceType: metadata?.sourceType as 'policy' | 'context' | 'manual_answer' | 'knowledge_base_document', updatedAt: metadata?.updatedAt, }; }); diff --git a/apps/app/src/lib/vector/core/find-similar.ts b/apps/app/src/lib/vector/core/find-similar.ts index 934932eeb..233099a51 100644 --- a/apps/app/src/lib/vector/core/find-similar.ts +++ b/apps/app/src/lib/vector/core/find-similar.ts @@ -8,13 +8,14 @@ export interface SimilarContentResult { id: string; score: number; content: string; - sourceType: 'policy' | 'context' | 'document_hub' | 'attachment' | 'questionnaire'; + sourceType: 'policy' | 'context' | 'document_hub' | 'attachment' | 'questionnaire' | 'manual_answer' | 'knowledge_base_document'; sourceId: string; policyName?: string; contextQuestion?: string; vendorId?: string; vendorName?: string; questionnaireQuestion?: string; + documentName?: string; } /** @@ -61,24 +62,25 @@ export async function findSimilarContent( const metadata = result.metadata as any; const hasCorrectOrg = metadata?.organizationId === organizationId; const hasMinScore = result.score >= MIN_SIMILARITY_SCORE; - // Exclude questionnaire Q&A from results - we only use Policy and Context as sources + // Exclude questionnaire Q&A from results - we use Policy, Context, and Manual Answers as sources const isNotQuestionnaire = metadata?.sourceType !== 'questionnaire'; return hasCorrectOrg && hasMinScore && isNotQuestionnaire; }) .slice(0, limit) // Take only the top N after filtering - .map((result) => { + .map((result): SimilarContentResult => { const metadata = result.metadata as any; return { id: String(result.id), score: result.score, content: metadata?.content || '', - sourceType: metadata?.sourceType || 'policy', + sourceType: (metadata?.sourceType || 'policy') as SimilarContentResult['sourceType'], sourceId: metadata?.sourceId || '', policyName: metadata?.policyName, contextQuestion: metadata?.contextQuestion, vendorId: metadata?.vendorId, vendorName: metadata?.vendorName, questionnaireQuestion: metadata?.questionnaireQuestion, + documentName: metadata?.documentName, }; }); diff --git a/apps/app/src/lib/vector/core/upsert-embedding.ts b/apps/app/src/lib/vector/core/upsert-embedding.ts index 9cf199471..61dfadd77 100644 --- a/apps/app/src/lib/vector/core/upsert-embedding.ts +++ b/apps/app/src/lib/vector/core/upsert-embedding.ts @@ -4,7 +4,7 @@ import { vectorIndex } from './client'; import { generateEmbedding } from './generate-embedding'; import { logger } from '@/utils/logger'; -export type SourceType = 'policy' | 'context' | 'document_hub' | 'attachment' | 'questionnaire'; +export type SourceType = 'policy' | 'context' | 'document_hub' | 'attachment' | 'questionnaire' | 'manual_answer' | 'knowledge_base_document'; export interface EmbeddingMetadata { organizationId: string; @@ -16,6 +16,7 @@ export interface EmbeddingMetadata { vendorId?: string; vendorName?: string; questionnaireQuestion?: string; + documentName?: string; updatedAt?: string; // ISO timestamp for incremental sync comparison } @@ -31,7 +32,14 @@ export async function upsertEmbedding( metadata: EmbeddingMetadata, ): Promise { if (!vectorIndex) { - throw new Error('Upstash Vector is not configured'); + const errorMsg = 'Upstash Vector is not configured - check UPSTASH_VECTOR_REST_URL and UPSTASH_VECTOR_REST_TOKEN'; + logger.error(errorMsg, { + id, + sourceType: metadata.sourceType, + hasUrl: !!process.env.UPSTASH_VECTOR_REST_URL, + hasToken: !!process.env.UPSTASH_VECTOR_REST_TOKEN, + }); + throw new Error(errorMsg); } if (!text || text.trim().length === 0) { @@ -43,25 +51,56 @@ export async function upsertEmbedding( // Generate embedding const embedding = await generateEmbedding(text); + // Prepare metadata + const vectorMetadata = { + organizationId: metadata.organizationId, + sourceType: metadata.sourceType, + sourceId: metadata.sourceId, + content: text.substring(0, 1000), // Store first 1000 chars for reference + ...(metadata.policyName && { policyName: metadata.policyName }), + ...(metadata.contextQuestion && { contextQuestion: metadata.contextQuestion }), + ...(metadata.vendorId && { vendorId: metadata.vendorId }), + ...(metadata.vendorName && { vendorName: metadata.vendorName }), + ...(metadata.questionnaireQuestion && { questionnaireQuestion: metadata.questionnaireQuestion }), + ...(metadata.documentName && { documentName: metadata.documentName }), + ...(metadata.updatedAt && { updatedAt: metadata.updatedAt }), + }; + + // Log detailed info for manual_answer type (for debugging) + if (metadata.sourceType === 'manual_answer') { + logger.info('Upserting manual answer embedding', { + id, + embeddingId: id, + vectorLength: embedding.length, + vectorPreview: embedding.slice(0, 5).map(v => v.toFixed(6)), // First 5 dimensions + vectorStats: { + min: Math.min(...embedding), + max: Math.max(...embedding), + mean: embedding.reduce((a, b) => a + b, 0) / embedding.length, + }, + metadata: vectorMetadata, + textPreview: text.substring(0, 200), + }); + } + // Upsert into Upstash Vector - await vectorIndex.upsert({ + const upsertResult = await vectorIndex.upsert({ id, vector: embedding, - metadata: { - organizationId: metadata.organizationId, - sourceType: metadata.sourceType, - sourceId: metadata.sourceId, - content: text.substring(0, 1000), // Store first 1000 chars for reference - ...(metadata.policyName && { policyName: metadata.policyName }), - ...(metadata.contextQuestion && { contextQuestion: metadata.contextQuestion }), - ...(metadata.vendorId && { vendorId: metadata.vendorId }), - ...(metadata.vendorName && { vendorName: metadata.vendorName }), - ...(metadata.questionnaireQuestion && { questionnaireQuestion: metadata.questionnaireQuestion }), - ...(metadata.updatedAt && { updatedAt: metadata.updatedAt }), - }, + metadata: vectorMetadata, }); - // Removed per-embedding success logging for performance (only log errors) + // Log success for manual_answer type with upsert result + if (metadata.sourceType === 'manual_answer') { + logger.info('✅ Successfully upserted manual answer embedding', { + id, + embeddingId: id, + organizationId: metadata.organizationId, + sourceId: metadata.sourceId, + upsertResult: upsertResult ? 'success' : 'unknown', + vectorIndexConfigured: !!vectorIndex, + }); + } } catch (error) { logger.error('Failed to upsert embedding', { id, @@ -132,6 +171,7 @@ export async function batchUpsertEmbeddings( ...(item.metadata.questionnaireQuestion && { questionnaireQuestion: item.metadata.questionnaireQuestion, }), + ...(item.metadata.documentName && { documentName: item.metadata.documentName }), ...(item.metadata.updatedAt && { updatedAt: item.metadata.updatedAt }), }, }); diff --git a/apps/app/src/lib/vector/index.ts b/apps/app/src/lib/vector/index.ts index 2a950457c..8e1baa8ac 100644 --- a/apps/app/src/lib/vector/index.ts +++ b/apps/app/src/lib/vector/index.ts @@ -14,7 +14,9 @@ export type { ExistingEmbedding } from './core/find-existing-embeddings'; // Sync functionality export { syncOrganizationEmbeddings } from './sync/sync-organization'; +export { syncManualAnswerToVector, deleteManualAnswerFromVector } from './sync/sync-manual-answer'; // Utilities +export { countEmbeddings, listManualAnswerEmbeddings } from './core/count-embeddings'; export { chunkText } from './utils/chunk-text'; export { extractTextFromPolicy } from './utils/extract-policy-text'; diff --git a/apps/app/src/lib/vector/sync/sync-manual-answer.ts b/apps/app/src/lib/vector/sync/sync-manual-answer.ts new file mode 100644 index 000000000..fcddffa55 --- /dev/null +++ b/apps/app/src/lib/vector/sync/sync-manual-answer.ts @@ -0,0 +1,174 @@ +import 'server-only'; + +import { upsertEmbedding } from '../core/upsert-embedding'; +import { vectorIndex } from '../core/client'; +import { db } from '@db'; +import { logger } from '@/utils/logger'; + +/** + * Syncs a single manual answer to vector database SYNCHRONOUSLY + * Fast operation (~1-2 seconds) - acceptable for UX + * This ensures manual answers are immediately available for answer generation + */ +export async function syncManualAnswerToVector( + manualAnswerId: string, + organizationId: string, +): Promise<{ success: boolean; error?: string; embeddingId?: string }> { + // Check if vectorIndex is configured + if (!vectorIndex) { + logger.error('❌ Upstash Vector not configured - check UPSTASH_VECTOR_REST_URL and UPSTASH_VECTOR_REST_TOKEN', { + manualAnswerId, + organizationId, + hasUrl: !!process.env.UPSTASH_VECTOR_REST_URL, + hasToken: !!process.env.UPSTASH_VECTOR_REST_TOKEN, + }); + return { success: false, error: 'Vector DB not configured' }; + } + + logger.info('🔍 Vector Index configuration check', { + vectorIndexExists: !!vectorIndex, + manualAnswerId, + organizationId, + }); + + try { + const manualAnswer = await db.securityQuestionnaireManualAnswer.findUnique({ + where: { id: manualAnswerId, organizationId }, + }); + + if (!manualAnswer) { + logger.warn('Manual answer not found for sync', { manualAnswerId, organizationId }); + return { success: false, error: 'Manual answer not found' }; + } + + // Create embedding ID: manual_answer_{id} + const embeddingId = `manual_answer_${manualAnswerId}`; + + // Combine question and answer for better semantic search + const text = `${manualAnswer.question}\n\n${manualAnswer.answer}`; + + logger.info('🔄 Starting sync manual answer to vector DB', { + manualAnswerId, + organizationId, + embeddingId, + question: manualAnswer.question.substring(0, 100), + answer: manualAnswer.answer.substring(0, 100), + textLength: text.length, + }); + + await upsertEmbedding(embeddingId, text, { + organizationId, + sourceType: 'manual_answer', + sourceId: manualAnswerId, + content: text, + updatedAt: manualAnswer.updatedAt.toISOString(), + }); + + // Verify the embedding was actually added by querying for it + try { + const { findEmbeddingsForSource } = await import('../core/find-existing-embeddings'); + const foundEmbeddings = await findEmbeddingsForSource( + manualAnswerId, + 'manual_answer', + organizationId, + ); + + const wasFound = foundEmbeddings.some((e) => e.id === embeddingId); + + logger.info('✅ Successfully synced manual answer to vector DB', { + manualAnswerId, + organizationId, + embeddingId, + question: manualAnswer.question.substring(0, 100), + answer: manualAnswer.answer.substring(0, 100), + verified: wasFound, + foundEmbeddingsCount: foundEmbeddings.length, + foundEmbeddingIds: foundEmbeddings.map((e) => e.id), + metadata: { + organizationId, + sourceType: 'manual_answer', + sourceId: manualAnswerId, + updatedAt: manualAnswer.updatedAt.toISOString(), + }, + }); + + if (!wasFound) { + logger.warn('⚠️ Embedding was upserted but not found in verification query', { + embeddingId, + manualAnswerId, + organizationId, + }); + } + } catch (verifyError) { + logger.warn('Failed to verify embedding after upsert', { + embeddingId, + manualAnswerId, + error: verifyError instanceof Error ? verifyError.message : 'Unknown error', + }); + } + return { + success: true, + embeddingId, // Return embedding ID for verification + }; + } catch (error) { + logger.error('Failed to sync manual answer to vector DB', { + manualAnswerId, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} + +/** + * Deletes manual answer from vector database + * Called when manual answer is deleted + */ +export async function deleteManualAnswerFromVector( + manualAnswerId: string, + organizationId: string, +): Promise<{ success: boolean; error?: string }> { + if (!vectorIndex) { + return { success: false, error: 'Vector DB not configured' }; + } + + try { + // Find existing embeddings for this manual answer + // We need to search for embeddings with this sourceId + const embeddingId = `manual_answer_${manualAnswerId}`; + + // Try to delete directly by ID (most efficient) + try { + await vectorIndex.delete([embeddingId]); + logger.info('Deleted manual answer from vector DB', { + manualAnswerId, + organizationId, + embeddingId, + }); + return { success: true }; + } catch (deleteError) { + // If direct delete fails (embedding might not exist), log and continue + logger.warn('Failed to delete manual answer embedding (may not exist)', { + manualAnswerId, + embeddingId, + error: deleteError instanceof Error ? deleteError.message : 'Unknown error', + }); + // Still return success - embedding might not exist + return { success: true }; + } + } catch (error) { + logger.error('Failed to delete manual answer from vector DB', { + manualAnswerId, + organizationId, + error: error instanceof Error ? error.message : 'Unknown error', + }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} + diff --git a/apps/app/src/lib/vector/sync/sync-organization.ts b/apps/app/src/lib/vector/sync/sync-organization.ts index d5b14d756..342fe5ba3 100644 --- a/apps/app/src/lib/vector/sync/sync-organization.ts +++ b/apps/app/src/lib/vector/sync/sync-organization.ts @@ -8,6 +8,8 @@ import { deleteOrganizationEmbeddings } from '../core/delete-embeddings'; import { findAllOrganizationEmbeddings, type ExistingEmbedding } from '../core/find-existing-embeddings'; import { vectorIndex } from '../core/client'; import { logger } from '@/utils/logger'; +import { tasks } from '@trigger.dev/sdk'; +import { processKnowledgeBaseDocumentTask } from '@/jobs/tasks/vector/process-knowledge-base-document'; /** * Lock map to prevent concurrent syncs for the same organization @@ -307,10 +309,165 @@ async function performSync(organizationId: string): Promise { total: contextEntries.length, }); - // Step 6: Delete orphaned embeddings (policies/context that no longer exist in DB) + // Step 6: Sync manual answers (ensure they're always up-to-date) + const manualAnswers = await db.securityQuestionnaireManualAnswer.findMany({ + where: { organizationId }, + select: { + id: true, + question: true, + answer: true, + updatedAt: true, + }, + }); + + logger.info('Syncing manual answers', { + organizationId, + count: manualAnswers.length, + }); + + let manualAnswersCreated = 0; + let manualAnswersUpdated = 0; + let manualAnswersSkipped = 0; + + if (manualAnswers.length > 0) { + const manualAnswerItems = manualAnswers.map((ma) => { + const embeddingId = `manual_answer_${ma.id}`; + const text = `${ma.question}\n\n${ma.answer}`; + const updatedAt = ma.updatedAt.toISOString(); + + // Check if embedding exists and needs update + const existingManualAnswerEmbeddings = existingEmbeddings.get(ma.id) || []; + const needsUpdate = existingManualAnswerEmbeddings.length === 0 || + existingManualAnswerEmbeddings[0]?.updatedAt !== updatedAt; + + if (needsUpdate) { + if (existingManualAnswerEmbeddings.length === 0) { + manualAnswersCreated++; + } else { + manualAnswersUpdated++; + } + } else { + manualAnswersSkipped++; + } + + return { + id: embeddingId, + text, + metadata: { + organizationId, + sourceType: 'manual_answer' as const, + sourceId: ma.id, + content: text, + updatedAt, + }, + }; + }); + + // Batch upsert all manual answers + if (manualAnswerItems.length > 0) { + await batchUpsertEmbeddings(manualAnswerItems); + } + } + + logger.info('Manual answers sync completed', { + organizationId, + created: manualAnswersCreated, + updated: manualAnswersUpdated, + skipped: manualAnswersSkipped, + total: manualAnswers.length, + }); + + // Step 7: Sync Knowledge Base documents + // Note: Documents are processed via Trigger.dev tasks, but we sync completed documents here + // and trigger processing for pending/failed documents + const knowledgeBaseDocuments = await db.knowledgeBaseDocument.findMany({ + where: { organizationId }, + select: { + id: true, + name: true, + s3Key: true, + fileType: true, + processingStatus: true, + updatedAt: true, + }, + }); + + logger.info('Found Knowledge Base documents to sync', { + organizationId, + count: knowledgeBaseDocuments.length, + }); + + let documentsProcessed = 0; + let documentsTriggered = 0; + let documentsSkipped = 0; + + // Trigger processing for pending/failed documents + for (const document of knowledgeBaseDocuments) { + if (document.processingStatus === 'pending' || document.processingStatus === 'failed') { + try { + // Trigger Trigger.dev task to process document + await tasks.trigger( + 'process-knowledge-base-document', + { + documentId: document.id, + organizationId, + }, + ); + documentsTriggered++; + } catch (error) { + logger.warn('Failed to trigger document processing', { + documentId: document.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } else if (document.processingStatus === 'completed') { + // Check if embeddings exist and are up-to-date + const documentEmbeddings = existingEmbeddings.get(document.id) || []; + const documentUpdatedAt = document.updatedAt.toISOString(); + + const needsUpdate = documentEmbeddings.length === 0 || + documentEmbeddings.some((e: ExistingEmbedding) => !e.updatedAt || e.updatedAt < documentUpdatedAt); + + if (needsUpdate) { + // Trigger reprocessing if embeddings are outdated + try { + await tasks.trigger( + 'process-knowledge-base-document', + { + documentId: document.id, + organizationId, + }, + ); + documentsTriggered++; + } catch (error) { + logger.warn('Failed to trigger document reprocessing', { + documentId: document.id, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } else { + documentsSkipped++; + } + documentsProcessed++; + } else { + documentsSkipped++; + } + } + + logger.info('Knowledge Base documents sync completed', { + organizationId, + processed: documentsProcessed, + triggered: documentsTriggered, + skipped: documentsSkipped, + total: knowledgeBaseDocuments.length, + }); + + // Step 8: Delete orphaned embeddings (policies/context/manual_answers/knowledge_base_documents that no longer exist in DB) // Use the embeddings we already fetched (no additional API call needed) const dbPolicyIds = new Set(policies.map(p => p.id)); const dbContextIds = new Set(contextEntries.map(c => c.id)); + const dbManualAnswerIds = new Set(manualAnswers.map(ma => ma.id)); + const dbKnowledgeBaseDocumentIds = new Set(knowledgeBaseDocuments.map(d => d.id)); let orphanedDeleted = 0; // Check for orphaned embeddings using the pre-fetched map @@ -318,9 +475,13 @@ async function performSync(organizationId: string): Promise { for (const [sourceId, embeddings] of existingEmbeddings.entries()) { const isPolicy = embeddings[0]?.sourceType === 'policy'; const isContext = embeddings[0]?.sourceType === 'context'; + const isManualAnswer = embeddings[0]?.sourceType === 'manual_answer'; + const isKnowledgeBaseDocument = embeddings[0]?.sourceType === 'knowledge_base_document'; const shouldExist = (isPolicy && dbPolicyIds.has(sourceId)) || - (isContext && dbContextIds.has(sourceId)); + (isContext && dbContextIds.has(sourceId)) || + (isManualAnswer && dbManualAnswerIds.has(sourceId)) || + (isKnowledgeBaseDocument && dbKnowledgeBaseDocumentIds.has(sourceId)); if (!shouldExist && vectorIndex) { // Delete orphaned embeddings @@ -330,7 +491,7 @@ async function performSync(organizationId: string): Promise { orphanedDeleted += idsToDelete.length; logger.info('Deleted orphaned embeddings', { sourceId, - sourceType: isPolicy ? 'policy' : 'context', + sourceType: isPolicy ? 'policy' : isContext ? 'context' : isManualAnswer ? 'manual_answer' : 'knowledge_base_document', deletedCount: idsToDelete.length, }); } catch (error) { @@ -363,6 +524,18 @@ async function performSync(organizationId: string): Promise { updated: contextUpdated, skipped: contextSkipped, }, + manualAnswers: { + total: manualAnswers.length, + created: manualAnswersCreated, + updated: manualAnswersUpdated, + skipped: manualAnswersSkipped, + }, + knowledgeBaseDocuments: { + total: knowledgeBaseDocuments.length, + processed: documentsProcessed, + triggered: documentsTriggered, + skipped: documentsSkipped, + }, orphanedDeleted, }); } catch (error) { diff --git a/bun.lock b/bun.lock index 3c9f1cd5e..1a758f815 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "comp", @@ -201,6 +202,7 @@ "geist": "^1.3.1", "jspdf": "^3.0.2", "lucide-react": "^0.544.0", + "mammoth": "^1.11.0", "motion": "^12.9.2", "next": "^15.4.6", "next-safe-action": "^8.0.3", @@ -2379,6 +2381,8 @@ "@webgpu/types": ["@webgpu/types@0.1.66", "", {}, "sha512-YA2hLrwLpDsRueNDXIMqN9NTzD6bCDkuXbOSe0heS+f8YE8usA6Gbv1prj81pzVHrbaAma7zObnIC+I6/sXJgA=="], + "@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="], + "@xobotyi/scrollbar-width": ["@xobotyi/scrollbar-width@1.9.5", "", {}, "sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ=="], "@xtuc/ieee754": ["@xtuc/ieee754@1.2.0", "", {}, "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA=="], @@ -2575,6 +2579,8 @@ "block-stream": ["block-stream@0.0.9", "", { "dependencies": { "inherits": "~2.0.0" } }, "sha512-OorbnJVPII4DuUKbjARAe8u8EfqOmkEEaSFIyoQ7OjTHn6kafxWl0wLgoZ2rXaYd7MyLcDaU4TmhfxtwgcccMQ=="], + "bluebird": ["bluebird@3.4.7", "", {}, "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA=="], + "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="], "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], @@ -2973,6 +2979,8 @@ "diff": ["diff@4.0.2", "", {}, "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A=="], + "dingbat-to-unicode": ["dingbat-to-unicode@1.0.1", "", {}, "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w=="], + "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], "discord-api-types": ["discord-api-types@0.38.34", "", {}, "sha512-muq7xKGznj5MSFCzuIm/2TO7DpttuomUTemVM82fRqgnMl70YRzEyY24jlbiV6R9tzOTq6A6UnZ0bsfZeKD38Q=="], @@ -3009,6 +3017,8 @@ "dub": ["dub@0.66.5", "", { "dependencies": { "zod": "^3.20.0" } }, "sha512-VIaRWbjAv0w3R317LRjPh7G4Ws9wRqMwvaBQWdFxghQfkJTh457NZFNOfywYX5DIjWBhZLPG4/itvoi3AlcxxQ=="], + "duck": ["duck@0.1.12", "", { "dependencies": { "underscore": "^1.13.1" } }, "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="], @@ -3863,6 +3873,8 @@ "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], + "lop": ["lop@0.4.2", "", { "dependencies": { "duck": "^0.1.12", "option": "~0.2.1", "underscore": "^1.13.1" } }, "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw=="], + "loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], "lowlight": ["lowlight@1.20.0", "", { "dependencies": { "fault": "^1.0.0", "highlight.js": "~10.7.0" } }, "sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw=="], @@ -3887,6 +3899,8 @@ "makeerror": ["makeerror@1.0.12", "", { "dependencies": { "tmpl": "1.0.5" } }, "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg=="], + "mammoth": ["mammoth@1.11.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.6", "argparse": "~1.0.3", "base64-js": "^1.5.1", "bluebird": "~3.4.0", "dingbat-to-unicode": "^1.0.1", "jszip": "^3.7.1", "lop": "^0.4.2", "path-is-absolute": "^1.0.0", "underscore": "^1.13.1", "xmlbuilder": "^10.0.0" }, "bin": { "mammoth": "bin/mammoth" } }, "sha512-BcEqqY/BOwIcI1iR5tqyVlqc3KIaMRa4egSoK83YAVrBf6+yqdAAbtUcFDCWX8Zef8/fgNZ6rl4VUv+vVX8ddQ=="], + "markdown-it": ["markdown-it@14.1.0", "", { "dependencies": { "argparse": "^2.0.1", "entities": "^4.4.0", "linkify-it": "^5.0.0", "mdurl": "^2.0.0", "punycode.js": "^2.3.1", "uc.micro": "^2.1.0" }, "bin": { "markdown-it": "bin/markdown-it.mjs" } }, "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg=="], "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], @@ -4171,6 +4185,8 @@ "open": ["open@10.2.0", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "wsl-utils": "^0.1.0" } }, "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA=="], + "option": ["option@0.2.4", "", {}, "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A=="], + "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], "ora": ["ora@8.2.0", "", { "dependencies": { "chalk": "^5.3.0", "cli-cursor": "^5.0.0", "cli-spinners": "^2.9.2", "is-interactive": "^2.0.0", "is-unicode-supported": "^2.0.0", "log-symbols": "^6.0.0", "stdin-discarder": "^0.2.2", "string-width": "^7.2.0", "strip-ansi": "^7.1.0" } }, "sha512-weP+BZ8MVNnlCm8c0Qdc1WSWq4Qn7I+9CJGm7Qali6g44e/PUzbjNqJX5NJ9ljlNMosfJvg1fKEGILklK9cwnw=="], @@ -5105,6 +5121,8 @@ "uncrypto": ["uncrypto@0.1.3", "", {}, "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q=="], + "underscore": ["underscore@1.13.7", "", {}, "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g=="], + "undici": ["undici@7.16.0", "", {}, "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g=="], "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], @@ -5287,7 +5305,7 @@ "xml2js": ["xml2js@0.6.2", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA=="], - "xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], + "xmlbuilder": ["xmlbuilder@10.1.1", "", {}, "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg=="], "xmlchars": ["xmlchars@2.2.0", "", {}, "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="], @@ -6481,6 +6499,8 @@ "wrap-ansi-cjs/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + "xml2js/xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], + "yauzl/buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], "@angular-devkit/core/ajv/json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], @@ -6501,6 +6521,8 @@ "@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="], + "@azure/core-http/xml2js/xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], + "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], "@browserbasehq/sdk/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], diff --git a/packages/db/prisma/migrations/20251118160603_add_supplement_documents/migration.sql b/packages/db/prisma/migrations/20251118160603_add_supplement_documents/migration.sql new file mode 100644 index 000000000..752b440db --- /dev/null +++ b/packages/db/prisma/migrations/20251118160603_add_supplement_documents/migration.sql @@ -0,0 +1,31 @@ +-- CreateEnum +CREATE TYPE "KnowledgeBaseDocumentProcessingStatus" AS ENUM ('pending', 'processing', 'completed', 'failed'); + +-- CreateTable +CREATE TABLE "KnowledgeBaseDocument" ( + "id" TEXT NOT NULL DEFAULT generate_prefixed_cuid('kbd'::text), + "name" TEXT NOT NULL, + "description" TEXT, + "s3Key" TEXT NOT NULL, + "fileType" TEXT NOT NULL, + "fileSize" INTEGER NOT NULL, + "processingStatus" "KnowledgeBaseDocumentProcessingStatus" NOT NULL DEFAULT 'pending', + "processedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "organizationId" TEXT NOT NULL, + + CONSTRAINT "KnowledgeBaseDocument_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "KnowledgeBaseDocument_organizationId_idx" ON "KnowledgeBaseDocument"("organizationId"); + +-- CreateIndex +CREATE INDEX "KnowledgeBaseDocument_organizationId_processingStatus_idx" ON "KnowledgeBaseDocument"("organizationId", "processingStatus"); + +-- CreateIndex +CREATE INDEX "KnowledgeBaseDocument_s3Key_idx" ON "KnowledgeBaseDocument"("s3Key"); + +-- AddForeignKey +ALTER TABLE "KnowledgeBaseDocument" ADD CONSTRAINT "KnowledgeBaseDocument_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20251118171710_add_trigger_run_id_to_knowledge_base_document/migration.sql b/packages/db/prisma/migrations/20251118171710_add_trigger_run_id_to_knowledge_base_document/migration.sql new file mode 100644 index 000000000..9384e0260 --- /dev/null +++ b/packages/db/prisma/migrations/20251118171710_add_trigger_run_id_to_knowledge_base_document/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "KnowledgeBaseDocument" ADD COLUMN "triggerRunId" TEXT; + +-- CreateIndex +CREATE INDEX "KnowledgeBaseDocument_triggerRunId_idx" ON "KnowledgeBaseDocument"("triggerRunId"); diff --git a/packages/db/prisma/migrations/20251118183908_add_questionnaire_tables/migration.sql b/packages/db/prisma/migrations/20251118183908_add_questionnaire_tables/migration.sql new file mode 100644 index 000000000..01003c3b3 --- /dev/null +++ b/packages/db/prisma/migrations/20251118183908_add_questionnaire_tables/migration.sql @@ -0,0 +1,64 @@ +-- CreateEnum +CREATE TYPE "QuestionnaireStatus" AS ENUM ('parsing', 'completed', 'failed'); + +-- CreateEnum +CREATE TYPE "QuestionnaireAnswerStatus" AS ENUM ('untouched', 'generated', 'manual'); + +-- CreateTable +CREATE TABLE "Questionnaire" ( + "id" TEXT NOT NULL DEFAULT generate_prefixed_cuid('qst'::text), + "filename" TEXT NOT NULL, + "s3Key" TEXT NOT NULL, + "fileType" TEXT NOT NULL, + "fileSize" INTEGER NOT NULL, + "status" "QuestionnaireStatus" NOT NULL DEFAULT 'parsing', + "parsedAt" TIMESTAMP(3), + "totalQuestions" INTEGER NOT NULL DEFAULT 0, + "answeredQuestions" INTEGER NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "organizationId" TEXT NOT NULL, + + CONSTRAINT "Questionnaire_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "QuestionnaireQuestionAnswer" ( + "id" TEXT NOT NULL DEFAULT generate_prefixed_cuid('qqa'::text), + "question" TEXT NOT NULL, + "answer" TEXT, + "status" "QuestionnaireAnswerStatus" NOT NULL DEFAULT 'untouched', + "questionIndex" INTEGER NOT NULL, + "sources" JSONB, + "generatedAt" TIMESTAMP(3), + "updatedBy" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "questionnaireId" TEXT NOT NULL, + + CONSTRAINT "QuestionnaireQuestionAnswer_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "Questionnaire_organizationId_idx" ON "Questionnaire"("organizationId"); + +-- CreateIndex +CREATE INDEX "Questionnaire_organizationId_createdAt_idx" ON "Questionnaire"("organizationId", "createdAt"); + +-- CreateIndex +CREATE INDEX "Questionnaire_status_idx" ON "Questionnaire"("status"); + +-- CreateIndex +CREATE INDEX "QuestionnaireQuestionAnswer_questionnaireId_idx" ON "QuestionnaireQuestionAnswer"("questionnaireId"); + +-- CreateIndex +CREATE INDEX "QuestionnaireQuestionAnswer_questionnaireId_questionIndex_idx" ON "QuestionnaireQuestionAnswer"("questionnaireId", "questionIndex"); + +-- CreateIndex +CREATE INDEX "QuestionnaireQuestionAnswer_status_idx" ON "QuestionnaireQuestionAnswer"("status"); + +-- AddForeignKey +ALTER TABLE "Questionnaire" ADD CONSTRAINT "Questionnaire_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "QuestionnaireQuestionAnswer" ADD CONSTRAINT "QuestionnaireQuestionAnswer_questionnaireId_fkey" FOREIGN KEY ("questionnaireId") REFERENCES "Questionnaire"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20251118220527_add_security_questionnaire_manual_answers/migration.sql b/packages/db/prisma/migrations/20251118220527_add_security_questionnaire_manual_answers/migration.sql new file mode 100644 index 000000000..3d9b236be --- /dev/null +++ b/packages/db/prisma/migrations/20251118220527_add_security_questionnaire_manual_answers/migration.sql @@ -0,0 +1,36 @@ +-- CreateTable +CREATE TABLE "SecurityQuestionnaireManualAnswer" ( + "id" TEXT NOT NULL DEFAULT generate_prefixed_cuid('sqma'::text), + "question" TEXT NOT NULL, + "answer" TEXT NOT NULL, + "tags" TEXT[] DEFAULT ARRAY[]::TEXT[], + "sourceQuestionnaireId" TEXT, + "createdBy" TEXT, + "updatedBy" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "organizationId" TEXT NOT NULL, + + CONSTRAINT "SecurityQuestionnaireManualAnswer_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "SecurityQuestionnaireManualAnswer_organizationId_idx" ON "SecurityQuestionnaireManualAnswer"("organizationId"); + +-- CreateIndex +CREATE INDEX "SecurityQuestionnaireManualAnswer_organizationId_question_idx" ON "SecurityQuestionnaireManualAnswer"("organizationId", "question"); + +-- CreateIndex +CREATE INDEX "SecurityQuestionnaireManualAnswer_tags_idx" ON "SecurityQuestionnaireManualAnswer"("tags"); + +-- CreateIndex +CREATE INDEX "SecurityQuestionnaireManualAnswer_createdAt_idx" ON "SecurityQuestionnaireManualAnswer"("createdAt"); + +-- CreateUniqueIndex +CREATE UNIQUE INDEX "SecurityQuestionnaireManualAnswer_organizationId_question_key" ON "SecurityQuestionnaireManualAnswer"("organizationId", "question"); + +-- AddForeignKey +ALTER TABLE "SecurityQuestionnaireManualAnswer" ADD CONSTRAINT "SecurityQuestionnaireManualAnswer_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SecurityQuestionnaireManualAnswer" ADD CONSTRAINT "SecurityQuestionnaireManualAnswer_sourceQuestionnaireId_fkey" FOREIGN KEY ("sourceQuestionnaireId") REFERENCES "Questionnaire"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema/knowledge-base-document.prisma b/packages/db/prisma/schema/knowledge-base-document.prisma new file mode 100644 index 000000000..9c2aca6d2 --- /dev/null +++ b/packages/db/prisma/schema/knowledge-base-document.prisma @@ -0,0 +1,32 @@ +model KnowledgeBaseDocument { + id String @id @default(dbgenerated("generate_prefixed_cuid('kbd'::text)")) + name String // Original filename + description String? // Optional user description/notes + s3Key String // S3 storage key (e.g., "org123/knowledge-base-documents/timestamp-file.pdf") + fileType String // MIME type (e.g., "application/pdf") + fileSize Int // File size in bytes + processingStatus KnowledgeBaseDocumentProcessingStatus @default(pending) // Track indexing status + processedAt DateTime? // When indexing completed + triggerRunId String? // Trigger.dev run ID for tracking processing progress + + // Dates + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // Relationships + organizationId String + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + + @@index([organizationId]) + @@index([organizationId, processingStatus]) + @@index([s3Key]) + @@index([triggerRunId]) +} + +enum KnowledgeBaseDocumentProcessingStatus { + pending // Uploaded but not yet processed/indexed + processing // Currently being processed/indexed + completed // Successfully indexed in vector database + failed // Processing failed +} + diff --git a/packages/db/prisma/schema/organization.prisma b/packages/db/prisma/schema/organization.prisma index a4abdfb04..a81ca74d8 100644 --- a/packages/db/prisma/schema/organization.prisma +++ b/packages/db/prisma/schema/organization.prisma @@ -34,6 +34,9 @@ model Organization { trustAccessRequests TrustAccessRequest[] trustNdaAgreements TrustNDAAgreement[] trustDocuments TrustDocument[] + knowledgeBaseDocuments KnowledgeBaseDocument[] + questionnaires Questionnaire[] + securityQuestionnaireManualAnswers SecurityQuestionnaireManualAnswer[] @@index([slug]) } diff --git a/packages/db/prisma/schema/questionnaire.prisma b/packages/db/prisma/schema/questionnaire.prisma new file mode 100644 index 000000000..2f0811334 --- /dev/null +++ b/packages/db/prisma/schema/questionnaire.prisma @@ -0,0 +1,61 @@ +model Questionnaire { + id String @id @default(dbgenerated("generate_prefixed_cuid('qst'::text)")) + filename String // Original filename + s3Key String // S3 storage key for the uploaded file + fileType String // MIME type (e.g., "application/pdf") + fileSize Int // File size in bytes + status QuestionnaireStatus @default(parsing) // Parsing status + parsedAt DateTime? // When parsing completed + totalQuestions Int @default(0) // Total number of questions parsed + answeredQuestions Int @default(0) // Number of questions with answers + + // Dates + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // Relationships + organizationId String + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + questions QuestionnaireQuestionAnswer[] + manualAnswers SecurityQuestionnaireManualAnswer[] // Manual answers saved from this questionnaire + + @@index([organizationId]) + @@index([organizationId, createdAt]) + @@index([status]) +} + +model QuestionnaireQuestionAnswer { + id String @id @default(dbgenerated("generate_prefixed_cuid('qqa'::text)")) + question String // The question text + answer String? // The answer (nullable if not provided in file or not generated yet) + status QuestionnaireAnswerStatus @default(untouched) // Answer status + questionIndex Int // Order/index of the question in the questionnaire + sources Json? // Sources used for generated answers (array of source objects) + generatedAt DateTime? // When answer was generated (if status is generated) + updatedBy String? // User ID who last updated the answer (if manual) + + // Dates + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // Relationships + questionnaireId String + questionnaire Questionnaire @relation(fields: [questionnaireId], references: [id], onDelete: Cascade) + + @@index([questionnaireId]) + @@index([questionnaireId, questionIndex]) + @@index([status]) +} + +enum QuestionnaireStatus { + parsing // Currently being parsed + completed // Successfully parsed + failed // Parsing failed +} + +enum QuestionnaireAnswerStatus { + untouched // No answer yet (empty or not generated) + generated // AI generated answer + manual // Manually written/edited by user +} + diff --git a/packages/db/prisma/schema/security-questionnaire-manual-answer.prisma b/packages/db/prisma/schema/security-questionnaire-manual-answer.prisma new file mode 100644 index 000000000..c41ad7394 --- /dev/null +++ b/packages/db/prisma/schema/security-questionnaire-manual-answer.prisma @@ -0,0 +1,29 @@ +model SecurityQuestionnaireManualAnswer { + id String @id @default(dbgenerated("generate_prefixed_cuid('sqma'::text)")) + question String // The question text + answer String // The answer text (required for saved answers) + tags String[] @default([]) // Optional tags for categorization + + // Optional reference to original questionnaire (for tracking) + sourceQuestionnaireId String? + sourceQuestionnaire Questionnaire? @relation(fields: [sourceQuestionnaireId], references: [id], onDelete: SetNull) + + // User who created/updated this answer + createdBy String? // User ID + updatedBy String? // User ID + + // Dates + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // Relationships + organizationId String + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + + @@index([organizationId]) + @@index([organizationId, question]) + @@index([tags]) + @@index([createdAt]) + @@unique([organizationId, question]) // Prevent duplicate questions per organization +} + From dd6284725884b37fe4117c975f4ce22bba0a994b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:17:57 -0500 Subject: [PATCH 2/7] CS-54 [New Framework] - ISO 9001 (#1810) * feat(db): add iso9001 column to trust table * feat(app): add new framework: ISO 9001 * feat(db): create new db version --------- Co-authored-by: chasprowebdev --- apps/app/public/badges/iso9001.svg | 21 +++ .../components/FrameworksOverview.tsx | 4 + .../actions/update-trust-portal-frameworks.ts | 6 + .../components/TrustPortalSwitch.tsx | 53 +++++- .../trust-portal/components/logos.tsx | 165 ++++++++++++------ .../[orgId]/settings/trust-portal/page.tsx | 9 +- packages/db/package.json | 2 +- .../migration.sql | 4 + packages/db/prisma/schema/trust.prisma | 2 + 9 files changed, 210 insertions(+), 56 deletions(-) create mode 100644 apps/app/public/badges/iso9001.svg create mode 100644 packages/db/prisma/migrations/20251120210257_add_iso9001_fields/migration.sql diff --git a/apps/app/public/badges/iso9001.svg b/apps/app/public/badges/iso9001.svg new file mode 100644 index 000000000..64c9b6e7d --- /dev/null +++ b/apps/app/public/badges/iso9001.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/app/src/app/(app)/[orgId]/frameworks/components/FrameworksOverview.tsx b/apps/app/src/app/(app)/[orgId]/frameworks/components/FrameworksOverview.tsx index db848d64e..e7782c4c2 100644 --- a/apps/app/src/app/(app)/[orgId]/frameworks/components/FrameworksOverview.tsx +++ b/apps/app/src/app/(app)/[orgId]/frameworks/components/FrameworksOverview.tsx @@ -48,6 +48,10 @@ export function mapFrameworkToBadge(framework: FrameworkInstanceWithControls) { return '/badges/nen7510.svg'; } + if (framework.framework.name === 'ISO 9001') { + return '/badges/iso9001.svg'; + } + return null; } diff --git a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/actions/update-trust-portal-frameworks.ts b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/actions/update-trust-portal-frameworks.ts index 03b9193fb..dd705077d 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/actions/update-trust-portal-frameworks.ts +++ b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/actions/update-trust-portal-frameworks.ts @@ -15,6 +15,7 @@ interface UpdateTrustPortalFrameworksParams { hipaa?: boolean; pcidss?: boolean; nen7510?: boolean; + iso9001?: boolean; soc2type1Status?: 'started' | 'in_progress' | 'compliant'; soc2type2Status?: 'started' | 'in_progress' | 'compliant'; iso27001Status?: 'started' | 'in_progress' | 'compliant'; @@ -23,6 +24,7 @@ interface UpdateTrustPortalFrameworksParams { hipaaStatus?: 'started' | 'in_progress' | 'compliant'; pcidssStatus?: 'started' | 'in_progress' | 'compliant'; nen7510Status?: 'started' | 'in_progress' | 'compliant'; + iso9001Status?: 'started' | 'in_progress' | 'compliant'; } export async function updateTrustPortalFrameworks({ @@ -35,6 +37,8 @@ export async function updateTrustPortalFrameworks({ hipaa, pcidss, nen7510, + iso9001, + iso9001Status, soc2type1Status, soc2type2Status, iso27001Status, @@ -85,6 +89,8 @@ export async function updateTrustPortalFrameworks({ hipaa_status: hipaaStatus ?? trustPortal.hipaa_status, pci_dss_status: pcidssStatus ?? trustPortal.pci_dss_status, nen7510_status: nen7510Status ?? trustPortal.nen7510_status, + iso9001: iso9001 ?? trustPortal.iso9001, + iso9001_status: iso9001Status ?? trustPortal.iso9001_status, }, }); diff --git a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/TrustPortalSwitch.tsx b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/TrustPortalSwitch.tsx index 1a6d4c8cb..e90cb32c6 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/TrustPortalSwitch.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/TrustPortalSwitch.tsx @@ -17,7 +17,17 @@ import { z } from 'zod'; import { isFriendlyAvailable } from '../actions/is-friendly-available'; import { trustPortalSwitchAction } from '../actions/trust-portal-switch'; import { updateTrustPortalFrameworks } from '../actions/update-trust-portal-frameworks'; -import { GDPR, HIPAA, ISO27001, SOC2Type1, SOC2Type2, PCIDSS, ISO42001, NEN7510 } from './logos'; +import { + GDPR, + HIPAA, + ISO27001, + ISO42001, + ISO9001, + NEN7510, + PCIDSS, + SOC2Type1, + SOC2Type2, +} from './logos'; const trustPortalSwitchSchema = z.object({ enabled: z.boolean(), @@ -31,6 +41,7 @@ const trustPortalSwitchSchema = z.object({ hipaa: z.boolean(), pcidss: z.boolean(), nen7510: z.boolean(), + iso9001: z.boolean(), soc2type1Status: z.enum(['started', 'in_progress', 'compliant']), soc2type2Status: z.enum(['started', 'in_progress', 'compliant']), iso27001Status: z.enum(['started', 'in_progress', 'compliant']), @@ -39,6 +50,7 @@ const trustPortalSwitchSchema = z.object({ hipaaStatus: z.enum(['started', 'in_progress', 'compliant']), pcidssStatus: z.enum(['started', 'in_progress', 'compliant']), nen7510Status: z.enum(['started', 'in_progress', 'compliant']), + iso9001Status: z.enum(['started', 'in_progress', 'compliant']), }); export function TrustPortalSwitch({ @@ -64,6 +76,8 @@ export function TrustPortalSwitch({ pcidssStatus, nen7510, nen7510Status, + iso9001, + iso9001Status, friendlyUrl, }: { enabled: boolean; @@ -88,6 +102,8 @@ export function TrustPortalSwitch({ hipaaStatus: 'started' | 'in_progress' | 'compliant'; pcidssStatus: 'started' | 'in_progress' | 'compliant'; nen7510Status: 'started' | 'in_progress' | 'compliant'; + iso9001: boolean; + iso9001Status: 'started' | 'in_progress' | 'compliant'; friendlyUrl: string | null; }) { const trustPortalSwitch = useAction(trustPortalSwitchAction, { @@ -114,6 +130,7 @@ export function TrustPortalSwitch({ hipaa: hipaa ?? false, pcidss: pcidss ?? false, nen7510: nen7510 ?? false, + iso9001: iso9001 ?? false, soc2type1Status: soc2type1Status ?? 'started', soc2type2Status: soc2type2Status ?? 'started', iso27001Status: iso27001Status ?? 'started', @@ -122,6 +139,7 @@ export function TrustPortalSwitch({ hipaaStatus: hipaaStatus ?? 'started', pcidssStatus: pcidssStatus ?? 'started', nen7510Status: nen7510Status ?? 'started', + iso9001Status: iso9001Status ?? 'started', friendlyUrl: friendlyUrl ?? undefined, }, }); @@ -578,6 +596,35 @@ export function TrustPortalSwitch({ } }} /> + {/* ISO 9001 */} + { + try { + await updateTrustPortalFrameworks({ + orgId, + iso9001Status: value as 'started' | 'in_progress' | 'compliant', + }); + toast.success('ISO 9001 status updated'); + } catch (error) { + toast.error('Failed to update ISO 9001 status'); + } + }} + onToggle={async (checked) => { + try { + await updateTrustPortalFrameworks({ + orgId, + iso9001: checked, + }); + toast.success('ISO 9001 status updated'); + } catch (error) { + toast.error('Failed to update ISO 9001 status'); + } + }} + />
@@ -638,6 +685,10 @@ function ComplianceFramework({
+ ) : title === 'ISO 9001' ? ( +
+ +
) : null; return ( diff --git a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/logos.tsx b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/logos.tsx index 3e5999279..98089e954 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/logos.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/components/logos.tsx @@ -179,58 +179,58 @@ export const HIPAA = (props: React.SVGProps) => ( export const SOC2Type1 = (props: React.SVGProps) => ( - - - - - - - - - - - + + + + + + + + + + + - + @@ -290,7 +290,7 @@ export const SOC2Type2 = (props: React.SVGProps) => ( - + @@ -330,7 +330,7 @@ export const PCIDSS = (props: React.SVGProps) => ( d="M41.6025 85.6312C42.7148 86.1791 44.0184 86.1801 45.1316 85.634L75.3157 70.8264C76.0703 70.4562 76.9448 71.0342 76.8998 71.8736V71.8736C76.879 72.2619 76.6549 72.6104 76.3102 72.7904L44.2924 89.5162C43.7122 89.8193 43.0205 89.8193 42.4403 89.5163L10.4728 72.8169C10.1 72.6221 9.86621 72.2363 9.86621 71.8156V71.8156C9.86621 70.9788 10.7443 70.4325 11.495 70.8022L41.6025 85.6312Z" fill="#004F3B" /> - + ) => ( d="M41.6025 85.6312C42.7148 86.1791 44.0184 86.1801 45.1316 85.634L75.3157 70.8264C76.0703 70.4562 76.9448 71.0342 76.8998 71.8736C76.879 72.2619 76.6549 72.6104 76.3102 72.7904L44.2924 89.5162C43.7122 89.8193 43.0205 89.8193 42.4403 89.5163L10.4728 72.8169C10.1 72.6221 9.86621 72.2363 9.86621 71.8156C9.86621 70.9788 10.7443 70.4325 11.495 70.8022L41.6025 85.6312Z" fill="#004F3B" /> - + ) => ( /> - + ); + +export const ISO9001 = (props: React.SVGProps) => ( + + + + + + + + + + + + + + +); diff --git a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/page.tsx b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/page.tsx index d2cd6f276..e86bb91f5 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/trust-portal/page.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/trust-portal/page.tsx @@ -31,6 +31,7 @@ export default async function TrustPortalSettings({ hipaa={trustPortal?.hipaa ?? false} pcidss={trustPortal?.pcidss ?? false} nen7510={trustPortal?.nen7510 ?? false} + iso9001={trustPortal?.iso9001 ?? false} soc2type1Status={trustPortal?.soc2type1Status ?? 'started'} soc2type2Status={trustPortal?.soc2type2Status ?? 'started'} iso27001Status={trustPortal?.iso27001Status ?? 'started'} @@ -39,6 +40,7 @@ export default async function TrustPortalSettings({ hipaaStatus={trustPortal?.hipaaStatus ?? 'started'} pcidssStatus={trustPortal?.pcidssStatus ?? 'started'} nen7510Status={trustPortal?.nen7510Status ?? 'started'} + iso9001Status={trustPortal?.iso9001Status ?? 'started'} friendlyUrl={trustPortal?.friendlyUrl ?? null} /> { pcidss: trustPortal?.pci_dss, nen7510: trustPortal?.nen7510, soc2type1Status: trustPortal?.soc2type1_status, - soc2type2Status: !trustPortal?.soc2type2 && trustPortal?.soc2 ? trustPortal?.soc2_status : trustPortal?.soc2type2_status, + soc2type2Status: + !trustPortal?.soc2type2 && trustPortal?.soc2 + ? trustPortal?.soc2_status + : trustPortal?.soc2type2_status, iso27001Status: trustPortal?.iso27001_status, iso42001Status: trustPortal?.iso42001_status, gdprStatus: trustPortal?.gdpr_status, hipaaStatus: trustPortal?.hipaa_status, pcidssStatus: trustPortal?.pci_dss_status, nen7510Status: trustPortal?.nen7510_status, + iso9001: trustPortal?.iso9001, + iso9001Status: trustPortal?.iso9001_status, isVercelDomain: trustPortal?.isVercelDomain, vercelVerification: trustPortal?.vercelVerification, friendlyUrl: trustPortal?.friendlyUrl, diff --git a/packages/db/package.json b/packages/db/package.json index 0bd086bf4..bcef9db8d 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,7 +1,7 @@ { "name": "@trycompai/db", "description": "Database package with Prisma client and schema for Comp AI", - "version": "1.3.17", + "version": "1.3.18", "dependencies": { "@prisma/client": "^6.13.0", "dotenv": "^16.4.5", diff --git a/packages/db/prisma/migrations/20251120210257_add_iso9001_fields/migration.sql b/packages/db/prisma/migrations/20251120210257_add_iso9001_fields/migration.sql new file mode 100644 index 000000000..e908a1aec --- /dev/null +++ b/packages/db/prisma/migrations/20251120210257_add_iso9001_fields/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "public"."Trust" ADD COLUMN "iso9001" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "iso9001_status" "public"."FrameworkStatus" NOT NULL DEFAULT 'started'; + diff --git a/packages/db/prisma/schema/trust.prisma b/packages/db/prisma/schema/trust.prisma index f4faf442f..e4ef133a0 100644 --- a/packages/db/prisma/schema/trust.prisma +++ b/packages/db/prisma/schema/trust.prisma @@ -20,6 +20,7 @@ model Trust { gdpr Boolean @default(false) hipaa Boolean @default(false) pci_dss Boolean @default(false) + iso9001 Boolean @default(false) soc2_status FrameworkStatus @default(started) soc2type1_status FrameworkStatus @default(started) @@ -30,6 +31,7 @@ model Trust { gdpr_status FrameworkStatus @default(started) hipaa_status FrameworkStatus @default(started) pci_dss_status FrameworkStatus @default(started) + iso9001_status FrameworkStatus @default(started) @@id([status, organizationId]) @@unique([organizationId]) From 388e8fa9990a0ee71da26e85d9cb2b97f7d2c4b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:30:38 -0500 Subject: [PATCH 3/7] refactor(trust): restructure Trust Access Management page layout (#1812) Co-authored-by: Daniel Fu Co-authored-by: Mariano Fuentes --- apps/app/src/app/(app)/[orgId]/trust/page.tsx | 21 ++++++++++--------- bunfig.toml | 2 ++ 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 bunfig.toml diff --git a/apps/app/src/app/(app)/[orgId]/trust/page.tsx b/apps/app/src/app/(app)/[orgId]/trust/page.tsx index ecdde9d33..967e3e262 100644 --- a/apps/app/src/app/(app)/[orgId]/trust/page.tsx +++ b/apps/app/src/app/(app)/[orgId]/trust/page.tsx @@ -4,18 +4,19 @@ import { TrustAccessRequestsClient } from './components/trust-access-request-cli export default async function TrustAccessPage({ params }: { params: Promise<{ orgId: string }> }) { const { orgId } = await params; + return ( - -
-
-

Trust Access Management

-

- Manage data access requests and grants -

+
+ +
+
+

Trust Access Management

+

Manage data access requests and grants

+
+
- -
- + +
); } diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 000000000..57f75baf9 --- /dev/null +++ b/bunfig.toml @@ -0,0 +1,2 @@ +[install] +linker = "hoisted" From fa9796212905aa60b533d63f0fe82f6b5a9dc4de Mon Sep 17 00:00:00 2001 From: Min Chun Fu <70210356+Itsnotaka@users.noreply.github.com> Date: Sat, 22 Nov 2025 00:10:14 +0900 Subject: [PATCH 4/7] feat(trust): add friendlyUrl parameter for access request methods (#1813) --- .../trust-portal/trust-access.controller.ts | 11 +++++ .../src/trust-portal/trust-access.service.ts | 47 +++++++++++-------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/apps/api/src/trust-portal/trust-access.controller.ts b/apps/api/src/trust-portal/trust-access.controller.ts index 323456f55..895e3f076 100644 --- a/apps/api/src/trust-portal/trust-access.controller.ts +++ b/apps/api/src/trust-portal/trust-access.controller.ts @@ -14,6 +14,7 @@ import { import { ApiHeader, ApiOperation, + ApiParam, ApiResponse, ApiSecurity, ApiTags, @@ -44,11 +45,16 @@ export class TrustAccessController { description: 'External users submit request for data access from trust site', }) + @ApiParam({ + name: 'friendlyUrl', + description: 'Trust Portal friendly URL or Organization ID', + }) @ApiResponse({ status: HttpStatus.CREATED, description: 'Access request created and sent for review', }) async createAccessRequest( + // Note: friendlyUrl can be either the custom friendly URL or the organization ID @Param('friendlyUrl') friendlyUrl: string, @Body() dto: CreateAccessRequestDto, @Req() req: Request, @@ -365,11 +371,16 @@ export class TrustAccessController { description: 'Generate access link for users with existing grants to redownload data', }) + @ApiParam({ + name: 'friendlyUrl', + description: 'Trust Portal friendly URL or Organization ID', + }) @ApiResponse({ status: HttpStatus.OK, description: 'Access link sent to email', }) async reclaimAccess( + // Note: friendlyUrl can be either the custom friendly URL or the organization ID @Param('friendlyUrl') friendlyUrl: string, @Body() dto: ReclaimAccessDto, ) { diff --git a/apps/api/src/trust-portal/trust-access.service.ts b/apps/api/src/trust-portal/trust-access.service.ts index f95b883d9..2ad19ff23 100644 --- a/apps/api/src/trust-portal/trust-access.service.ts +++ b/apps/api/src/trust-portal/trust-access.service.ts @@ -28,6 +28,28 @@ export class TrustAccessService { return randomBytes(length).toString('base64url').slice(0, length); } + private async findPublishedTrustByRouteId(id: string) { + // First, try treating `id` as the existing friendlyUrl. + let trust = await db.trust.findUnique({ + where: { friendlyUrl: id }, + include: { organization: true }, + }); + + // If none found, fall back to treating `id` as organizationId. + if (!trust) { + trust = await db.trust.findFirst({ + where: { organizationId: id }, + include: { organization: true }, + }); + } + + if (!trust || trust.status !== 'published') { + throw new NotFoundException('Trust site not found or not published'); + } + + return trust; + } + constructor( private readonly ndaPdfService: NdaPdfService, private readonly emailService: TrustEmailService, @@ -60,19 +82,12 @@ export class TrustAccessService { } async createAccessRequest( - friendlyUrl: string, + id: string, dto: CreateAccessRequestDto, ipAddress: string | undefined, userAgent: string | undefined, ) { - const trust = await db.trust.findUnique({ - where: { friendlyUrl }, - include: { organization: true }, - }); - - if (!trust || trust.status !== 'published') { - throw new NotFoundException('Trust site not found or not published'); - } + const trust = await this.findPublishedTrustByRouteId(id); // Check if the email already has an active grant const existingGrant = await db.trustAccessGrant.findFirst({ @@ -791,15 +806,8 @@ export class TrustAccessService { }; } - async reclaimAccess(friendlyUrl: string, email: string) { - const trust = await db.trust.findUnique({ - where: { friendlyUrl }, - include: { organization: true }, - }); - - if (!trust || trust.status !== 'published') { - throw new NotFoundException('Trust site not found or not published'); - } + async reclaimAccess(id: string, email: string) { + const trust = await this.findPublishedTrustByRouteId(id); const grant = await db.trustAccessGrant.findFirst({ where: { @@ -849,7 +857,8 @@ export class TrustAccessService { }); } - const accessLink = `${this.TRUST_APP_URL}/${friendlyUrl}/access/${accessToken}`; + const urlId = trust.friendlyUrl || trust.organizationId; + const accessLink = `${this.TRUST_APP_URL}/${urlId}/access/${accessToken}`; await this.emailService.sendAccessReclaimEmail({ toEmail: email, From 02dfd71c17b312ae63fffca2869eba9d9d36387b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Nov 2025 11:53:11 -0500 Subject: [PATCH 5/7] [dev] [Itsnotaka] daniel/table-ui (#1814) * feat(grants): add search and filter functionality to grants tab * feat(trust): enhance NDA response with portal URL and status messages * feat(trust): enhance NDA response with portal URL and status messages --------- Co-authored-by: Daniel Fu --- .../src/trust-portal/trust-access.service.ts | 56 ++++-- .../trust/components/grant-columns.tsx | 112 +++++++++++ .../trust/components/grant-data-table.tsx | 25 +++ .../[orgId]/trust/components/grants-tab.tsx | 124 ++++-------- .../trust/components/request-columns.tsx | 160 ++++++++++++++++ .../trust/components/request-data-table.tsx | 40 ++++ .../[orgId]/trust/components/request-tab.tsx | 180 +++++------------- .../components/ui/data-table/DataTable.tsx | 2 +- .../ui/data-table/DataTableHeader.tsx | 2 +- bun.lock | 2 +- packages/docs/openapi.json | 2 + 11 files changed, 477 insertions(+), 228 deletions(-) create mode 100644 apps/app/src/app/(app)/[orgId]/trust/components/grant-columns.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx diff --git a/apps/api/src/trust-portal/trust-access.service.ts b/apps/api/src/trust-portal/trust-access.service.ts index 2ad19ff23..930bede9b 100644 --- a/apps/api/src/trust-portal/trust-access.service.ts +++ b/apps/api/src/trust-portal/trust-access.service.ts @@ -485,6 +485,7 @@ export class TrustAccessService { organization: true, }, }, + grant: true, }, }); @@ -492,26 +493,59 @@ export class TrustAccessService { throw new NotFoundException('NDA agreement not found'); } + const trust = await db.trust.findUnique({ + where: { organizationId: nda.organizationId }, + select: { friendlyUrl: true }, + }); + + const portalUrl = trust?.friendlyUrl + ? `${this.TRUST_APP_URL}/${trust.friendlyUrl}` + : null; + + const baseResponse = { + id: nda.id, + organizationName: nda.accessRequest.organization.name, + requesterName: nda.accessRequest.name, + requesterEmail: nda.accessRequest.email, + expiresAt: nda.signTokenExpiresAt, + portalUrl, + }; + if (nda.signTokenExpiresAt < new Date()) { - throw new BadRequestException('NDA signing link has expired'); + return { + ...baseResponse, + status: 'expired', + message: 'NDA signing link has expired', + }; } if (nda.status === 'void') { - throw new BadRequestException( - 'This NDA has been revoked and is no longer valid', - ); + return { + ...baseResponse, + status: 'void', + message: 'This NDA has been revoked and is no longer valid', + }; } - if (nda.status !== 'pending') { - throw new BadRequestException('NDA has already been signed'); + if (nda.status === 'signed') { + let accessUrl = portalUrl; + if (nda.grant?.accessToken && nda.grant.status === 'active') { + if (trust?.friendlyUrl) { + accessUrl = `${this.TRUST_APP_URL}/${trust.friendlyUrl}/access/${nda.grant.accessToken}`; + } + } + + return { + ...baseResponse, + status: 'signed', + message: 'NDA has already been signed', + portalUrl: accessUrl, + }; } return { - id: nda.id, - organizationName: nda.accessRequest.organization.name, - requesterName: nda.accessRequest.name, - requesterEmail: nda.accessRequest.email, - expiresAt: nda.signTokenExpiresAt, + ...baseResponse, + status: 'pending', }; } diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/grant-columns.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/grant-columns.tsx new file mode 100644 index 000000000..492b40de8 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/trust/components/grant-columns.tsx @@ -0,0 +1,112 @@ +'use client'; + +import type { AccessGrant } from '@/hooks/use-access-requests'; +import { Badge } from '@comp/ui/badge'; +import { Button } from '@comp/ui/button'; +import type { ColumnDef } from '@tanstack/react-table'; +import { Copy } from 'lucide-react'; +import { toast } from 'sonner'; + +export type GrantTableRow = AccessGrant; + +interface GrantColumnHandlers { + onRevoke: (row: AccessGrant) => void; +} + +export function buildGrantColumns({ + onRevoke, +}: GrantColumnHandlers): ColumnDef[] { + return [ + { + id: 'date', + accessorKey: 'createdAt', + header: 'Date', + cell: ({ row }) => { + return ( + + {new Date(row.original.createdAt).toLocaleDateString()} + + ); + }, + }, + { + id: 'identity', + accessorKey: 'subjectEmail', + header: 'Identity', + cell: ({ row }) => { + return {row.original.subjectEmail}; + }, + }, + { + id: 'status', + accessorKey: 'status', + header: 'Status', + cell: ({ row }) => { + const status = row.original.status; + return ( + + {status} + + ); + }, + }, + { + id: 'expires', + accessorKey: 'expiresAt', + header: 'Expires', + cell: ({ row }) => { + return ( + + {new Date(row.original.expiresAt).toLocaleDateString()} + + ); + }, + }, + { + id: 'revokedAt', + accessorKey: 'revokedAt', + header: 'Revoked', + cell: ({ row }) => { + if (!row.original.revokedAt) { + return —; + } + return ( + + {new Date(row.original.revokedAt).toLocaleDateString()} + + ); + }, + }, + { + id: 'actions', + header: 'Actions', + cell: ({ row }) => { + const grant = row.original; + + if (grant.status === 'active') { + return ( + + ); + } + + return null; + }, + }, + ]; +} diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx new file mode 100644 index 000000000..0055d8303 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx @@ -0,0 +1,25 @@ +'use client'; + +import { DataTable } from '@/components/ui/data-table/DataTable'; +import type { AccessGrant } from '@/hooks/use-access-requests'; +import { buildGrantColumns, type GrantTableRow } from './grant-columns'; + +interface GrantDataTableProps { + data: AccessGrant[]; + isLoading?: boolean; + onRevoke: (row: AccessGrant) => void; +} + +export function GrantDataTable({ data, isLoading, onRevoke }: GrantDataTableProps) { + const columns = buildGrantColumns({ onRevoke }); + + return ( + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/grants-tab.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/grants-tab.tsx index 0e2384124..72005148e 100644 --- a/apps/app/src/app/(app)/[orgId]/trust/components/grants-tab.tsx +++ b/apps/app/src/app/(app)/[orgId]/trust/components/grants-tab.tsx @@ -1,97 +1,57 @@ import { useAccessGrants } from '@/hooks/use-access-requests'; -import { Badge } from '@comp/ui/badge'; -import { Button } from '@comp/ui/button'; -import { Skeleton } from '@comp/ui/skeleton'; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@comp/ui/table'; +import { Input } from '@comp/ui/input'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@comp/ui/select'; import { useState } from 'react'; +import { GrantDataTable } from './grant-data-table'; import { RevokeDialog } from './revoke-dialog'; export function GrantsTab({ orgId }: { orgId: string }) { const { data, isLoading } = useAccessGrants(orgId); const [revokeId, setRevokeId] = useState(null); + const [search, setSearch] = useState(''); + const [status, setStatus] = useState('all'); + + const filtered = (data ?? []).filter((grant) => { + const matchesSearch = + !search || grant.subjectEmail.toLowerCase().includes(search.toLowerCase()); + + const matchesStatus = status === 'all' || grant.status === status; + return matchesSearch && matchesStatus; + }); + return ( -
- - - - Email - Status - Expires - Revoked - Actions - - - - {isLoading - ? Array.from({ length: 5 }).map((_, index) => ( - - - - - - - - - - - - - - - - - - )) - : data && data.length > 0 - ? data.map((grant) => ( - - {grant.subjectEmail} - - - {grant.status} - - - {new Date(grant.expiresAt).toLocaleDateString()} - - {grant.revokedAt ? new Date(grant.revokedAt).toLocaleDateString() : '-'} - - - {grant.status === 'active' && ( - - )} - - - )) - : ( - - - No access grants yet - - - )} - -
+
+
+ setSearch(e.target.value)} + className="h-8 max-w-md" + /> + +
+ + setRevokeId(row.id)} + /> + {revokeId && ( setRevokeId(null)} /> )}
); } + diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx new file mode 100644 index 000000000..22a1bff6b --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx @@ -0,0 +1,160 @@ +'use client'; + +import type { AccessRequest } from '@/hooks/use-access-requests'; +import { Badge } from '@comp/ui/badge'; +import { Button } from '@comp/ui/button'; +import type { ColumnDef } from '@tanstack/react-table'; +import { Copy } from 'lucide-react'; +import { toast } from 'sonner'; + +export type RequestTableRow = AccessRequest; + +interface RequestColumnHandlers { + onApprove: (row: AccessRequest) => void; + onDeny: (row: AccessRequest) => void; + onResendNda: (row: AccessRequest) => void; + onPreviewNda: (row: AccessRequest) => void; +} + +export function buildRequestColumns({ + onApprove, + onDeny, + onResendNda, + onPreviewNda, +}: RequestColumnHandlers): ColumnDef[] { + return [ + { + id: 'date', + accessorKey: 'createdAt', + header: 'Date', + cell: ({ row }) => { + return ( + + {new Date(row.original.createdAt).toLocaleDateString()} + + ); + }, + }, + { + id: 'identity', + accessorKey: 'email', + header: 'Identity', + cell: ({ row }) => { + return ( +
+ {row.original.name} + {row.original.email} + {row.original.company && ( + {row.original.company} + )} +
+ ); + }, + }, + { + id: 'purpose', + accessorKey: 'purpose', + header: 'Purpose', + cell: ({ row }) => { + return ( +
+ {row.original.purpose || '—'} +
+ ); + }, + }, + { + id: 'duration', + accessorKey: 'requestedDurationDays', + header: 'Duration', + cell: ({ row }) => { + return {row.original.requestedDurationDays ?? 30}d; + }, + }, + { + id: 'status', + accessorKey: 'status', + header: 'Status', + cell: ({ row }) => { + const status = row.original.status; + return ( + + {status.replace('_', ' ')} + + ); + }, + }, + { + id: 'ndaCheck', + accessorKey: 'grant', + header: 'NDA Status', + cell: ({ row }) => { + const ndaPending = row.original.status === 'approved' && !row.original.grant; + + if (ndaPending) { + // @ts-expect-error - warning variant might be custom + return Pending; + } + + if (row.original.grant) { + return Signed; + } + + return —; + }, + }, + { + id: 'actions', + header: 'Actions', + cell: ({ row }) => { + const request = row.original; + const ndaPending = request.status === 'approved' && !request.grant; + + return ( +
+ {request.status === 'under_review' && ( + <> + + + + )} + + {ndaPending && ( + + )} + + +
+ ); + }, + }, + ]; +} diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx new file mode 100644 index 000000000..c8dfbef70 --- /dev/null +++ b/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx @@ -0,0 +1,40 @@ +'use client'; + +import { DataTable } from '@/components/ui/data-table/DataTable'; +import type { AccessRequest } from '@/hooks/use-access-requests'; +import { buildRequestColumns, type RequestTableRow } from './request-columns'; + +interface RequestDataTableProps { + data: AccessRequest[]; + isLoading?: boolean; + onApprove: (row: AccessRequest) => void; + onDeny: (row: AccessRequest) => void; + onResendNda: (row: AccessRequest) => void; + onPreviewNda: (row: AccessRequest) => void; +} + +export function RequestDataTable({ + data, + isLoading, + onApprove, + onDeny, + onResendNda, + onPreviewNda +}: RequestDataTableProps) { + const columns = buildRequestColumns({ + onApprove, + onDeny, + onResendNda, + onPreviewNda + }); + + return ( + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/request-tab.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/request-tab.tsx index 7ab59c057..8814fc3d2 100644 --- a/apps/app/src/app/(app)/[orgId]/trust/components/request-tab.tsx +++ b/apps/app/src/app/(app)/[orgId]/trust/components/request-tab.tsx @@ -1,12 +1,11 @@ import { useAccessRequests, usePreviewNda, useResendNda } from '@/hooks/use-access-requests'; -import { Badge } from '@comp/ui/badge'; -import { Button } from '@comp/ui/button'; -import { Skeleton } from '@comp/ui/skeleton'; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@comp/ui/table'; +import { Input } from '@comp/ui/input'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@comp/ui/select'; import { useState } from 'react'; import { toast } from 'sonner'; import { ApproveDialog } from './approve-dialog'; import { DenyDialog } from './deny-dialog'; +import { RequestDataTable } from './request-data-table'; export function RequestsTab({ orgId }: { orgId: string }) { const { data, isLoading } = useAccessRequests(orgId); @@ -15,6 +14,9 @@ export function RequestsTab({ orgId }: { orgId: string }) { const [approveId, setApproveId] = useState(null); const [denyId, setDenyId] = useState(null); + const [search, setSearch] = useState(''); + const [status, setStatus] = useState('all'); + const handleResendNda = (requestId: string) => { toast.promise(resendNda(requestId), { loading: 'Resending...', @@ -37,135 +39,48 @@ export function RequestsTab({ orgId }: { orgId: string }) { ); }; + const filtered = (data ?? []).filter((request) => { + const matchesSearch = + !search || + request.email.toLowerCase().includes(search.toLowerCase()) || + request.name.toLowerCase().includes(search.toLowerCase()) || + (request.company ?? '').toLowerCase().includes(search.toLowerCase()); + + const matchesStatus = status === 'all' || request.status === status; + return matchesSearch && matchesStatus; + }); + return ( -
- - - - Requested - Name - Email - Company - Purpose - Duration - Status - NDA - Actions - - - - {isLoading ? ( - Array.from({ length: 5 }).map((_, index) => ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - )) - ) : data && data.length > 0 ? ( - data.map((request) => { - const ndaPending = request.status === 'approved' && !request.grant; - return ( - - {new Date(request.createdAt).toLocaleDateString()} - {request.name} - {request.email} - {request.company || '-'} - {request.purpose || '-'} - {request.requestedDurationDays ?? 30}d - - - {request.status} - - - - {ndaPending ? ( - pending - ) : request.grant ? ( - signed - ) : ( - '-' - )} - - -
- - - {ndaPending && ( - - )} - -
- - - ); - }) - ) : ( - - - No access requests yet - - - )} - -
+
+
+ setSearch(e.target.value)} + className="h-8 max-w-md" + /> + +
+ + setApproveId(row.id)} + onDeny={(row) => setDenyId(row.id)} + onResendNda={(row) => handleResendNda(row.id)} + onPreviewNda={(row) => handlePreviewNda(row.id)} + /> + {approveId && ( setApproveId(null)} /> )} @@ -173,3 +88,4 @@ export function RequestsTab({ orgId }: { orgId: string }) {
); } + diff --git a/apps/app/src/components/ui/data-table/DataTable.tsx b/apps/app/src/components/ui/data-table/DataTable.tsx index 5335d0eff..d1bc6550e 100644 --- a/apps/app/src/components/ui/data-table/DataTable.tsx +++ b/apps/app/src/components/ui/data-table/DataTable.tsx @@ -255,7 +255,7 @@ export function DataTable({ {row.getVisibleCells().map((cell) => ( ({ table }: DataTableHeaderProps) { {headerGroup.headers.map((header) => ( {header.isPlaceholder ? null : ( diff --git a/bun.lock b/bun.lock index 1a758f815..d94b161b1 100644 --- a/bun.lock +++ b/bun.lock @@ -333,7 +333,7 @@ }, "packages/db": { "name": "@trycompai/db", - "version": "1.3.17", + "version": "1.3.18", "bin": { "comp-prisma-postinstall": "./dist/postinstall.js", }, diff --git a/packages/docs/openapi.json b/packages/docs/openapi.json index e09c94a31..a1393b099 100644 --- a/packages/docs/openapi.json +++ b/packages/docs/openapi.json @@ -6909,6 +6909,7 @@ "name": "friendlyUrl", "required": true, "in": "path", + "description": "Trust Portal friendly URL or Organization ID", "schema": { "type": "string" } @@ -7369,6 +7370,7 @@ "name": "friendlyUrl", "required": true, "in": "path", + "description": "Trust Portal friendly URL or Organization ID", "schema": { "type": "string" } From f9e0daddd6d22bd871e6a196290decddfe20afd7 Mon Sep 17 00:00:00 2001 From: Min Chun Fu <70210356+Itsnotaka@users.noreply.github.com> Date: Sat, 22 Nov 2025 02:06:31 +0900 Subject: [PATCH 6/7] refactor(grant-data-table): remove unused density prop from DataTable (#1815) * refactor(grant-data-table): remove unused density prop from DataTable * refactor(request-columns): remove unused imports and comments --- .../app/(app)/[orgId]/trust/components/grant-data-table.tsx | 3 +-- .../src/app/(app)/[orgId]/trust/components/request-columns.tsx | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx index 0055d8303..964f1d5d0 100644 --- a/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx +++ b/apps/app/src/app/(app)/[orgId]/trust/components/grant-data-table.tsx @@ -2,7 +2,7 @@ import { DataTable } from '@/components/ui/data-table/DataTable'; import type { AccessGrant } from '@/hooks/use-access-requests'; -import { buildGrantColumns, type GrantTableRow } from './grant-columns'; +import { buildGrantColumns } from './grant-columns'; interface GrantDataTableProps { data: AccessGrant[]; @@ -19,7 +19,6 @@ export function GrantDataTable({ data, isLoading, onRevoke }: GrantDataTableProp columns={columns} isLoading={isLoading} emptyMessage="No access grants yet" - density="compact" /> ); } diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx index 22a1bff6b..6fd77cf39 100644 --- a/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx +++ b/apps/app/src/app/(app)/[orgId]/trust/components/request-columns.tsx @@ -4,8 +4,6 @@ import type { AccessRequest } from '@/hooks/use-access-requests'; import { Badge } from '@comp/ui/badge'; import { Button } from '@comp/ui/button'; import type { ColumnDef } from '@tanstack/react-table'; -import { Copy } from 'lucide-react'; -import { toast } from 'sonner'; export type RequestTableRow = AccessRequest; @@ -97,7 +95,6 @@ export function buildRequestColumns({ const ndaPending = row.original.status === 'approved' && !row.original.grant; if (ndaPending) { - // @ts-expect-error - warning variant might be custom return Pending; } From 02b7e22e3aea5e2c4827954764a71001cdbfc2ee Mon Sep 17 00:00:00 2001 From: Min Chun Fu <70210356+Itsnotaka@users.noreply.github.com> Date: Sat, 22 Nov 2025 02:12:39 +0900 Subject: [PATCH 7/7] refactor(request-data-table): remove unused density prop from DataTable (#1816) --- .../trust/components/request-data-table.tsx | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx b/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx index c8dfbef70..001556659 100644 --- a/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx +++ b/apps/app/src/app/(app)/[orgId]/trust/components/request-data-table.tsx @@ -2,7 +2,7 @@ import { DataTable } from '@/components/ui/data-table/DataTable'; import type { AccessRequest } from '@/hooks/use-access-requests'; -import { buildRequestColumns, type RequestTableRow } from './request-columns'; +import { buildRequestColumns } from './request-columns'; interface RequestDataTableProps { data: AccessRequest[]; @@ -13,19 +13,19 @@ interface RequestDataTableProps { onPreviewNda: (row: AccessRequest) => void; } -export function RequestDataTable({ - data, - isLoading, - onApprove, - onDeny, - onResendNda, - onPreviewNda +export function RequestDataTable({ + data, + isLoading, + onApprove, + onDeny, + onResendNda, + onPreviewNda, }: RequestDataTableProps) { - const columns = buildRequestColumns({ - onApprove, - onDeny, - onResendNda, - onPreviewNda + const columns = buildRequestColumns({ + onApprove, + onDeny, + onResendNda, + onPreviewNda, }); return ( @@ -34,7 +34,6 @@ export function RequestDataTable({ columns={columns} isLoading={isLoading} emptyMessage="No access requests yet" - density="compact" /> ); }