diff --git a/bin.ts b/bin.ts index 7c82ae0..259c1e4 100644 --- a/bin.ts +++ b/bin.ts @@ -79,6 +79,12 @@ yargs(hideBin(process.argv)) 'PostHog personal API key (phx_xxx) for authentication\nenv: POSTHOG_WIZARD_API_KEY', type: 'string', }, + interactive: { + default: false, + describe: + 'Review and approve planned events before implementation\nenv: POSTHOG_WIZARD_INTERACTIVE', + type: 'boolean', + }, }) .command( ['$0'], diff --git a/src/__tests__/cli.test.ts b/src/__tests__/cli.test.ts index 98b965b..a299c8a 100644 --- a/src/__tests__/cli.test.ts +++ b/src/__tests__/cli.test.ts @@ -26,6 +26,7 @@ describe('CLI argument parsing', () => { delete process.env.POSTHOG_WIZARD_CI; delete process.env.POSTHOG_WIZARD_API_KEY; delete process.env.POSTHOG_WIZARD_INSTALL_DIR; + delete process.env.POSTHOG_WIZARD_INTERACTIVE; // Mock process.exit to prevent test runner from exiting process.exit = jest.fn() as any; @@ -254,6 +255,31 @@ describe('CLI argument parsing', () => { }); }); + describe('--interactive flag', () => { + test('defaults to false when not specified', async () => { + await runCLI([]); + + const args = getLastCallArgs(mockRunWizard); + expect(args.interactive).toBe(false); + }); + + test('can be set to true', async () => { + await runCLI(['--interactive']); + + const args = getLastCallArgs(mockRunWizard); + expect(args.interactive).toBe(true); + }); + + test('respects POSTHOG_WIZARD_INTERACTIVE env var', async () => { + process.env.POSTHOG_WIZARD_INTERACTIVE = 'true'; + + await runCLI([]); + + const args = getLastCallArgs(mockRunWizard); + expect(args.interactive).toBe(true); + }); + }); + describe('CI environment variables', () => { test('respects POSTHOG_WIZARD_CI', async () => { process.env.POSTHOG_WIZARD_CI = 'true'; diff --git a/src/lib/__tests__/agent-interface.test.ts b/src/lib/__tests__/agent-interface.test.ts index beed455..6409589 100644 --- a/src/lib/__tests__/agent-interface.test.ts +++ b/src/lib/__tests__/agent-interface.test.ts @@ -31,6 +31,7 @@ describe('runAgent', () => { signup: false, localMcp: false, ci: false, + interactive: false, menu: false, }; @@ -251,4 +252,292 @@ describe('runAgent', () => { expect(mockClack.log.error).not.toHaveBeenCalled(); }); }); + + describe('interactive approval flow', () => { + it('should trigger onApprovalNeeded callback when approval signal is detected', async () => { + const mockOnApproval = jest.fn().mockResolvedValue({ + approved: true, + feedback: 'The user approved the plan. Proceed with implementation.', + }); + + function* mockGeneratorWithApproval() { + yield { + type: 'system', + subtype: 'init', + model: 'claude-opus-4-5-20251101', + tools: [], + mcp_servers: [], + }; + + yield { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: '[WIZARD-APPROVAL-NEEDED]\n1. page_view | src/app.tsx:15 | Fires on route change\n[/WIZARD-APPROVAL-NEEDED]', + }, + ], + }, + }; + + yield { + type: 'result', + subtype: 'success', + is_error: false, + result: 'Agent completed successfully', + }; + } + + mockQuery.mockReturnValue(mockGeneratorWithApproval()); + + const result = await runAgent( + defaultAgentConfig, + 'test prompt', + defaultOptions, + mockSpinner as unknown as ReturnType, + { + successMessage: 'Test success', + errorMessage: 'Test error', + onApprovalNeeded: mockOnApproval, + }, + ); + + expect(mockOnApproval).toHaveBeenCalledWith( + '1. page_view | src/app.tsx:15 | Fires on route change', + ); + expect(result).toEqual({}); + }); + + it('should skip approval when no callback is provided', async () => { + function* mockGeneratorWithApprovalSignalButNoCallback() { + yield { + type: 'system', + subtype: 'init', + model: 'claude-opus-4-5-20251101', + tools: [], + mcp_servers: [], + }; + + yield { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: '[WIZARD-APPROVAL-NEEDED]\n1. page_view | src/app.tsx:15 | Fires on route change\n[/WIZARD-APPROVAL-NEEDED]', + }, + ], + }, + }; + + yield { + type: 'result', + subtype: 'success', + is_error: false, + result: 'Agent completed successfully', + }; + } + + mockQuery.mockReturnValue(mockGeneratorWithApprovalSignalButNoCallback()); + + // No onApprovalNeeded callback — should complete without blocking + const result = await runAgent( + defaultAgentConfig, + 'test prompt', + defaultOptions, + mockSpinner as unknown as ReturnType, + { + successMessage: 'Test success', + errorMessage: 'Test error', + }, + ); + + expect(result).toEqual({}); + expect(mockSpinner.stop).toHaveBeenCalledWith('Test success'); + }); + + it('should handle approval callback errors gracefully', async () => { + const mockOnApproval = jest + .fn() + .mockRejectedValue(new Error('User cancelled')); + + function* mockGeneratorWithApprovalError() { + yield { + type: 'system', + subtype: 'init', + model: 'claude-opus-4-5-20251101', + tools: [], + mcp_servers: [], + }; + + yield { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: '[WIZARD-APPROVAL-NEEDED]\n1. page_view | src/app.tsx:15 | Fires on route change\n[/WIZARD-APPROVAL-NEEDED]', + }, + ], + }, + }; + + yield { + type: 'result', + subtype: 'success', + is_error: false, + result: 'Agent completed successfully', + }; + } + + mockQuery.mockReturnValue(mockGeneratorWithApprovalError()); + + const result = await runAgent( + defaultAgentConfig, + 'test prompt', + defaultOptions, + mockSpinner as unknown as ReturnType, + { + successMessage: 'Test success', + errorMessage: 'Test error', + onApprovalNeeded: mockOnApproval, + }, + ); + + // Should still complete — error handler sends fallback message + expect(result).toEqual({}); + }); + + it('should support multiple rounds of approval (modify then approve)', async () => { + let callCount = 0; + const mockOnApproval = jest.fn().mockImplementation(() => { + callCount++; + if (callCount === 1) { + return Promise.resolve({ + approved: false, + feedback: + 'The user wants to modify: remove event_2. Output updated plan.', + }); + } + return Promise.resolve({ + approved: true, + feedback: 'The user approved the plan. Proceed with implementation.', + }); + }); + + function* mockGeneratorWithTwoApprovalRounds() { + yield { + type: 'system', + subtype: 'init', + model: 'claude-opus-4-5-20251101', + tools: [], + mcp_servers: [], + }; + + // First plan + yield { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: '[WIZARD-APPROVAL-NEEDED]\n1. event_1 | src/a.tsx:10 | desc1\n2. event_2 | src/b.tsx:20 | desc2\n[/WIZARD-APPROVAL-NEEDED]', + }, + ], + }, + }; + + // Revised plan after modification + yield { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: '[WIZARD-APPROVAL-NEEDED]\n1. event_1 | src/a.tsx:10 | desc1\n[/WIZARD-APPROVAL-NEEDED]', + }, + ], + }, + }; + + yield { + type: 'result', + subtype: 'success', + is_error: false, + result: 'Agent completed successfully', + }; + } + + mockQuery.mockReturnValue(mockGeneratorWithTwoApprovalRounds()); + + const result = await runAgent( + defaultAgentConfig, + 'test prompt', + defaultOptions, + mockSpinner as unknown as ReturnType, + { + successMessage: 'Test success', + errorMessage: 'Test error', + onApprovalNeeded: mockOnApproval, + }, + ); + + expect(mockOnApproval).toHaveBeenCalledTimes(2); + expect(result).toEqual({}); + }); + + it('should return APPROVAL_CANCELLED error when agent emits the signal', async () => { + const mockOnApproval = jest.fn().mockResolvedValue({ + approved: true, + feedback: 'Approved', + }); + + function* mockGeneratorWithApprovalCancelled() { + yield { + type: 'system', + subtype: 'init', + model: 'claude-opus-4-5-20251101', + tools: [], + mcp_servers: [], + }; + + yield { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: '[ERROR-APPROVAL-CANCELLED] Approval cancelled by user or error.', + }, + ], + }, + }; + + yield { + type: 'result', + subtype: 'success', + is_error: false, + result: 'Agent completed', + }; + } + + mockQuery.mockReturnValue(mockGeneratorWithApprovalCancelled()); + + const result = await runAgent( + defaultAgentConfig, + 'test prompt', + defaultOptions, + mockSpinner as unknown as ReturnType, + { + successMessage: 'Test success', + errorMessage: 'Test error', + onApprovalNeeded: mockOnApproval, + }, + ); + + expect(result).toEqual({ error: 'WIZARD_APPROVAL_CANCELLED' }); + expect(mockSpinner.stop).toHaveBeenCalledWith('Approval flow cancelled'); + }); + }); }); diff --git a/src/lib/agent-interface.ts b/src/lib/agent-interface.ts index 3b81bdd..f6ec56f 100644 --- a/src/lib/agent-interface.ts +++ b/src/lib/agent-interface.ts @@ -49,6 +49,12 @@ export const AgentSignals = { ERROR_RESOURCE_MISSING: '[ERROR-RESOURCE-MISSING]', /** Signal emitted when the agent provides a remark about its run */ WIZARD_REMARK: '[WIZARD-REMARK]', + /** Signal emitted when the agent has an event plan ready for user approval */ + APPROVAL_NEEDED: '[WIZARD-APPROVAL-NEEDED]', + /** End marker for the approval plan block */ + APPROVAL_END: '[/WIZARD-APPROVAL-NEEDED]', + /** Signal emitted when the approval flow was cancelled or errored */ + ERROR_APPROVAL_CANCELLED: '[ERROR-APPROVAL-CANCELLED]', } as const; export type AgentSignal = (typeof AgentSignals)[keyof typeof AgentSignals]; @@ -66,6 +72,8 @@ export enum AgentErrorType { RATE_LIMIT = 'WIZARD_RATE_LIMIT', /** Generic API error */ API_ERROR = 'WIZARD_API_ERROR', + /** Approval flow was cancelled or errored */ + APPROVAL_CANCELLED = 'WIZARD_APPROVAL_CANCELLED', } export type AgentConfig = { @@ -415,6 +423,9 @@ export async function runAgent( spinnerMessage?: string; successMessage?: string; errorMessage?: string; + onApprovalNeeded?: ( + planText: string, + ) => Promise<{ approved: boolean; feedback: string }>; }, ): Promise<{ error?: AgentErrorType; message?: string }> { const { @@ -422,6 +433,7 @@ export async function runAgent( spinnerMessage = 'Customizing your PostHog setup...', successMessage = 'PostHog integration complete', errorMessage = 'Integration failed', + onApprovalNeeded, } = config ?? {}; const { query } = await getSDKModule(); @@ -442,6 +454,31 @@ export async function runAgent( // Track if we received a successful result (before any cleanup errors) let receivedSuccessResult = false; + // Interactive approval state: when --interactive is set, the agent outputs an event + // plan and pauses before writing code. Supports multiple rounds of plan review — + // each call to waitForFeedback() returns a new promise that resolves when + // sendFeedback() is called, allowing the approval loop to repeat until approved. + let _resolveFeedback: + | ((value: { approved: boolean; feedback: string }) => void) + | null = null; + + function waitForFeedback(): Promise<{ approved: boolean; feedback: string }> { + return new Promise((resolve) => { + _resolveFeedback = resolve; + }); + } + + function sendFeedback(value: { approved: boolean; feedback: string }): void { + if (_resolveFeedback) { + const resolve = _resolveFeedback; + _resolveFeedback = null; + resolve(value); + } + } + + // canUseTool blocks Write/Edit while awaitingApproval is true as a safety net. + let awaitingApproval = false; + // Workaround for SDK bug: stdin closes before canUseTool responses can be sent. // The fix is to use an async generator for the prompt that stays open until // the result is received, keeping the stdin stream alive for permission responses. @@ -459,6 +496,26 @@ export async function runAgent( message: { role: 'user', content: prompt }, parent_tool_use_id: null, }; + + // In interactive mode, loop until the user approves the plan. + // Each iteration: wait for feedback → yield it → if modification, loop again. + if (onApprovalNeeded) { + while (true) { + const result = await waitForFeedback(); + if (result.feedback) { + yield { + type: 'user', + session_id: '', + message: { role: 'user', content: result.feedback }, + parent_tool_use_id: null, + }; + } + if (result.approved) { + break; + } + } + } + await resultReceived; }; @@ -542,6 +599,20 @@ export async function runAgent( }, canUseTool: (toolName: string, input: unknown) => { logToFile('canUseTool called:', { toolName, input }); + + // Block Write/Edit while awaiting user approval of event plan + if ( + awaitingApproval && + (toolName === 'Write' || toolName === 'Edit') + ) { + logToFile(`Blocking ${toolName} while awaiting approval`); + return Promise.resolve({ + behavior: 'deny' as const, + message: + 'Waiting for user to review the event plan. Do not write or edit code until approval is received.', + }); + } + const result = wizardCanUseTool( toolName, input as Record, @@ -597,6 +668,18 @@ export async function runAgent( spinner, collectedText, receivedSuccessResult, + onApprovalNeeded + ? { + onApprovalNeeded, + isAwaiting: () => awaitingApproval, + setAwaiting: (v: boolean) => { + awaitingApproval = v; + }, + resolve: (result: { approved: boolean; feedback: string }) => { + sendFeedback(result); + }, + } + : undefined, ); // Signal completion when result received @@ -625,6 +708,12 @@ export async function runAgent( return { error: AgentErrorType.RESOURCE_MISSING }; } + if (outputText.includes(AgentSignals.ERROR_APPROVAL_CANCELLED)) { + logToFile('Agent error: APPROVAL_CANCELLED'); + spinner.stop('Approval flow cancelled'); + return { error: AgentErrorType.APPROVAL_CANCELLED }; + } + // Check for API errors (rate limits, etc.) // Extract just the API error line(s), not the entire output const apiErrorMatch = outputText.match(/API Error: [^\n]+/g); @@ -700,6 +789,14 @@ function handleSDKMessage( spinner: ReturnType, collectedText: string[], receivedSuccessResult = false, + approvalContext?: { + onApprovalNeeded: ( + planText: string, + ) => Promise<{ approved: boolean; feedback: string }>; + isAwaiting: () => boolean; + setAwaiting: (v: boolean) => void; + resolve: (result: { approved: boolean; feedback: string }) => void; + }, ): void { logToFile(`SDK Message: ${message.type}`, JSON.stringify(message, null, 2)); @@ -729,6 +826,44 @@ function handleSDKMessage( spinner.stop(statusMatch[1].trim()); spinner.start('Integrating PostHog...'); } + + // Check for approval signal (interactive mode) + if ( + approvalContext && + !approvalContext.isAwaiting() && + block.text.includes(AgentSignals.APPROVAL_NEEDED) + ) { + const startMarker = AgentSignals.APPROVAL_NEEDED; + const endMarker = AgentSignals.APPROVAL_END; + const text = block.text as string; + const startIdx = text.indexOf(startMarker) + startMarker.length; + const endIdx = text.indexOf(endMarker); + const planText = + endIdx > startIdx + ? text.slice(startIdx, endIdx).trim() + : text.slice(startIdx).trim(); + + logToFile('Approval signal detected, plan text:', planText); + approvalContext.setAwaiting(true); + + approvalContext + .onApprovalNeeded(planText) + .then((result) => { + approvalContext.setAwaiting(false); + approvalContext.resolve(result); + logToFile( + `Approval feedback sent to agent (approved: ${result.approved})`, + ); + }) + .catch((err) => { + logToFile('Approval flow error:', err); + approvalContext.setAwaiting(false); + approvalContext.resolve({ + approved: true, + feedback: `The approval flow was cancelled or encountered an error. Stop immediately — do not write or modify any files. You must emit: ${AgentSignals.ERROR_APPROVAL_CANCELLED} Approval cancelled by user or error.`, + }); + }); + } } } } diff --git a/src/lib/agent-runner.ts b/src/lib/agent-runner.ts index 46ed98b..f883fa8 100644 --- a/src/lib/agent-runner.ts +++ b/src/lib/agent-runner.ts @@ -6,6 +6,7 @@ import { import type { WizardOptions } from '../utils/types'; import { abort, + abortIfCancelled, askForAIConsent, confirmContinueIfNoOrDirtyGitRepo, ensurePackageIsInstalled, @@ -27,6 +28,7 @@ import { } from './agent-interface'; import { getCloudUrlFromRegion } from '../utils/urls'; import chalk from 'chalk'; +import path from 'path'; import * as semver from 'semver'; import { addMCPServerToClientsStep, @@ -165,6 +167,7 @@ export async function runAgentWizard( typescript: typeScriptDetected, projectApiKey, host, + interactive: options.interactive ?? false, }, frameworkContext, ); @@ -203,6 +206,76 @@ export async function runAgentWizard( spinnerMessage: SPINNER_MESSAGE, successMessage: config.ui.successMessage, errorMessage: 'Integration failed', + onApprovalNeeded: options.interactive + ? async (planText: string) => { + spinner.stop('Event plan ready for review'); + + const events = parsePlanText(planText); + + if (events.length === 0) { + await abort( + 'The agent did not produce a valid event plan. Exiting.', + 1, + ); + } + + clack.log.info( + chalk.cyan('Here are the events the agent plans to track:\n'), + ); + for (const [i, event] of events.entries()) { + const fileDisplay = event.file + ? chalk.dim(` (${path.basename(event.file)})`) + : ''; + clack.log.step( + `${i + 1}. ${chalk.bold(event.name)}${fileDisplay} — ${ + event.description + }`, + ); + } + + const action: string = await abortIfCancelled( + clack.select({ + message: 'What would you like to do?', + options: [ + { + value: 'approve', + label: 'Approve and continue', + hint: 'Implement all listed events', + }, + { + value: 'modify', + label: 'Modify the plan', + hint: 'Describe changes in text', + }, + ], + }), + ); + + if (action === 'approve') { + spinner.start('Implementing approved event plan...'); + return { + approved: true, + feedback: + 'The user approved the plan. Proceed with implementation of all listed events.', + }; + } + + // action === 'modify' + const modifications: string = await abortIfCancelled( + clack.text({ + message: 'How should the plan be modified?', + placeholder: + 'e.g., remove signup_clicked, rename page_view to route_changed, add checkout_started on payment page', + }), + ); + + spinner.start('Revising event plan...'); + return { + approved: false, + feedback: `The user wants to modify the event plan:\n${modifications}\n\nApply these changes and output the UPDATED event plan using the EXACT same format between ${AgentSignals.APPROVAL_NEEDED} and ${AgentSignals.APPROVAL_END} markers. Do NOT implement yet — wait for the user to approve the updated plan.`, + }; + } + : undefined, }, ); @@ -258,6 +331,22 @@ ${chalk.cyan(config.metadata.docsUrl)}`; process.exit(1); } + if (agentResult.error === AgentErrorType.APPROVAL_CANCELLED) { + analytics.capture(WIZARD_INTERACTION_EVENT_NAME, { + action: 'approval cancelled', + integration: config.metadata.integration, + error_type: AgentErrorType.APPROVAL_CANCELLED, + }); + + clack.outro( + `${chalk.yellow( + 'Wizard cancelled.', + )} No changes were made to your project.`, + ); + await analytics.shutdown('cancelled'); + process.exit(0); + } + if ( agentResult.error === AgentErrorType.RATE_LIMIT || agentResult.error === AgentErrorType.API_ERROR @@ -360,6 +449,7 @@ function buildIntegrationPrompt( typescript: boolean; projectApiKey: string; host: string; + interactive: boolean; }, frameworkContext: Record, ): string { @@ -403,7 +493,27 @@ STEP 3: Run the installation command using Bash: STEP 4: Load the installed skill's SKILL.md file to understand what references are available. STEP 5: Follow the skill's workflow files in sequence. Look for numbered workflow files in the references (e.g., files with patterns like "1.0-", "1.1-", "1.2-"). Start with the first one and proceed through each step until completion. Each workflow file will tell you what to do and which file comes next. - +${ + context.interactive + ? ` +STEP 5.5 (INTERACTIVE APPROVAL — MANDATORY): Before writing ANY integration code, you MUST output your complete event tracking plan. Follow this format EXACTLY — every event on its own numbered line, with three pipe-separated fields: + +${AgentSignals.APPROVAL_NEEDED} +1. {event_name} | {file_path}:{line_number} | {description of when this event fires} +2. {event_name} | {file_path}:{line_number} | {description of when this event fires} +${AgentSignals.APPROVAL_END} + +Rules: +- Replace the {placeholders} with real values derived from this project's codebase +- Every line MUST follow: NUMBER. EVENT_NAME | FILE_PATH:LINE | DESCRIPTION +- Include the line number where you plan to insert the tracking code +- Do NOT use markdown, extra text, or any other format between the markers +- List ALL events you plan to implement — nothing should be left out + +Then STOP and wait for user feedback. Do not proceed to write integration code until you receive a follow-up message approving the plan. The user may remove events, add new ones, or rename events. If the user requests modifications, apply them and output the COMPLETE updated plan in the EXACT same format between the same ${AgentSignals.APPROVAL_NEEDED} and ${AgentSignals.APPROVAL_END} markers. Then STOP and wait again. Repeat this cycle until the user explicitly approves. Only after approval should you implement the approved events. +` + : '' +} STEP 6: Set up environment variables for PostHog using the env-file-tools MCP server (this runs locally — secret values never leave the machine): - Use check_env_keys to see which keys already exist in the project's .env file (e.g. .env.local or .env). - Use set_env_values to create or update the PostHog API key and host, using the appropriate naming convention for ${ @@ -415,3 +525,35 @@ Important: Look for lockfiles (pnpm-lock.yaml, package-lock.json, yarn.lock, bun `; } + +type PlanEvent = { + name: string; + file: string; + description: string; +}; + +/** + * Parses the agent's structured event plan output into individual events. + * + * Expects each line to follow the format the agent is prompted to use: + * `1. event_name | file_path:line | description` + * + * Lines that don't match (blanks, markdown, extra commentary) are silently + * skipped so the parser is tolerant of minor LLM formatting deviations. + */ +function parsePlanText(planText: string): PlanEvent[] { + const events: PlanEvent[] = []; + for (const line of planText.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) continue; + const match = trimmed.match(/^\d+\.\s*(.+?)\s*\|\s*(.+?)\s*\|\s*(.+)$/); + if (match) { + events.push({ + name: match[1].trim(), + file: match[2].trim(), + description: match[3].trim(), + }); + } + } + return events; +} diff --git a/src/run.ts b/src/run.ts index e5e5311..22c6ffc 100644 --- a/src/run.ts +++ b/src/run.ts @@ -26,6 +26,7 @@ type Args = { localMcp?: boolean; ci?: boolean; apiKey?: string; + interactive?: boolean; menu?: boolean; }; @@ -56,6 +57,7 @@ export async function runWizard(argv: Args) { localMcp: finalArgs.localMcp ?? false, ci: finalArgs.ci ?? false, apiKey: finalArgs.apiKey, + interactive: finalArgs.interactive ?? false, menu: finalArgs.menu ?? false, }; @@ -65,6 +67,14 @@ export async function runWizard(argv: Args) { clack.log.info(chalk.dim('Running in CI mode')); } + if (wizardOptions.interactive) { + clack.log.info( + chalk.dim( + 'Running in interactive mode. You will review the event plan before implementation', + ), + ); + } + const integration = finalArgs.integration ?? (await getIntegrationForSetup(wizardOptions)); diff --git a/src/utils/types.ts b/src/utils/types.ts index 7379156..8ec8d27 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -56,6 +56,11 @@ export type WizardOptions = { */ apiKey?: string; + /** + * Enable interactive mode with event plan approval before implementation + */ + interactive: boolean; + /** * Whether to show the menu for manual integration selection instead of auto-detecting. */