diff --git a/.env.example b/.env.example index 80a79e6..69b5994 100644 --- a/.env.example +++ b/.env.example @@ -1 +1,2 @@ -ANTHROPIC_API_KEY= \ No newline at end of file +ANTHROPIC_API_KEY= +OPENAI_API_KEY= diff --git a/package-lock.json b/package-lock.json index e4df48a..fbb3c46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,10 @@ "electron-debug": "^3.2.0", "electron-log": "^4.4.8", "electron-updater": "^6.1.4", + "form-data": "^4.0.1", "framer-motion": "^11.11.9", + "node-fetch": "^3.3.2", + "openai": "^4.68.4", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^5.3.0", @@ -91,6 +94,10 @@ "webpack-cli": "^5.1.4", "webpack-dev-server": "^4.15.1", "webpack-merge": "^5.9.0" + }, + "engines": { + "node": ">=14.x", + "npm": ">=7.x" } }, "node_modules/@adobe/css-tools": { @@ -136,11 +143,53 @@ "undici-types": "~5.26.4" } }, + "node_modules/@anthropic-ai/sdk/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "node_modules/@anthropic-ai/sdk/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/@anthropic-ai/sdk/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", @@ -4105,6 +4154,29 @@ "semver": "bin/semver.js" } }, + "node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/@mapbox/node-pre-gyp/node_modules/nopt": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", @@ -4155,6 +4227,37 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@mapbox/node-pre-gyp/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "peer": true + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/@mdn/browser-compat-data": { "version": "5.5.51", "resolved": "https://registry.npmjs.org/@mdn/browser-compat-data/-/browser-compat-data-5.5.51.tgz", @@ -9058,6 +9161,15 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -11854,6 +11966,38 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/fetch-blob/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -12144,9 +12288,9 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -12174,6 +12318,18 @@ "node": ">= 12.20" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -14053,6 +14209,48 @@ "whatwg-fetch": "^3.4.1" } }, + "node_modules/isomorphic-fetch/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/isomorphic-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/isomorphic-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/isomorphic-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", @@ -16373,41 +16571,21 @@ } }, "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", "dependencies": { - "whatwg-url": "^5.0.0" + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" }, "engines": { - "node": "4.x || >=6.0.0" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" } }, "node_modules/node-forge": { @@ -16778,6 +16956,89 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "4.68.4", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.68.4.tgz", + "integrity": "sha512-LRinV8iU9VQplkr25oZlyrsYGPGasIwYN8KFMAAFTHHLHjHhejtJ5BALuLFrkGzY4wfbKhOhuT+7lcHZ+F3iEA==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.59", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", + "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/openai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/openai/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/openai/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/openai/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/opener": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", diff --git a/package.json b/package.json index d1c03d9..826ed2a 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,10 @@ "electron-debug": "^3.2.0", "electron-log": "^4.4.8", "electron-updater": "^6.1.4", + "form-data": "^4.0.1", "framer-motion": "^11.11.9", + "node-fetch": "^3.3.2", + "openai": "^4.68.4", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^5.3.0", diff --git a/src/main/main.ts b/src/main/main.ts index 553a7d2..86e548f 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -15,6 +15,12 @@ import path from 'path'; import { mainZustandBridge } from 'zutron/main'; import { createMainWindow } from './window'; import { store } from './store/create'; +import { resolveHtmlPath } from './util'; +import { OpenAI } from 'openai'; +import fs from 'fs'; +import os from 'os'; + +// const openai = new OpenAI(); class AppUpdater { constructor() { @@ -30,6 +36,43 @@ ipcMain.on('ipc-example', async (event, arg) => { event.reply('ipc-example', msgTemplate('pong')); }); +ipcMain.handle('stt', async (event, arrayBuffer) => { + try { + // Convert ArrayBuffer to Buffer + const buffer = Buffer.from(arrayBuffer); + + // Save the audio data to a temporary file + const tempFilePath = path.join( + os.tmpdir(), + `temp_recording_${Date.now()}.webm`, + ); + fs.writeFileSync(tempFilePath, buffer); + + // Read the file as a stream + const fileStream = fs.createReadStream(tempFilePath); + + // Call the OpenAI API + let response = null; + try { + response = await new OpenAI().audio.transcriptions.create( + { file: fileStream, model: 'whisper-1' }, // Model name + ); + } catch (apiError: any) { + console.error('OpenAI API error:', apiError); + throw new Error(`OpenAI API error: ${apiError.message}`); + } + + // Delete the temporary file + fs.unlinkSync(tempFilePath); + + // Send the transcription back to the renderer process + return response.text; + } catch (error: any) { + console.error('Error during speech-to-text conversion:', error); + throw error; + } +}); + if (process.env.NODE_ENV === 'production') { const sourceMapSupport = require('source-map-support'); sourceMapSupport.install(); diff --git a/src/main/preload.ts b/src/main/preload.ts index 300af24..b601426 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -21,9 +21,13 @@ const electronHandler = { ipcRenderer.removeListener(channel, subscription); }; }, + invoke: (channel: Channels, data: any) => ipcRenderer.invoke(channel, data), once(channel: Channels, func: (...args: unknown[]) => void) { ipcRenderer.once(channel, (_event, ...args) => func(...args)); }, + removeListener(channel: Channels, func: (...args: unknown[]) => void) { + ipcRenderer.removeListener(channel, func); + }, }, // Add window controls windowControls: { @@ -31,6 +35,9 @@ const electronHandler = { maximize: () => ipcRenderer.invoke('maximize-window'), close: () => ipcRenderer.invoke('close-window'), }, + process: { + env: process.env, + }, }; // Initialize Zutron bridge diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index a981f9d..584afcd 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -21,6 +21,15 @@ import { RunHistory } from './RunHistory'; function Main() { const dispatch = useDispatch(window.zutron); + + const [voiceOn, setVoiceOn] = React.useState(false); + const [transcribing, setTranscribing] = React.useState(false); + const mediaRecorderRef = React.useRef(null); + const audioChunksRef = React.useRef([]); + const [mediaStream, setMediaStream] = React.useState( + null, + ); + const { instructions: savedInstructions, fullyAuto, @@ -47,6 +56,97 @@ function Main() { } }; + const stt = async (audioBlob: Blob) => { + const reader = new FileReader(); + reader.onloadend = async () => { + setTranscribing(true); + const arrayBuffer = reader.result as ArrayBuffer; + try { + const result = await window.electron.ipcRenderer.invoke( + 'stt', + arrayBuffer, + ); + setLocalInstructions(result); + } catch (error) { + toast({ + description: `Speech-to-text error: ${error.message}`, + status: 'error', + duration: 5000, + isClosable: true, + }); + } finally { + setTranscribing(false); + } + }; + reader.readAsArrayBuffer(audioBlob); + }; + + React.useEffect(() => { + if (navigator.mediaDevices && window.MediaRecorder) { + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((stream) => { + setMediaStream(stream); + + const mediaRecorder = new MediaRecorder(stream); + mediaRecorderRef.current = mediaRecorder; + + mediaRecorder.ondataavailable = (event: BlobEvent) => { + if (event.data.size > 0) { + audioChunksRef.current.push(event.data); + } + }; + + mediaRecorder.onstop = () => { + const audioBlob = new Blob(audioChunksRef.current, { + type: 'audio/webm', + }); + stt(audioBlob); + audioChunksRef.current = []; // Reset the array for the next recording + }; + }) + .catch((error) => { + toast({ + description: `Error accessing microphone: ${error.message}`, + status: 'error', + duration: 3000, + isClosable: true, + }); + }); + } else { + toast({ + description: 'MediaRecorder is not supported in this browser.', + status: 'warning', + duration: 3000, + isClosable: true, + }); + } + + return () => { + if (mediaStream) { + mediaStream.getTracks().forEach((track) => track.stop()); + } + }; + }, []); + + React.useEffect(() => { + if (voiceOn) { + if ( + mediaRecorderRef.current && + mediaRecorderRef.current.state === 'inactive' + ) { + mediaRecorderRef.current.start(); + } + } else { + if ( + mediaRecorderRef.current && + mediaRecorderRef.current.state === 'recording' + ) { + mediaRecorderRef.current.stop(); + } + } + }, [voiceOn]); + return ( - ) => { setLocalInstructions(e.target.value); - // Auto-adjust height e.target.style.height = 'auto'; e.target.style.height = `${e.target.scrollHeight}px`; }} onKeyDown={handleKeyDown} /> + ; + }; // ... existing definitions ... windowControls: { minimize: () => Promise;