From 9af3bbb461e2197a941820c3b68fbe9fa7cdab1d Mon Sep 17 00:00:00 2001 From: Yukhym Rubin Date: Fri, 25 Oct 2024 02:00:04 +0200 Subject: [PATCH 1/4] voice mode --- .env.example | 1 - package-lock.json | 360 +++++++++++++++++++++++++++++++++++++++---- package.json | 4 + src/main/main.ts | 33 ++++ src/main/preload.ts | 18 ++- src/renderer/App.tsx | 142 ++++++++++++++++- 6 files changed, 515 insertions(+), 43 deletions(-) delete mode 100644 .env.example diff --git a/.env.example b/.env.example deleted file mode 100644 index 80a79e6..0000000 --- a/.env.example +++ /dev/null @@ -1 +0,0 @@ -ANTHROPIC_API_KEY= \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index e4df48a..46f47c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,11 @@ "electron-debug": "^3.2.0", "electron-log": "^4.4.8", "electron-updater": "^6.1.4", + "fluent-ffmpeg": "^2.1.3", + "form-data": "^4.0.1", "framer-motion": "^11.11.9", + "node-fetch": "^3.3.2", + "openai": "^4.68.4", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^5.3.0", @@ -91,6 +95,10 @@ "webpack-cli": "^5.1.4", "webpack-dev-server": "^4.15.1", "webpack-merge": "^5.9.0" + }, + "engines": { + "node": ">=14.x", + "npm": ">=7.x" } }, "node_modules/@adobe/css-tools": { @@ -136,11 +144,53 @@ "undici-types": "~5.26.4" } }, + "node_modules/@anthropic-ai/sdk/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "node_modules/@anthropic-ai/sdk/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/@anthropic-ai/sdk/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", @@ -4105,6 +4155,29 @@ "semver": "bin/semver.js" } }, + "node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/@mapbox/node-pre-gyp/node_modules/nopt": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", @@ -4155,6 +4228,37 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@mapbox/node-pre-gyp/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "peer": true + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/@mdn/browser-compat-data": { "version": "5.5.51", "resolved": "https://registry.npmjs.org/@mdn/browser-compat-data/-/browser-compat-data-5.5.51.tgz", @@ -9058,6 +9162,15 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -11854,6 +11967,38 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/fetch-blob/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -12072,6 +12217,36 @@ "dev": true, "license": "ISC" }, + "node_modules/fluent-ffmpeg": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.3.tgz", + "integrity": "sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==", + "license": "MIT", + "dependencies": { + "async": "^0.2.9", + "which": "^1.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fluent-ffmpeg/node_modules/async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==" + }, + "node_modules/fluent-ffmpeg/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, "node_modules/focus-lock": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/focus-lock/-/focus-lock-1.3.5.tgz", @@ -12144,9 +12319,9 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -12174,6 +12349,18 @@ "node": ">= 12.20" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -14053,6 +14240,48 @@ "whatwg-fetch": "^3.4.1" } }, + "node_modules/isomorphic-fetch/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/isomorphic-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/isomorphic-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/isomorphic-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", @@ -16373,41 +16602,21 @@ } }, "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", "dependencies": { - "whatwg-url": "^5.0.0" + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" }, "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" } }, "node_modules/node-forge": { @@ -16778,6 +16987,89 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "4.68.4", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.68.4.tgz", + "integrity": "sha512-LRinV8iU9VQplkr25oZlyrsYGPGasIwYN8KFMAAFTHHLHjHhejtJ5BALuLFrkGzY4wfbKhOhuT+7lcHZ+F3iEA==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.59", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", + "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/openai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/openai/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/openai/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/openai/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/opener": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", diff --git a/package.json b/package.json index d1c03d9..847cd16 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,11 @@ "electron-debug": "^3.2.0", "electron-log": "^4.4.8", "electron-updater": "^6.1.4", + "fluent-ffmpeg": "^2.1.3", + "form-data": "^4.0.1", "framer-motion": "^11.11.9", + "node-fetch": "^3.3.2", + "openai": "^4.68.4", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^5.3.0", diff --git a/src/main/main.ts b/src/main/main.ts index 1ac3c03..83377cc 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -16,6 +16,10 @@ import { mainZustandBridge } from 'zutron/main'; import MenuBuilder from './menu'; import { store } from './store/create'; import { resolveHtmlPath } from './util'; +import { OpenAI } from 'openai'; +import fs from 'fs'; + +const openai = new OpenAI(); class AppUpdater { constructor() { @@ -33,6 +37,35 @@ ipcMain.on('ipc-example', async (event, arg) => { event.reply('ipc-example', msgTemplate('pong')); }); +ipcMain.handle('stt', async (event, arrayBuffer) => { + try { + // Convert ArrayBuffer to Buffer + const buffer = Buffer.from(arrayBuffer); + + // Save the audio data to a temporary file + const tempFilePath = path.join(__dirname, 'temp_recording.webm'); + console.error('tempFilePath:', tempFilePath); + fs.writeFileSync(tempFilePath, buffer); + + // Read the file as a stream + const fileStream = fs.createReadStream(tempFilePath); + + // Call the OpenAI API + const response = await openai.audio.transcriptions.create( + { file: fileStream, model: 'whisper-1' }, // Model name + ); + + // Delete the temporary file + fs.unlinkSync(tempFilePath); + + // Send the transcription back to the renderer process + return response.text; + } catch (error: any) { + console.error('Error during speech-to-text conversion:', error); + event.sender.send('stt-result', `Error: ${error.message}`); + } +}); + if (process.env.NODE_ENV === 'production') { const sourceMapSupport = require('source-map-support'); sourceMapSupport.install(); diff --git a/src/main/preload.ts b/src/main/preload.ts index 300af24..18613d0 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -13,17 +13,23 @@ const electronHandler = { ipcRenderer.send(channel, ...args); }, on(channel: Channels, func: (...args: unknown[]) => void) { - const subscription = (_event: IpcRendererEvent, ...args: unknown[]) => - func(...args); - ipcRenderer.on(channel, subscription); + // const subscription = (_event: IpcRendererEvent, ...args: unknown[]) => + // func(...args); + // ipcRenderer.on(channel, subscription); - return () => { - ipcRenderer.removeListener(channel, subscription); - }; + // return () => { + // ipcRenderer.removeListener(channel, subscription); + // }; + console.error('on:', channel); + ipcRenderer.on(channel, (event, data) => func(data)); }, + invoke: (channel: Channels, data: any) => ipcRenderer.invoke(channel, data), once(channel: Channels, func: (...args: unknown[]) => void) { ipcRenderer.once(channel, (_event, ...args) => func(...args)); }, + removeListener(channel: Channels, func: (...args: unknown[]) => void) { + ipcRenderer.removeListener(channel, func); + }, }, // Add window controls windowControls: { diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index a981f9d..581ccf6 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -21,6 +21,14 @@ import { RunHistory } from './RunHistory'; function Main() { const dispatch = useDispatch(window.zutron); + + const [voiceOn, setVoiceOn] = React.useState(false); + const mediaRecorderRef = React.useRef(null); + const audioChunksRef = React.useRef([]); + const [mediaStream, setMediaStream] = React.useState( + null, + ); + const { instructions: savedInstructions, fullyAuto, @@ -47,6 +55,109 @@ function Main() { } }; + const stt = async (audioBlob: Blob) => { + console.log('stt function called'); + const reader = new FileReader(); + reader.onloadend = async () => { + const arrayBuffer = reader.result as ArrayBuffer; + console.log('Invoking stt handler in main process'); + try { + const result = await window.electron.ipcRenderer.invoke( + 'stt', + arrayBuffer, + ); + console.log('Received transcription result:', result); + setLocalInstructions(result); + } catch (error) { + console.error('Error in stt:', error); + toast({ + description: `Speech-to-text error: ${error.message}`, + status: 'error', + duration: 5000, + isClosable: true, + }); + } + }; + reader.readAsArrayBuffer(audioBlob); + }; + + React.useEffect(() => { + if (navigator.mediaDevices && window.MediaRecorder) { + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((stream) => { + setMediaStream(stream); + + const mediaRecorder = new MediaRecorder(stream); + mediaRecorderRef.current = mediaRecorder; + + mediaRecorder.ondataavailable = (event: BlobEvent) => { + console.log( + 'ondataavailable event received, data size:', + event.data.size, + ); + if (event.data.size > 0) { + audioChunksRef.current.push(event.data); + } + }; + + mediaRecorder.onstop = () => { + console.log( + 'mediaRecorder stopped, audioChunks length:', + audioChunksRef.current.length, + ); + const audioBlob = new Blob(audioChunksRef.current, { + type: 'audio/webm', + }); + console.log('Created audioBlob:', audioBlob); + stt(audioBlob); + audioChunksRef.current = []; // Reset the array for the next recording + }; + }) + .catch((error) => { + console.error('Error accessing microphone: ', error); + toast({ + description: `Error accessing microphone: ${error.message}`, + status: 'error', + duration: 3000, + isClosable: true, + }); + }); + } else { + console.warn('MediaRecorder is not supported in this browser.'); + toast({ + description: 'MediaRecorder is not supported in this browser.', + status: 'warning', + duration: 3000, + isClosable: true, + }); + } + + return () => { + if (mediaStream) { + mediaStream.getTracks().forEach((track) => track.stop()); + } + }; + }, []); + + React.useEffect(() => { + if (voiceOn) { + if ( + mediaRecorderRef.current && + mediaRecorderRef.current.state === 'inactive' + ) { + mediaRecorderRef.current.start(); + } + } else { + if ( + mediaRecorderRef.current && + mediaRecorderRef.current.state === 'recording' + ) { + mediaRecorderRef.current.stop(); + } + } + }, [voiceOn]); + return ( - ) => { setLocalInstructions(e.target.value); - // Auto-adjust height e.target.style.height = 'auto'; e.target.style.height = `${e.target.scrollHeight}px`; }} onKeyDown={handleKeyDown} /> + Date: Fri, 25 Oct 2024 02:06:18 +0200 Subject: [PATCH 2/4] updated .env.example --- .env.example | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..69b5994 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +ANTHROPIC_API_KEY= +OPENAI_API_KEY= From d89c4fb538005517a7db73e6b367e646bfe339ac Mon Sep 17 00:00:00 2001 From: Yukhym Rubin Date: Fri, 25 Oct 2024 12:27:10 +0200 Subject: [PATCH 3/4] returned ipcRenderer.on definition and removed fluent-ffmpeg --- package-lock.json | 31 ------------------------------- package.json | 1 - src/main/preload.ts | 14 ++++++-------- 3 files changed, 6 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index 46f47c6..fbb3c46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,6 @@ "electron-debug": "^3.2.0", "electron-log": "^4.4.8", "electron-updater": "^6.1.4", - "fluent-ffmpeg": "^2.1.3", "form-data": "^4.0.1", "framer-motion": "^11.11.9", "node-fetch": "^3.3.2", @@ -12217,36 +12216,6 @@ "dev": true, "license": "ISC" }, - "node_modules/fluent-ffmpeg": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.3.tgz", - "integrity": "sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==", - "license": "MIT", - "dependencies": { - "async": "^0.2.9", - "which": "^1.1.1" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/fluent-ffmpeg/node_modules/async": { - "version": "0.2.10", - "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", - "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==" - }, - "node_modules/fluent-ffmpeg/node_modules/which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "which": "bin/which" - } - }, "node_modules/focus-lock": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/focus-lock/-/focus-lock-1.3.5.tgz", diff --git a/package.json b/package.json index 847cd16..826ed2a 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,6 @@ "electron-debug": "^3.2.0", "electron-log": "^4.4.8", "electron-updater": "^6.1.4", - "fluent-ffmpeg": "^2.1.3", "form-data": "^4.0.1", "framer-motion": "^11.11.9", "node-fetch": "^3.3.2", diff --git a/src/main/preload.ts b/src/main/preload.ts index 18613d0..f083ca4 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -13,15 +13,13 @@ const electronHandler = { ipcRenderer.send(channel, ...args); }, on(channel: Channels, func: (...args: unknown[]) => void) { - // const subscription = (_event: IpcRendererEvent, ...args: unknown[]) => - // func(...args); - // ipcRenderer.on(channel, subscription); + const subscription = (_event: IpcRendererEvent, ...args: unknown[]) => + func(...args); + ipcRenderer.on(channel, subscription); - // return () => { - // ipcRenderer.removeListener(channel, subscription); - // }; - console.error('on:', channel); - ipcRenderer.on(channel, (event, data) => func(data)); + return () => { + ipcRenderer.removeListener(channel, subscription); + }; }, invoke: (channel: Channels, data: any) => ipcRenderer.invoke(channel, data), once(channel: Channels, func: (...args: unknown[]) => void) { From 0fc6b8c8741a8f17f25d49e358393b8c6529a24e Mon Sep 17 00:00:00 2001 From: Yukhym Rubin Date: Fri, 25 Oct 2024 16:00:13 +0200 Subject: [PATCH 4/4] Better error handling and UX improvement --- src/main/main.ts | 23 ++++++++++++++++------- src/main/preload.ts | 3 +++ src/renderer/App.tsx | 38 ++++++++++++++++++++++---------------- src/renderer/global.d.ts | 3 +++ 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/main/main.ts b/src/main/main.ts index ff1bfd0..86e548f 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -18,8 +18,9 @@ import { store } from './store/create'; import { resolveHtmlPath } from './util'; import { OpenAI } from 'openai'; import fs from 'fs'; +import os from 'os'; -const openai = new OpenAI(); +// const openai = new OpenAI(); class AppUpdater { constructor() { @@ -41,17 +42,25 @@ ipcMain.handle('stt', async (event, arrayBuffer) => { const buffer = Buffer.from(arrayBuffer); // Save the audio data to a temporary file - const tempFilePath = path.join(__dirname, 'temp_recording.webm'); - console.error('tempFilePath:', tempFilePath); + const tempFilePath = path.join( + os.tmpdir(), + `temp_recording_${Date.now()}.webm`, + ); fs.writeFileSync(tempFilePath, buffer); // Read the file as a stream const fileStream = fs.createReadStream(tempFilePath); // Call the OpenAI API - const response = await openai.audio.transcriptions.create( - { file: fileStream, model: 'whisper-1' }, // Model name - ); + let response = null; + try { + response = await new OpenAI().audio.transcriptions.create( + { file: fileStream, model: 'whisper-1' }, // Model name + ); + } catch (apiError: any) { + console.error('OpenAI API error:', apiError); + throw new Error(`OpenAI API error: ${apiError.message}`); + } // Delete the temporary file fs.unlinkSync(tempFilePath); @@ -60,7 +69,7 @@ ipcMain.handle('stt', async (event, arrayBuffer) => { return response.text; } catch (error: any) { console.error('Error during speech-to-text conversion:', error); - event.sender.send('stt-result', `Error: ${error.message}`); + throw error; } }); diff --git a/src/main/preload.ts b/src/main/preload.ts index f083ca4..b601426 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -35,6 +35,9 @@ const electronHandler = { maximize: () => ipcRenderer.invoke('maximize-window'), close: () => ipcRenderer.invoke('close-window'), }, + process: { + env: process.env, + }, }; // Initialize Zutron bridge diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 581ccf6..584afcd 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -23,6 +23,7 @@ function Main() { const dispatch = useDispatch(window.zutron); const [voiceOn, setVoiceOn] = React.useState(false); + const [transcribing, setTranscribing] = React.useState(false); const mediaRecorderRef = React.useRef(null); const audioChunksRef = React.useRef([]); const [mediaStream, setMediaStream] = React.useState( @@ -56,26 +57,25 @@ function Main() { }; const stt = async (audioBlob: Blob) => { - console.log('stt function called'); const reader = new FileReader(); reader.onloadend = async () => { + setTranscribing(true); const arrayBuffer = reader.result as ArrayBuffer; - console.log('Invoking stt handler in main process'); try { const result = await window.electron.ipcRenderer.invoke( 'stt', arrayBuffer, ); - console.log('Received transcription result:', result); setLocalInstructions(result); } catch (error) { - console.error('Error in stt:', error); toast({ description: `Speech-to-text error: ${error.message}`, status: 'error', duration: 5000, isClosable: true, }); + } finally { + setTranscribing(false); } }; reader.readAsArrayBuffer(audioBlob); @@ -92,30 +92,20 @@ function Main() { mediaRecorderRef.current = mediaRecorder; mediaRecorder.ondataavailable = (event: BlobEvent) => { - console.log( - 'ondataavailable event received, data size:', - event.data.size, - ); if (event.data.size > 0) { audioChunksRef.current.push(event.data); } }; mediaRecorder.onstop = () => { - console.log( - 'mediaRecorder stopped, audioChunks length:', - audioChunksRef.current.length, - ); const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm', }); - console.log('Created audioBlob:', audioBlob); stt(audioBlob); audioChunksRef.current = []; // Reset the array for the next recording }; }) .catch((error) => { - console.error('Error accessing microphone: ', error); toast({ description: `Error accessing microphone: ${error.message}`, status: 'error', @@ -124,7 +114,6 @@ function Main() { }); }); } else { - console.warn('MediaRecorder is not supported in this browser.'); toast({ description: 'MediaRecorder is not supported in this browser.', status: 'warning', @@ -272,6 +261,19 @@ function Main() { border="1px solid" borderColor="blackAlpha.200" onClick={() => { + if ( + !window.electron.process.env.OPENAI_API_KEY || + window.electron.process.env.OPENAI_API_KEY === '' + ) { + toast({ + description: + 'Add OpenAI API key to environment variables to enable voice input.', + status: 'error', + duration: 5000, + isClosable: true, + }); + return; + } setVoiceOn(!voiceOn); }} sx={{ @@ -283,7 +285,11 @@ function Main() { }, }} > - {voiceOn ? 'Recording' : 'Start Recording'} + {voiceOn + ? 'Recording' + : transcribing + ? 'Transcribing...' + : 'Start Recording'} diff --git a/src/renderer/global.d.ts b/src/renderer/global.d.ts index 67229d5..8641176 100644 --- a/src/renderer/global.d.ts +++ b/src/renderer/global.d.ts @@ -1,5 +1,8 @@ interface Window { electron: { + process: { + env: Record; + }; // ... existing definitions ... windowControls: { minimize: () => Promise;