From 400c69860a3a663f0187e73cf3ba9e19bceb5b0e Mon Sep 17 00:00:00 2001 From: Fred Bechtoldt Date: Mon, 28 Apr 2025 12:01:56 +0000 Subject: [PATCH 1/2] feat: add Ollama pkg and set EnvVars in prepare stage --- .env.example | 22 +++++++- .gitignore | 3 ++ ci.yml | 72 +++++++++++++++++++++++++ codesphere-API/createEnvVars.js | 90 ++++++++++++++++++++++++++++++++ codesphere-API/package-lock.json | 80 ++++++++++++++++++++++++++++ codesphere-API/package.json | 7 +++ vite.config.ts | 3 ++ 7 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 ci.yml create mode 100644 codesphere-API/createEnvVars.js create mode 100644 codesphere-API/package-lock.json create mode 100644 codesphere-API/package.json diff --git a/.env.example b/.env.example index c38bf88bfb9..c788abc2fbe 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,11 @@ +# OPEN_WEBUI ENV VARS + # Ollama URL for the backend to connect # The path '/ollama' will be redirected to the specified backend URL -OLLAMA_BASE_URL='http://localhost:11434' +OLLAMA_BASE_URL='https://-3000.2.codesphere.com' # insert your workspace ID + +ENABLE_OLLAMA_API='True' +ENABLE_WEBSOCKET_SUPPORT='True' OPENAI_API_BASE_URL='' OPENAI_API_KEY='' @@ -10,4 +15,17 @@ OPENAI_API_KEY='' # DO NOT TRACK SCARF_NO_ANALYTICS=true DO_NOT_TRACK=true -ANONYMIZED_TELEMETRY=false \ No newline at end of file +ANONYMIZED_TELEMETRY=false + + +# OLLAMA ENV VARS +OLLAMA_HOST='0.0.0.0:3000' +OLLAMA_MODELS='/home/user/app/models' + + +# UV ENV VARS +UV_BIN_DIR='/home/user/app/.codesphere-internal/.uv' +UV_PYTHON_INSTALL_DIR='/home/user/app/.codesphere-internal/.uv/python' + + + diff --git a/.gitignore b/.gitignore index 32271f8087e..1cb4db18d7c 100644 --- a/.gitignore +++ b/.gitignore @@ -307,3 +307,6 @@ dist cypress/videos cypress/screenshots .vscode/settings.json + +# Ollama +models/ diff --git a/ci.yml b/ci.yml new file mode 100644 index 00000000000..bb6a7af802c --- /dev/null +++ b/ci.yml @@ -0,0 +1,72 @@ +schemaVersion: v0.2 +prepare: + steps: + - name: Install Ollama + command: nix-env -iA nixpkgs.ollama + - name: Download llms + command: nohup ollama serve & sleep 5 && ollama pull llama3.2 + - name: Install codesphere API dependencies + command: cd codesphere-API && npm i + - name: Create Env Vars + command: cd codesphere-API && node createEnvVars.js + - name: Install uv and create Python venv + command: cd backend && if [ ! -d ".venv" ]; then nix-env -iA nixpkgs.uv && uv + venv --python 3.11; fi; + - name: Install Python dependencies + command: cd backend && uv pip install -r requirements.txt + - name: Update node + command: sudo n 22.10 + - name: Install node dependencies + command: npm i + - name: Build frontend + command: true || (export NODE_OPTIONS="--max-old-space-size=4096" && npm run + build) +test: + steps: [] +run: + frontend: + steps: + - name: update node + command: sudo n 22.10 && node --version + - name: Start frontend + command: export NODE_OPTIONS="--max-old-space-size=4096" && npm run preview -- + --host 0.0.0.0 --port 3000 + plan: 22 + replicas: 1 + isPublic: true + network: + path: / + stripPath: false + backend: + steps: + - name: Start Backend + command: npx serve & cd backend && source .venv/bin/activate && ./start.sh + plan: 22 + replicas: 1 + isPublic: true + network: + ports: + - port: 3000 + isPublic: true + - port: 8080 + isPublic: true + paths: + - port: 8080 + path: /api + stripPath: false + - port: 8080 + path: /ws + stripPath: false + - port: 8080 + path: /ollama + stripPath: false + - port: 8080 + path: /openai + stripPath: false + ollama: + steps: + - name: Run ollama + command: ollama serve + plan: 22 + replicas: 1 + isPublic: false diff --git a/codesphere-API/createEnvVars.js b/codesphere-API/createEnvVars.js new file mode 100644 index 00000000000..f8b0d0f7422 --- /dev/null +++ b/codesphere-API/createEnvVars.js @@ -0,0 +1,90 @@ +const wsLib = require("ws"); +const fs = require("fs"); +const path = require("path"); +const dotenv = require("dotenv"); +const dotenvExpand = require("dotenv-expand"); + +async function loadEnvFile() { + const domain = process.env.WORKSPACE_DEV_DOMAIN || ""; + let envFile; + let apiDomain; + + if (domain.includes("codesphere.com")) { + envFile = ".env.codesphere"; + apiDomain = "codesphere.com"; + } else { + envFile = ".env.local"; + } + + const envDefaultPath = path.resolve(__dirname, "..", ".env"); + const envPath = path.resolve(__dirname, "..", envFile); + console.log(envPath); + if (fs.existsSync(envPath)) { + dotenvExpand.expand(dotenv.config({ path: envDefaultPath })); + dotenvExpand.expand(dotenv.config({ path: envPath })); + + const envConfigDefault = dotenv.parse(fs.readFileSync(envDefaultPath)); + const envConfigSpecific = dotenv.parse(fs.readFileSync(envPath)); + + // Merge default .env and specific .env. + const envConfig = Object.assign({}, envConfigDefault, envConfigSpecific); + + const config = {}; + for (let [key, value] of Object.entries(envConfig)) { + if (key.includes("DIR") || key.includes("PATH")) { + config[key] = path.resolve(value); + } else { + config[key] = value; + } + } + console.log(`Loaded environment from ${envFile}`); + return { config, apiDomain }; + } else { + console.error(`.env file ${envFile} not found`); + } +} + +async function main() { + try { + let { config, apiDomain } = await loadEnvFile(); + + // All env vars to be set in all type of workspaces + config = { + ...config, + // VITE_WORKSPACE_ID: process.env.WORKSPACE_ID, + // VITE_WORKSPACE_DEV_URL: `https://${process.env.WORKSPACE_DEV_DOMAIN}`, + }; + + let envRequest = []; + for (let [key, value] of Object.entries(config)) { + envRequest.push({ + name: key, + value: value, + }); + } + + // Send request to put env vars + let flowResult = await fetch( + `https://${apiDomain}/api/workspaces/${process.env.WORKSPACE_ID}/env-vars`, + { + method: "PUT", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer " + process.env.INTERNAL_API_KEY, + }, + body: JSON.stringify(envRequest), + } + ); + if (flowResult.status != 200) { + throw new Error(`Failed to update Env Vars. flowResult status: (${flowResult.status})`); + } + + // Exit the process + process.exit(0); + } catch (error) { + console.error("Error:", error); + process.exit(1); + } +} + +main(); \ No newline at end of file diff --git a/codesphere-API/package-lock.json b/codesphere-API/package-lock.json new file mode 100644 index 00000000000..88da61af6e4 --- /dev/null +++ b/codesphere-API/package-lock.json @@ -0,0 +1,80 @@ +{ + "name": "codesphere-API", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "dotenv": "^16.4.5", + "dotenv-expand": "^11.0.6", + "ws": "^8.18.0" + } + }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dotenv-expand": { + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", + "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", + "dependencies": { + "dotenv": "^16.4.5" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + }, + "dependencies": { + "dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==" + }, + "dotenv-expand": { + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", + "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", + "requires": { + "dotenv": "^16.4.5" + } + }, + "ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "requires": {} + } + } +} diff --git a/codesphere-API/package.json b/codesphere-API/package.json new file mode 100644 index 00000000000..87fe3c694bf --- /dev/null +++ b/codesphere-API/package.json @@ -0,0 +1,7 @@ +{ + "dependencies": { + "dotenv": "^16.4.5", + "dotenv-expand": "^11.0.6", + "ws": "^8.18.0" + } +} \ No newline at end of file diff --git a/vite.config.ts b/vite.config.ts index ed690f5023b..270b7f965cf 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -30,6 +30,9 @@ export default defineConfig({ ] }) ], + preview: { + allowedHosts: ['.2.codesphere.com'], + }, define: { APP_VERSION: JSON.stringify(process.env.npm_package_version), APP_BUILD_HASH: JSON.stringify(process.env.APP_BUILD_HASH || 'dev-build') From 9c2119e6f3803ac8a8f369ec75eaa5da12568cbb Mon Sep 17 00:00:00 2001 From: Fred Bechtoldt Date: Mon, 28 Apr 2025 16:49:49 +0000 Subject: [PATCH 2/2] fix: improve create EnvVars --- ci.yml | 12 +++-- codesphere-API/createEnvVars.js | 90 -------------------------------- codesphere-API/package-lock.json | 80 ---------------------------- codesphere-API/package.json | 7 --- 4 files changed, 8 insertions(+), 181 deletions(-) delete mode 100644 codesphere-API/createEnvVars.js delete mode 100644 codesphere-API/package-lock.json delete mode 100644 codesphere-API/package.json diff --git a/ci.yml b/ci.yml index bb6a7af802c..794d1b13dd7 100644 --- a/ci.yml +++ b/ci.yml @@ -1,14 +1,18 @@ schemaVersion: v0.2 prepare: steps: + - name: Install codesphere-cli + command: nix build --extra-experimental-features nix-command + --extra-experimental-features flakes github:datata1/codesphere-cli && + nix-env -i ./result && rm -rf ./result + - name: Copy .env.codesphere to .env when .env not exists + command: cp -n .env.codesphere .env + - name: Set EnvVars + command: codesphere-cli set-env-vars - name: Install Ollama command: nix-env -iA nixpkgs.ollama - name: Download llms command: nohup ollama serve & sleep 5 && ollama pull llama3.2 - - name: Install codesphere API dependencies - command: cd codesphere-API && npm i - - name: Create Env Vars - command: cd codesphere-API && node createEnvVars.js - name: Install uv and create Python venv command: cd backend && if [ ! -d ".venv" ]; then nix-env -iA nixpkgs.uv && uv venv --python 3.11; fi; diff --git a/codesphere-API/createEnvVars.js b/codesphere-API/createEnvVars.js deleted file mode 100644 index f8b0d0f7422..00000000000 --- a/codesphere-API/createEnvVars.js +++ /dev/null @@ -1,90 +0,0 @@ -const wsLib = require("ws"); -const fs = require("fs"); -const path = require("path"); -const dotenv = require("dotenv"); -const dotenvExpand = require("dotenv-expand"); - -async function loadEnvFile() { - const domain = process.env.WORKSPACE_DEV_DOMAIN || ""; - let envFile; - let apiDomain; - - if (domain.includes("codesphere.com")) { - envFile = ".env.codesphere"; - apiDomain = "codesphere.com"; - } else { - envFile = ".env.local"; - } - - const envDefaultPath = path.resolve(__dirname, "..", ".env"); - const envPath = path.resolve(__dirname, "..", envFile); - console.log(envPath); - if (fs.existsSync(envPath)) { - dotenvExpand.expand(dotenv.config({ path: envDefaultPath })); - dotenvExpand.expand(dotenv.config({ path: envPath })); - - const envConfigDefault = dotenv.parse(fs.readFileSync(envDefaultPath)); - const envConfigSpecific = dotenv.parse(fs.readFileSync(envPath)); - - // Merge default .env and specific .env. - const envConfig = Object.assign({}, envConfigDefault, envConfigSpecific); - - const config = {}; - for (let [key, value] of Object.entries(envConfig)) { - if (key.includes("DIR") || key.includes("PATH")) { - config[key] = path.resolve(value); - } else { - config[key] = value; - } - } - console.log(`Loaded environment from ${envFile}`); - return { config, apiDomain }; - } else { - console.error(`.env file ${envFile} not found`); - } -} - -async function main() { - try { - let { config, apiDomain } = await loadEnvFile(); - - // All env vars to be set in all type of workspaces - config = { - ...config, - // VITE_WORKSPACE_ID: process.env.WORKSPACE_ID, - // VITE_WORKSPACE_DEV_URL: `https://${process.env.WORKSPACE_DEV_DOMAIN}`, - }; - - let envRequest = []; - for (let [key, value] of Object.entries(config)) { - envRequest.push({ - name: key, - value: value, - }); - } - - // Send request to put env vars - let flowResult = await fetch( - `https://${apiDomain}/api/workspaces/${process.env.WORKSPACE_ID}/env-vars`, - { - method: "PUT", - headers: { - "Content-Type": "application/json", - Authorization: "Bearer " + process.env.INTERNAL_API_KEY, - }, - body: JSON.stringify(envRequest), - } - ); - if (flowResult.status != 200) { - throw new Error(`Failed to update Env Vars. flowResult status: (${flowResult.status})`); - } - - // Exit the process - process.exit(0); - } catch (error) { - console.error("Error:", error); - process.exit(1); - } -} - -main(); \ No newline at end of file diff --git a/codesphere-API/package-lock.json b/codesphere-API/package-lock.json deleted file mode 100644 index 88da61af6e4..00000000000 --- a/codesphere-API/package-lock.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "name": "codesphere-API", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "dependencies": { - "dotenv": "^16.4.5", - "dotenv-expand": "^11.0.6", - "ws": "^8.18.0" - } - }, - "node_modules/dotenv": { - "version": "16.5.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", - "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dotenv-expand": { - "version": "11.0.7", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", - "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", - "dependencies": { - "dotenv": "^16.4.5" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/ws": { - "version": "8.18.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", - "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - } - }, - "dependencies": { - "dotenv": { - "version": "16.5.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", - "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==" - }, - "dotenv-expand": { - "version": "11.0.7", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", - "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", - "requires": { - "dotenv": "^16.4.5" - } - }, - "ws": { - "version": "8.18.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", - "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", - "requires": {} - } - } -} diff --git a/codesphere-API/package.json b/codesphere-API/package.json deleted file mode 100644 index 87fe3c694bf..00000000000 --- a/codesphere-API/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "dependencies": { - "dotenv": "^16.4.5", - "dotenv-expand": "^11.0.6", - "ws": "^8.18.0" - } -} \ No newline at end of file