From 4fcb8bab177381bd3666acd616cdc353843814b4 Mon Sep 17 00:00:00 2001 From: Mikita Tsybulka Date: Fri, 5 Sep 2025 20:02:38 +0200 Subject: [PATCH 1/4] [create] makefile + [fixed] docker-compose due to .sh config --- .gitignore | 6 +- Makefile | 128 ++++++++++++++++++++++++++++++ README.md | 164 ++++++++++++++++++++------------------- docker-compose.dev.yaml | 6 +- docker-compose.ghcr.yaml | 6 +- docker-compose.prod.yaml | 7 +- env.sh.example | 18 +++++ 7 files changed, 242 insertions(+), 93 deletions(-) create mode 100644 Makefile create mode 100644 env.sh.example diff --git a/.gitignore b/.gitignore index 9a7543d02..1f71da641 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ venv/ .env *.env +*env.sh # Django specific db.sqlite3 @@ -17,10 +18,9 @@ db.sqlite3 *.bak *.dump -# Migrations (опционально, если не хочешь версионировать) +# Migrations **/migrations/__pycache__/ **/migrations/*.pyc -# **/migrations/ можно добавить, если генерируешь их автоматически # IDE .idea/ @@ -33,7 +33,7 @@ htmlcov/ pytest_cache/ .cache/ -# Static/media files (если генерируются) +# Static/media files staticfiles/ media/ static/ diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..9fce8d707 --- /dev/null +++ b/Makefile @@ -0,0 +1,128 @@ + +LOGICWAY_DIR = logicway +ROUTE_ENGINE_DIR = route_engine +FRONTEND_DIR = frontend + + +configure: + . ./env.sh + +#==================================================================== +# ==== DEVELOPMENT ==== +#==================================================================== + +run-logicway: + poetry run python $(LOGICWAY_DIR)/manage.py runserver 8000 + +run-route-engine: + poetry run python $(ROUTE_ENGINE_DIR)/manage.py runserver 8001 + +run-frontend: + cd $(FRONTEND_DIR) && npm start + +tests: + poetry run pytest $(LOGICWAY_DIR) $(ROUTE_ENGINE_DIR) + +create-key: + python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())" + @echo "> Use the generated key in .sh as SECRET_KEY value" + +#==================================================================== +# ==== DOCKER COMPOSE ==== +#==================================================================== + +start-docker-dev: configure + docker compose -f docker-compose.dev.yaml up -d + +stop-docker-dev: configure + docker compose -f docker-compose.dev.yaml stop + +rm-docker-dev: configure + docker compose -f docker-compose.dev.yaml down --remove-orphans + +rm-docker-all-dev: configure + docker compose -f docker-compose.dev.yaml down \ + --rmi all --volumes --remove-orphans + +run-docker-jobs-dev: configure + docker compose -f docker-compose.dev.yaml run logicway \ + sh -c "INTERNAL=1 poetry run python database/upload_data.py \ + && poetry run python database/load_data.py" + + +start-docker-prod: configure + docker compose -f docker-compose.prod.yaml up -d + +stop-docker-prod: configure + docker compose -f docker-compose.prod.yaml stop + +rm-docker-prod: configure + docker compose -f docker-compose.prod.yaml down --remove-orphans + +rm-docker-all-prod: configure + docker compose -f docker-compose.prod.yaml down \ + --rmi all --volumes --remove-orphans + +run-docker-jobs-prod: configure + docker compose -f docker-compose.prod.yaml run logicway \ + sh -c "INTERNAL=1 poetry run python database/upload_data.py \ + && poetry run python database/load_data.py" + + +start-docker-ghcr: configure + docker compose -f docker-compose.ghcr.yaml up -d + +stop-docker-ghcr: configure + docker compose -f docker-compose.ghcr.yaml stop + +rm-docker-ghcr: configure + docker compose -f docker-compose.ghcr.yaml down --remove-orphans + +rm-docker-all-ghcr: configure + docker compose -f docker-compose.ghcr.yaml down \ + --rmi all --volumes --remove-orphans + +run-docker-jobs-ghcr: configure + docker compose -f docker-compose.ghcr.yaml run logicway \ + sh -c "INTERNAL=1 poetry run python database/upload_data.py \ + && poetry run python database/load_data.py" + +#==================================================================== +# ==== INFO ==== +#==================================================================== + +help: + @echo "Makefile commands:" + @echo "========== DEVELOPMENT ==========" + @echo " run-logicway - Run LogicWay backend server" + @echo " run-route-engine - Run Route Engine backend server" + @echo " run-frontend - Run Frontend server" + @echo " tests - Run tests for backend servers" + @echo " create-key - Create a new Django SECRET_KEY" + @echo "========== DOCKER ==========" + @echo "---- Development(with build) ----" + @echo " start-docker-dev - Start Docker containers (development)" + @echo " stop-docker-dev - Stop Docker containers (development)" + @echo " rm-docker-dev - Remove Docker containers (development)" + @echo " rm-docker-all-dev - Remove Docker containers, images, volumes (development)" + @echo " run-docker-jobs-dev - Run data upload and load data jobs in Docker (development)" + @echo "---- Production(with build) ----" + @echo " start-docker-prod - Start Docker containers (production)" + @echo " stop-docker-prod - Stop Docker containers (production)" + @echo " rm-docker-prod - Remove Docker containers (production)" + @echo " rm-docker-all-prod - Remove Docker containers, images, volumes (production)" + @echo " run-docker-jobs-prod - Run data upload and load data jobs in Docker (production)" + @echo "---- GitHub Container Registry ----" + @echo " start-docker-ghcr - Start Docker containers (GitHub Container Registry)" + @echo " stop-docker-ghcr - Stop Docker containers (GitHub Container Registry)" + @echo " rm-docker-ghcr - Remove Docker containers (GitHub Container Registry)" + @echo " rm-docker-all-ghcr - Remove Docker containers, images, volumes (GitHub Container Registry)" + @echo " run-docker-jobs-ghcr - Run data upload and load data jobs in Docker (GitHub Container Registry)" + @echo "========== INTERNAL ==========" + @echo " configure - Configure environment variables from env.sh" + +.PHONY: run-logicway run-route-engine run-frontend \ + run-tests configure create-key \ + start-docker-dev stop-docker-dev rm-docker-dev rm-docker-all-dev \ + start-docker-prod stop-docker-prod rm-docker-prod rm-docker-all-prod \ + start-docker-ghcr stop-docker-ghcr rm-docker-ghcr rm-docker-all-ghcr diff --git a/README.md b/README.md index f1a3ae56a..f1ee0c1bc 100644 --- a/README.md +++ b/README.md @@ -5,89 +5,97 @@ https://www.jetbrains.com/help/pycharm/poetry.html ### Install Dependencies - To install all dependencies: `poetry install` -- Install only for logicway service: `poetry install --with logicway` -- Install only for route_engine service: `poetry install --with route_engine` -#### To create new SECRET_KEY for Django +### Poetry Dependency Groups -``` bash -python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' -``` -### Create .sh with own vars - -``` -# env.sh +This project uses Poetry dependency groups to organize different types of dependencies: -export SECRET_KEY="django-insecure-genkey" -export ROUTE_ENGINE_SECRET_KEY="django-insecure-genkey" +- **Main dependencies**: Core packages needed for all services (Django, envsh, etc.) +- **logicway**: Dependencies specific to LogicWay service (SQLAlchemy, Selenium, pandas, etc.) +- **route_engine**: Dependencies for Route Engine service (routingpy, geopy) +- **dev**: Development and testing dependencies (pytest, ruff linter, etc.) -# Route Engine URL -export LOGICWAY_URL="http://localhost:8000" -export ROUTE_ENGINE_URL="http://localhost:8001" +Install specific groups: +```bash +# Install all dependencies (default) +poetry install -# PostgreSQL Database Settings -export DB_NAME="logic_way_db" -export DB_USER="myuser" -export DB_PASSWORD="mypassword" -export DB_HOST="localhost" -export DB_PORT="5432" +# Install main + specific groups +poetry install --with=logicway,route_engine -export REACT_APP_LOGICWAY_URL="$LOGICWAY_URL" -export REACT_APP_ROUTE_ENGINE_URL="$LOGICWAY_URL/routing/proxy_route_engine" +# Install only dev dependencies +poetry install --only=dev ``` -### Development and deployment with Docker Compose - -#### Development compose commands - -- Build and run the containers in detached mode: `docker compose -f docker-compose.dev.yaml up -d` -- Just stop the containers: `docker compose -f docker-compose.dev.yaml stop` -- Stop and remove the containers: `docker compose -f docker-compose.dev.yaml down --remove-orphans` -- If you are deleting all containers and images, you can use the following command: - ` - docker compose -f docker-compose.dev.yaml down --rmi all --volumes --remove-orphans - ` - -#### Production compose commands - -- Build and run the containers in detached mode: `docker compose -f docker-compose.prod.yaml up -d` -- Just stop the containers: `docker compose -f docker-compose.prod.yaml stop` -- Stop and remove the containers: `docker compose -f docker-compose.prod.yaml down --remove-orphans` -- If you are deleting all containers and images, you can use the following command: -` -docker compose -f docker-compose.prod.yaml down --rmi all --volumes --remove-orphans -` - -#### Production compose (images form GitHub Container Registry) - -- Build and run the containers in detached mode: `docker compose -f docker-compose.ghcr.yaml up -d` -- Just stop the containers: `docker compose -f docker-compose.ghcr.yaml stop` -- Stop and remove the containers: `docker compose -f docker-compose.ghcr.yaml down --remove-orphans` -- If you are deleting all containers and images, you can use the following command: - ` - docker compose -f docker-compose.ghcr.yaml down --rmi all --volumes --remove-orphans - ` - -#### Jobs - -Important: Ensure the database is properly initialized. -- Run loading data job: -` -docker compose -f docker-compose.dev.yaml run logicway sh -c "INTERNAL=1 poetry run python database/upload_data.py && poetry run python database/load_data.py" -` -- if production: -` -docker compose -f docker-compose.prod.yaml run logicway sh -c "INTERNAL=1 poetry run python database/upload_data.py && poetry run python database/load_data.py" -` -- if production with GitHub Container Registry: -` -docker compose -f docker-compose.ghcr.yaml run logicway sh -c "INTERNAL=1 poetry run python database/upload_data.py && poetry run python database/load_data.py" -` - -### Production preparation and Deployment - -Generate requirements.txt files for each service: -``` bash -poetry export --with logicway --without-hashes -f requirements.txt -o logicway/requirements.txt -poetry export --with route_engine --without-hashes -f requirements.txt -o route_engine/requirements.txt -``` +### Make Commands + +This project uses a Makefile for common development and deployment tasks: + +#### Development Commands +- `make run-logicway` - Run LogicWay backend server on port 8000 +- `make run-route-engine` - Run Route Engine backend server on port 8001 +- `make run-frontend` - Run React frontend development server +- `make tests` - Run pytest tests for backend services +- `make create-key` - Generate a new Django SECRET_KEY for env.sh + +#### Docker Development +- `make start-docker-dev` - Start all services with docker-compose (development) +- `make stop-docker-dev` - Stop all Docker services (development) +- `make rm-docker-dev` - Remove Docker containers (development) +- `make rm-docker-all-dev` - Remove containers, images, volumes (development) +- `make run-docker-jobs-dev` - Run data upload/load jobs in Docker (development) + +#### Docker Production +- `make start-docker-prod` - Start all services (production) +- `make stop-docker-prod` - Stop all services (production) +- `make rm-docker-prod` - Remove containers (production) +- `make rm-docker-all-prod` - Remove containers, images, volumes (production) +- `make run-docker-jobs-prod` - Run data jobs (production) + +#### Docker GitHub Container Registry +- `make start-docker-ghcr` - Start services using GHCR images +- `make stop-docker-ghcr` - Stop GHCR services +- `make rm-docker-ghcr` - Remove GHCR containers +- `make rm-docker-all-ghcr` - Remove GHCR containers, images, volumes +- `make run-docker-jobs-ghcr` - Run data jobs with GHCR images + +#### Other Commands +- `make help` - Show all available commands with descriptions +- `make configure` - Load environment variables from env.sh + +**Note**: Most Docker commands automatically run `make configure` to load env.sh variables. + +### Quick Start + +1. **Setup environment**: + ```bash + # Create your env.sh file (see template below) + cp env.sh.example env.sh + # Edit env.sh with your settings + + # Install dependencies + poetry install + ``` + +2. **Development (local)**: + ```bash + # Start all services locally + make run-logicway # Terminal 1 + make run-route-engine # Terminal 2 + make run-frontend # Terminal 3 + + # Or run tests + make tests + ``` + +3. **Development (Docker)**: + ```bash + # Start all services in Docker + make start-docker-dev + + # Run data import jobs + make run-docker-jobs-dev + + # Stop services + make stop-docker-dev + ``` diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 0f6f09529..d408adf8b 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -21,9 +21,8 @@ services: build: context: . dockerfile: route_engine/Dockerfile - env_file: - - .env environment: + ROUTE_ENGINE_SECRET_KEY: ${ROUTE_ENGINE_SECRET_KEY} DJANGO_SETTINGS_MODULE: route_engine.settings.dev ports: - "8001:8001" @@ -34,14 +33,13 @@ services: build: context: . dockerfile: logicway/Dockerfile - env_file: - - .env ports: - "8000:8000" depends_on: - db - route-engine environment: + SECRET_KEY: ${SECRET_KEY} DJANGO_SETTINGS_MODULE: logicway.settings.dev ROUTE_ENGINE_URL: http://route-engine:8001 DB_NAME: ${DB_NAME:-logicway} diff --git a/docker-compose.ghcr.yaml b/docker-compose.ghcr.yaml index f3a0ed952..048defa44 100644 --- a/docker-compose.ghcr.yaml +++ b/docker-compose.ghcr.yaml @@ -21,9 +21,8 @@ services: route-engine: restart: always image: ghcr.io/logicwayteam/route-engine:latest - env_file: - - .env environment: + ROUTE_ENGINE_SECRET_KEY: ${ROUTE_ENGINE_SECRET_KEY} DJANGO_SETTINGS_MODULE: route_engine.settings.prod ports: - "8001:8001" @@ -33,14 +32,13 @@ services: logicway: image: ghcr.io/logicwayteam/logicway:latest restart: always - env_file: - - .env ports: - "8000:8000" depends_on: - db - route-engine environment: + SECRET_KEY: ${SECRET_KEY} DJANGO_SETTINGS_MODULE: logicway.settings.prod ROUTE_ENGINE_URL: http://route-engine:8001 DB_NAME: ${DB_NAME:-logicway} diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 9de9b9b80..ce8e03849 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -21,10 +21,10 @@ services: build: context: . dockerfile: route_engine/Dockerfile - env_file: - - .env ports: - "8001:8001" + environment: + ROUTE_ENGINE_SECRET_KEY: ${ROUTE_ENGINE_SECRET_KEY} networks: - logicnet @@ -32,14 +32,13 @@ services: build: context: . dockerfile: logicway/Dockerfile - env_file: - - .env ports: - "8000:8000" depends_on: - db - route-engine environment: + SECRET_KEY: ${SECRET_KEY} DJANGO_SETTINGS_MODULE: logicway.settings.prod ROUTE_ENGINE_URL: http://route-engine:8001 DB_NAME: ${DB_NAME:-logicway} diff --git a/env.sh.example b/env.sh.example new file mode 100644 index 000000000..39b5e3ce0 --- /dev/null +++ b/env.sh.example @@ -0,0 +1,18 @@ +# env.sh + +export SECRET_KEY="django-insecure-genkey" +export ROUTE_ENGINE_SECRET_KEY="django-insecure-genkey" + +# Route Engine URL +export LOGICWAY_URL="http://localhost:8000" +export ROUTE_ENGINE_URL="http://localhost:8001" + +# PostgreSQL Database Settings +export DB_NAME="logic_way_db" +export DB_USER="myuser" +export DB_PASSWORD="mypassword" +export DB_HOST="localhost" +export DB_PORT="5432" + +export REACT_APP_LOGICWAY_URL="$LOGICWAY_URL" +export REACT_APP_ROUTE_ENGINE_URL="$LOGICWAY_URL/routing/proxy_route_engine" From 35b41b82bc14e41311c40d913dc2b2a817456864 Mon Sep 17 00:00:00 2001 From: Mikita Tsybulka Date: Fri, 5 Sep 2025 20:03:20 +0200 Subject: [PATCH 2/4] [added] k3s route-engine tunnel in test env --- k8s/test-env/services/route-engine-service.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/test-env/services/route-engine-service.yaml b/k8s/test-env/services/route-engine-service.yaml index b7077526c..81a570b7c 100644 --- a/k8s/test-env/services/route-engine-service.yaml +++ b/k8s/test-env/services/route-engine-service.yaml @@ -5,6 +5,9 @@ metadata: labels: app: route-engine namespace: test + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "route-engine-test" spec: selector: app: route-engine From 8c518d7a7b56be87e9edcf5734f173953cfaee6b Mon Sep 17 00:00:00 2001 From: Mikita Tsybulka Date: Fri, 5 Sep 2025 20:24:19 +0200 Subject: [PATCH 3/4] [added] specifing envsh --- LICENSE | 2 +- Makefile | 5 ++++- http-client.env.json | 6 ------ 3 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 http-client.env.json diff --git a/LICENSE b/LICENSE index ca35b537a..9538fb937 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2024 LogicWayTeam + Copyright 2024-2025 LogicWayTeam Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile index 9fce8d707..a901bc4c9 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,10 @@ LOGICWAY_DIR = logicway ROUTE_ENGINE_DIR = route_engine FRONTEND_DIR = frontend +ENVSH ?= env.sh configure: - . ./env.sh + . ./${ENVSH} #==================================================================== # ==== DEVELOPMENT ==== @@ -120,6 +121,8 @@ help: @echo " run-docker-jobs-ghcr - Run data upload and load data jobs in Docker (GitHub Container Registry)" @echo "========== INTERNAL ==========" @echo " configure - Configure environment variables from env.sh" + @echo "========== VARIABLES ==========" + @echo " ENVSH=my_env.sh - Specify the environment file to source (default: env.sh)" .PHONY: run-logicway run-route-engine run-frontend \ run-tests configure create-key \ diff --git a/http-client.env.json b/http-client.env.json deleted file mode 100644 index 91c5c58ac..000000000 --- a/http-client.env.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "logicway-params": { - "ip": "localhost", - "port": 8000 - } -} \ No newline at end of file From 1139f1c8a8030e41e863ae2c0c345bf563645121 Mon Sep 17 00:00:00 2001 From: Mikita Tsybulka Date: Fri, 5 Sep 2025 20:42:19 +0200 Subject: [PATCH 4/4] [changed] source envsh in makefile for frontend --- Makefile | 2 +- frontend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a901bc4c9..d2f57118e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ run-logicway: run-route-engine: poetry run python $(ROUTE_ENGINE_DIR)/manage.py runserver 8001 -run-frontend: +run-frontend: configure cd $(FRONTEND_DIR) && npm start tests: diff --git a/frontend/package.json b/frontend/package.json index 54be0f973..4cd6984ab 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,7 +22,7 @@ "web-vitals": "^2.1.4" }, "scripts": { - "start": ". ../env.sh && react-scripts start", + "start": "react-scripts start", "build": "react-scripts build", "copy": "copyfiles -u 1 \"build/**/*\" \"../map/static/map/frontend\"", "clean": "rimraf ../map/static/map/frontend",