From 8653b482d5b2de134f496c7a254cff6e1d88e48a Mon Sep 17 00:00:00 2001 From: erangi-ar Date: Tue, 14 Oct 2025 19:05:21 +0530 Subject: [PATCH 1/4] gui running on 3003 issue fixed --- GUI/.env.development | 5 +- GUI/src/services/llmConnections.ts | 1 + docker-compose.yml | 74 +++++++++++++++--------------- 3 files changed, 39 insertions(+), 41 deletions(-) diff --git a/GUI/.env.development b/GUI/.env.development index 7ff4d8b..39f5e47 100644 --- a/GUI/.env.development +++ b/GUI/.env.development @@ -1,8 +1,7 @@ REACT_APP_RUUTER_API_URL=http://localhost:8086 REACT_APP_RUUTER_PRIVATE_API_URL=http://localhost:8088 -REACT_APP_EXTERNAL_API_URL=http://localhost:8000 REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004/et/dev-auth REACT_APP_SERVICE_ID=conversations,settings,monitoring -REACT_APP_NOTIFICATION_NODE_URL=http://localhost:4040 -REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost:8086 http://localhost:8088 http://localhost:8085 http://localhost:4040; +REACT_APP_NOTIFICATION_NODE_URL=http://localhost:3005 +REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost:8086 http://localhost:8088 http://localhost:3004 http://localhost:3005 ws://localhost; REACT_APP_ENABLE_HIDDEN_FEATURES=TRUE \ No newline at end of file diff --git a/GUI/src/services/llmConnections.ts b/GUI/src/services/llmConnections.ts index b385aaf..96d37e9 100644 --- a/GUI/src/services/llmConnections.ts +++ b/GUI/src/services/llmConnections.ts @@ -197,6 +197,7 @@ export async function checkBudgetStatus(): Promise { // Return null if no production connection found (404) or other errors return null; } +} export async function updateLLMConnectionStatus( id: string | number, diff --git a/docker-compose.yml b/docker-compose.yml index c61e6c9..b3a8473 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,12 +140,11 @@ services: container_name: gui environment: - NODE_ENV=development - - REACT_APP_RUUTER_API_URL=http://localhost/ruuter-public - - REACT_APP_RUUTER_PRIVATE_API_URL=http://localhost/ruuter-private - - REACT_APP_EXTERNAL_API_URL=http://localhost/dataset-gen-service - - REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost/authentication-layer/et/dev-auth - - REACT_APP_NOTIFICATION_NODE_URL=http://localhost/notifications-node - - REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost ws://localhost; + - REACT_APP_RUUTER_API_URL=http://localhost:8086 + - REACT_APP_RUUTER_PRIVATE_API_URL=http://localhost:8088 + - REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004 + - REACT_APP_NOTIFICATION_NODE_URL=http://localhost:3005 + - REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost:8086 http://localhost:8088 http://localhost:3004 http://localhost:3005 ws://localhost; - DEBUG_ENABLED=true - CHOKIDAR_USEPOLLING=true - PORT=3001 @@ -447,39 +446,38 @@ services: - bykstack restart: unless-stopped - # LLM Orchestration Service - # llm-orchestration-service: - # build: - # context: . - # dockerfile: Dockerfile.llm_orchestration_service - # container_name: llm-orchestration-service - # restart: always - # ports: - # - "8100:8100" - # env_file: - # - .env - # environment: - # - ENVIRONMENT=production - # - VAULT_ADDR=http://vault:8200 - # - VAULT_TOKEN=/agent/out/token - # volumes: - # # Mount configuration files - # - ./src/llm_config_module/config:/app/src/llm_config_module/config:ro - # # Mount logs directory for persistence - # - llm_orchestration_logs:/app/logs - # - ./vault/agent-out:/agent/out:ro - # networks: - # - bykstack - # depends_on: - # - vault - # - vault-agent-llm - # healthcheck: - # test: ["CMD", "curl", "-f", "http://llm-orchestration-service:8100/health"] - # interval: 30s - # timeout: 10s - # start_period: 40s - # retries: 3 + llm-orchestration-service: + build: + context: . + dockerfile: Dockerfile.llm_orchestration_service + container_name: llm-orchestration-service + restart: always + ports: + - "8100:8100" + env_file: + - .env + environment: + - ENVIRONMENT=production + - VAULT_ADDR=http://vault:8200 + - VAULT_TOKEN=/agent/out/token + volumes: + # Mount configuration files + - ./src/llm_config_module/config:/app/src/llm_config_module/config:ro + # Mount logs directory for persistence + - llm_orchestration_logs:/app/logs + - ./vault/agent-out:/agent/out:ro + networks: + - bykstack + depends_on: + - vault + - vault-agent-llm + healthcheck: + test: ["CMD", "curl", "-f", "http://llm-orchestration-service:8100/health"] + interval: 30s + timeout: 10s + start_period: 40s + retries: 3 volumes: loki-data: From 4f9dbc59cc83d5115eab89b283408ba7583558ae Mon Sep 17 00:00:00 2001 From: erangi-ar Date: Wed, 15 Oct 2025 10:07:17 +0530 Subject: [PATCH 2/4] Add SQL query to retrieve configuration values --- DSL/Resql/rag-search/POST/get-configuration.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 DSL/Resql/rag-search/POST/get-configuration.sql diff --git a/DSL/Resql/rag-search/POST/get-configuration.sql b/DSL/Resql/rag-search/POST/get-configuration.sql new file mode 100644 index 0000000..f03b322 --- /dev/null +++ b/DSL/Resql/rag-search/POST/get-configuration.sql @@ -0,0 +1,5 @@ +SELECT id, key, value +FROM configuration +WHERE key=:key +AND id IN (SELECT max(id) from configuration GROUP BY key) +AND NOT deleted; From c519a2e49482ef1ea812a789f381702c3284b8c2 Mon Sep 17 00:00:00 2001 From: erangi-ar Date: Thu, 16 Oct 2025 10:40:33 +0530 Subject: [PATCH 3/4] Refactor inference and connection handling in YAML and TypeScript files --- .../rag-search/POST/inference/production.yml | 7 ++++++- .../rag-search/POST/inference/test.yml | 14 ++++++++++---- .../rag-search/POST/llm-connections/add.yml | 11 ++++++++++- GUI/src/services/inference.ts | 4 ++-- GUI/src/utils/endpoints.ts | 2 +- docker-compose.yml | 5 ++--- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/DSL/Ruuter.private/rag-search/POST/inference/production.yml b/DSL/Ruuter.private/rag-search/POST/inference/production.yml index e9b90ee..5b13570 100644 --- a/DSL/Ruuter.private/rag-search/POST/inference/production.yml +++ b/DSL/Ruuter.private/rag-search/POST/inference/production.yml @@ -85,10 +85,15 @@ call_orchestrate_endpoint: headers: Content-Type: "application/json" result: orchestrate_result + next: assign_response + +assign_response: + assign: + response: "${orchestrate_result.response.body}" next: return_orchestrate_response return_orchestrate_response: - return: ${orchestrate_result.response.body} + return: ${response} next: end assign_disconnected_response: diff --git a/DSL/Ruuter.private/rag-search/POST/inference/test.yml b/DSL/Ruuter.private/rag-search/POST/inference/test.yml index 28f346a..61a5bd9 100644 --- a/DSL/Ruuter.private/rag-search/POST/inference/test.yml +++ b/DSL/Ruuter.private/rag-search/POST/inference/test.yml @@ -24,8 +24,9 @@ extract_request_data: get_connection: call: http.post args: - url: "[#RAG_SEARCH_RESQL]/get-production-connection" - body: {} + url: "[#RAG_SEARCH_RESQL]/get-llm-connection" + body: + connection_id: ${connectionId} result: connection_result next: validate_connection @@ -57,7 +58,7 @@ validate_budget_status: call_orchestrate_endpoint: call: http.post args: - url: "[#RAG_SEARCH_LLM_ORCHESTRATOR]" + url: "[#RAG_SEARCH_LLM_ORCHESTRATOR]/test" body: connectionId: ${connectionId} message: ${message} @@ -65,10 +66,15 @@ call_orchestrate_endpoint: headers: Content-Type: "application/json" result: orchestrate_result + next: assign_response + +assign_response: + assign: + response: "${orchestrate_result.response.body}" next: return_orchestrate_response return_orchestrate_response: - return: ${orchestrate_result.response.body} + return: ${response} next: end assign_disconnected_response: diff --git a/DSL/Ruuter.private/rag-search/POST/llm-connections/add.yml b/DSL/Ruuter.private/rag-search/POST/llm-connections/add.yml index 78dbeea..dffe487 100644 --- a/DSL/Ruuter.private/rag-search/POST/llm-connections/add.yml +++ b/DSL/Ruuter.private/rag-search/POST/llm-connections/add.yml @@ -140,10 +140,19 @@ add_llm_connection: access_key: ${access_key} embedding_model_api_key: ${embedding_model_api_key} result: connection_result + next: assign_connection_response + +assign_connection_response: + assign: + response: { + id: "${connection_result.response.body[0].id}", + status: 201, + operationSuccess: true + } next: return_success return_success: - return: "LLM connection added successfully" + return: ${response} status: 200 next: end diff --git a/GUI/src/services/inference.ts b/GUI/src/services/inference.ts index f469251..28de0e7 100644 --- a/GUI/src/services/inference.ts +++ b/GUI/src/services/inference.ts @@ -16,8 +16,8 @@ export interface InferenceResponse { } export async function viewInferenceResult(request: InferenceRequest): Promise { - const { data } = await apiDev.post(inferenceEndpoints.VIEW_INFERENCE_RESULT(), { - llmConnectionId: request.llmConnectionId, + const { data } = await apiDev.post(inferenceEndpoints.VIEW_TEST_INFERENCE_RESULT(), { + connectionId: request.llmConnectionId, message: request.message, }); return data; diff --git a/GUI/src/utils/endpoints.ts b/GUI/src/utils/endpoints.ts index 1eb98b9..37d8173 100644 --- a/GUI/src/utils/endpoints.ts +++ b/GUI/src/utils/endpoints.ts @@ -25,5 +25,5 @@ export const llmConnectionsEndpoints = { } export const inferenceEndpoints = { - VIEW_INFERENCE_RESULT: (): string => `/rag-search/inference/results/view`, + VIEW_TEST_INFERENCE_RESULT: (): string => `/rag-search/inference/test`, } diff --git a/docker-compose.yml b/docker-compose.yml index b3a8473..ffdff7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -142,8 +142,7 @@ services: - NODE_ENV=development - REACT_APP_RUUTER_API_URL=http://localhost:8086 - REACT_APP_RUUTER_PRIVATE_API_URL=http://localhost:8088 - - REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004 - - REACT_APP_NOTIFICATION_NODE_URL=http://localhost:3005 + - REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004/et/dev-auth - REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost:8086 http://localhost:8088 http://localhost:3004 http://localhost:3005 ws://localhost; - DEBUG_ENABLED=true - CHOKIDAR_USEPOLLING=true @@ -420,7 +419,7 @@ services: - ./vault/config:/vault/config # contains vault.hcl - ./vault/logs:/vault/logs expose: - - "8200" + - "8200" networks: - bykstack restart: unless-stopped From e5b52b2814cc7bd7fcf11751e0ee032bab6c15af Mon Sep 17 00:00:00 2001 From: erangi-ar Date: Thu, 16 Oct 2025 10:46:57 +0530 Subject: [PATCH 4/4] Add entry point script for Vector Indexer with command line interface --- run_vector_indexer.py | 179 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 run_vector_indexer.py diff --git a/run_vector_indexer.py b/run_vector_indexer.py new file mode 100644 index 0000000..b01150f --- /dev/null +++ b/run_vector_indexer.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +Entry point script for Vector Indexer - Contextual Retrieval Pipeline + +This script can be run directly or called by cron jobs for automated processing. + +Usage: + python run_vector_indexer.py [--config CONFIG_PATH] [--health-check] [--dry-run] + +Examples: + # Run with default config + python run_vector_indexer.py + + # Run with custom config + python run_vector_indexer.py --config /path/to/config.yaml + + # Health check only + python run_vector_indexer.py --health-check + + # Dry run (validate without processing) + python run_vector_indexer.py --dry-run +""" + +import argparse +import asyncio +import sys +from pathlib import Path + +# Add src to Python path +sys.path.insert(0, str(Path(__file__).parent / "src")) + +from src.vector_indexer.main_indexer import VectorIndexer + + +async def main(): + """Main entry point with command line argument parsing.""" + + parser = argparse.ArgumentParser( + description="Vector Indexer - Contextual Retrieval Pipeline", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + parser.add_argument( + "--config", + type=str, + default="src/vector_indexer/config/vector_indexer_config.yaml", + help="Path to configuration file (default: src/vector_indexer/config/vector_indexer_config.yaml)", + ) + + parser.add_argument( + "--health-check", action="store_true", help="Run health check only and exit" + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate configuration and connectivity without processing documents", + ) + + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress non-error output" + ) + + args = parser.parse_args() + + # Configure logging level based on arguments + log_level = "INFO" + if args.verbose: + log_level = "DEBUG" + elif args.quiet: + log_level = "ERROR" + + try: + # Initialize vector indexer with specified config + indexer = VectorIndexer(config_path=args.config) + + if args.health_check: + # Health check only + print("๐Ÿ” Running health check...") + health_ok = await indexer.run_health_check() + + if health_ok: + print("โœ… Health check passed!") + return 0 + else: + print("โŒ Health check failed!") + return 1 + + elif args.dry_run: + # Dry run - validate without processing + print("๐Ÿงช Running dry run validation...") + + health_ok = await indexer.run_health_check() + if not health_ok: + print("โŒ Validation failed!") + return 1 + + # Discover documents but don't process + documents = indexer.document_loader.discover_all_documents() + print(f"๐Ÿ“„ Found {len(documents)} documents ready for processing") + print("โœ… Dry run validation passed!") + return 0 + + else: + # Full processing run + print("๐Ÿš€ Starting Vector Indexer processing...") + + # Health check first + health_ok = await indexer.run_health_check() + if not health_ok: + print("โŒ Pre-processing health check failed!") + return 1 + + # Process all documents + stats = await indexer.process_all_documents() + + # Return appropriate exit code + if stats.documents_failed > 0: + print(f"โš ๏ธ Processing completed with {stats.documents_failed} failures") + return 2 # Partial success + else: + print("โœ… Processing completed successfully!") + return 0 + + except KeyboardInterrupt: + print("\nโน๏ธ Processing interrupted by user") + return 130 + except FileNotFoundError as e: + print(f"โŒ Configuration file not found: {e}") + return 1 + except Exception as e: + print(f"๐Ÿ’ฅ Fatal error: {e}") + return 1 + + +def cron_entry_point(): + """ + Entry point specifically designed for cron jobs. + + This function: + - Uses minimal output suitable for cron logs + - Returns appropriate exit codes for monitoring + - Handles errors gracefully for automated systems + """ + import logging + + # Configure minimal logging for cron + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - Vector Indexer - %(levelname)s - %(message)s", + ) + + try: + # Run with default configuration + result = asyncio.run(main()) + + if result == 0: + logging.info("Vector indexer completed successfully") + elif result == 2: + logging.warning("Vector indexer completed with some failures") + else: + logging.error("Vector indexer failed") + + return result + + except Exception as e: + logging.error(f"Vector indexer fatal error: {e}") + return 1 + + +if __name__ == "__main__": + # Run the async main function + exit_code = asyncio.run(main()) + sys.exit(exit_code)