Skip to content
Merged

fix #53

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion DSL/Ruuter.private/rag-search/POST/inference/production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,15 @@ call_orchestrate_endpoint:
headers:
Content-Type: "application/json"
result: orchestrate_result
next: assign_response

assign_response:
assign:
response: "${orchestrate_result.response.body}"
next: return_orchestrate_response

return_orchestrate_response:
return: ${orchestrate_result.response.body}
return: ${response}
next: end

assign_disconnected_response:
Expand Down
14 changes: 10 additions & 4 deletions DSL/Ruuter.private/rag-search/POST/inference/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ extract_request_data:
get_connection:
call: http.post
args:
url: "[#RAG_SEARCH_RESQL]/get-production-connection"
body: {}
url: "[#RAG_SEARCH_RESQL]/get-llm-connection"
body:
connection_id: ${connectionId}
result: connection_result
next: validate_connection

Expand Down Expand Up @@ -57,18 +58,23 @@ validate_budget_status:
call_orchestrate_endpoint:
call: http.post
args:
url: "[#RAG_SEARCH_LLM_ORCHESTRATOR]"
url: "[#RAG_SEARCH_LLM_ORCHESTRATOR]/test"
body:
connectionId: ${connectionId}
message: ${message}
environment: "test"
headers:
Content-Type: "application/json"
result: orchestrate_result
next: assign_response

assign_response:
assign:
response: "${orchestrate_result.response.body}"
next: return_orchestrate_response

return_orchestrate_response:
return: ${orchestrate_result.response.body}
return: ${response}
next: end

assign_disconnected_response:
Expand Down
11 changes: 10 additions & 1 deletion DSL/Ruuter.private/rag-search/POST/llm-connections/add.yml
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,19 @@ add_llm_connection:
access_key: ${access_key}
embedding_model_api_key: ${embedding_model_api_key}
result: connection_result
next: assign_connection_response

assign_connection_response:
assign:
response: {
id: "${connection_result.response.body[0].id}",
status: 201,
operationSuccess: true
}
next: return_success

return_success:
return: "LLM connection added successfully"
return: ${response}
status: 200
next: end

Expand Down
4 changes: 2 additions & 2 deletions GUI/src/services/inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ export interface InferenceResponse {
}

export async function viewInferenceResult(request: InferenceRequest): Promise<InferenceResponse> {
const { data } = await apiDev.post(inferenceEndpoints.VIEW_INFERENCE_RESULT(), {
llmConnectionId: request.llmConnectionId,
const { data } = await apiDev.post(inferenceEndpoints.VIEW_TEST_INFERENCE_RESULT(), {
connectionId: request.llmConnectionId,
message: request.message,
});
return data;
Expand Down
2 changes: 1 addition & 1 deletion GUI/src/utils/endpoints.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,5 @@ export const llmConnectionsEndpoints = {
}

export const inferenceEndpoints = {
VIEW_INFERENCE_RESULT: (): string => `/rag-search/inference/results/view`,
VIEW_TEST_INFERENCE_RESULT: (): string => `/rag-search/inference/test`,
}
4 changes: 2 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ services:
- NODE_ENV=development
- REACT_APP_RUUTER_API_URL=http://localhost:8086
- REACT_APP_RUUTER_PRIVATE_API_URL=http://localhost:8088
- REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004
- REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004/et/dev-auth
- REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost:8086 http://localhost:8088 http://localhost:3004 http://localhost:3005 ws://localhost;
- DEBUG_ENABLED=true
- CHOKIDAR_USEPOLLING=true
Expand Down Expand Up @@ -419,7 +419,7 @@ services:
- ./vault/config:/vault/config # contains vault.hcl
- ./vault/logs:/vault/logs
expose:
- "8200"
- "8200"
networks:
- bykstack
restart: unless-stopped
Expand Down
179 changes: 179 additions & 0 deletions run_vector_indexer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
#!/usr/bin/env python3
"""
Entry point script for Vector Indexer - Contextual Retrieval Pipeline

This script can be run directly or called by cron jobs for automated processing.

Usage:
python run_vector_indexer.py [--config CONFIG_PATH] [--health-check] [--dry-run]

Examples:
# Run with default config
python run_vector_indexer.py

# Run with custom config
python run_vector_indexer.py --config /path/to/config.yaml

# Health check only
python run_vector_indexer.py --health-check

# Dry run (validate without processing)
python run_vector_indexer.py --dry-run
"""

import argparse
import asyncio
import sys
from pathlib import Path

# Add src to Python path
sys.path.insert(0, str(Path(__file__).parent / "src"))

from src.vector_indexer.main_indexer import VectorIndexer


async def main():
"""Main entry point with command line argument parsing."""

parser = argparse.ArgumentParser(
description="Vector Indexer - Contextual Retrieval Pipeline",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)

parser.add_argument(
"--config",
type=str,
default="src/vector_indexer/config/vector_indexer_config.yaml",
help="Path to configuration file (default: src/vector_indexer/config/vector_indexer_config.yaml)",
)

parser.add_argument(
"--health-check", action="store_true", help="Run health check only and exit"
)

parser.add_argument(
"--dry-run",
action="store_true",
help="Validate configuration and connectivity without processing documents",
)

parser.add_argument(
"--verbose", "-v", action="store_true", help="Enable verbose logging"
)

parser.add_argument(
"--quiet", "-q", action="store_true", help="Suppress non-error output"
)

args = parser.parse_args()

# Configure logging level based on arguments
log_level = "INFO"
if args.verbose:
log_level = "DEBUG"
elif args.quiet:
log_level = "ERROR"

Check failure on line 76 in run_vector_indexer.py

View workflow job for this annotation

GitHub Actions / Ruff Lint & Format Check

Ruff (F841)

run_vector_indexer.py:76:9: F841 Local variable `log_level` is assigned to but never used

Check failure on line 76 in run_vector_indexer.py

View workflow job for this annotation

GitHub Actions / Ruff Lint & Format Check

Ruff (F841)

run_vector_indexer.py:76:9: F841 Local variable `log_level` is assigned to but never used

try:
# Initialize vector indexer with specified config
indexer = VectorIndexer(config_path=args.config)

if args.health_check:
# Health check only
print("🔍 Running health check...")
health_ok = await indexer.run_health_check()

if health_ok:
print("✅ Health check passed!")
return 0
else:
print("❌ Health check failed!")
return 1

elif args.dry_run:
# Dry run - validate without processing
print("🧪 Running dry run validation...")

health_ok = await indexer.run_health_check()
if not health_ok:
print("❌ Validation failed!")
return 1

# Discover documents but don't process
documents = indexer.document_loader.discover_all_documents()
print(f"📄 Found {len(documents)} documents ready for processing")
print("✅ Dry run validation passed!")
return 0

else:
# Full processing run
print("🚀 Starting Vector Indexer processing...")

# Health check first
health_ok = await indexer.run_health_check()
if not health_ok:
print("❌ Pre-processing health check failed!")
return 1

# Process all documents
stats = await indexer.process_all_documents()

# Return appropriate exit code
if stats.documents_failed > 0:
print(f"⚠️ Processing completed with {stats.documents_failed} failures")
return 2 # Partial success
else:
print("✅ Processing completed successfully!")
return 0

except KeyboardInterrupt:
print("\n⏹️ Processing interrupted by user")
return 130
except FileNotFoundError as e:
print(f"❌ Configuration file not found: {e}")
return 1
except Exception as e:
print(f"💥 Fatal error: {e}")
return 1


def cron_entry_point():
"""
Entry point specifically designed for cron jobs.

This function:
- Uses minimal output suitable for cron logs
- Returns appropriate exit codes for monitoring
- Handles errors gracefully for automated systems
"""
import logging

# Configure minimal logging for cron
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - Vector Indexer - %(levelname)s - %(message)s",
)

try:
# Run with default configuration
result = asyncio.run(main())

if result == 0:
logging.info("Vector indexer completed successfully")
elif result == 2:
logging.warning("Vector indexer completed with some failures")
else:
logging.error("Vector indexer failed")

return result

except Exception as e:
logging.error(f"Vector indexer fatal error: {e}")
return 1


if __name__ == "__main__":
# Run the async main function
exit_code = asyncio.run(main())
sys.exit(exit_code)
Loading