Enhance pipeline

.env.example

@@ -1,93 +1,222 @@
-# Single Family Loan Q&A Application Configuration
-# Copy this file to .env and update with your actual values
+# =============================================================================
+# converSQL - Environment Configuration
+# Multi-Provider AI-Powered SQL Generation
+# =============================================================================
 
-# ========================================
-# DATA CONFIGURATION
-# ========================================
+# -----------------------------------------------------------------------------
+# AI Provider Configuration (Multi-Provider Support)
+# -----------------------------------------------------------------------------
+
+# DEFAULT AI PROVIDER (Primary provider for SQL generation)
+# Options: bedrock | claude | gemini
+# 
+# - bedrock: AWS Bedrock (requires AWS credentials + model access)
+# - claude:  Anthropic Claude API (requires CLAUDE_API_KEY)
+# - gemini:  Google Gemini API (requires GOOGLE_API_KEY)
+#
+# Leave empty or set to 'auto' to use first available provider
+# The app will automatically fall back to other configured providers if the primary is unavailable
+AI_PROVIDER=claude
+
+# Enable prompt caching for supported providers (reduces costs and latency)
+# Options: true | false
+ENABLE_PROMPT_CACHE=true
+
+# -----------------------------------------------------------------------------
+# AWS Bedrock Configuration
+# -----------------------------------------------------------------------------
+# To use Bedrock: Configure AWS credentials + set AI_PROVIDER=bedrock
+# OR let it auto-select if credentials are available
+
+# AWS region for Bedrock service
+# Recommended: us-west-2, us-east-1 (check model availability by region)
+AWS_DEFAULT_REGION=us-west-2
+
+# Bedrock model identifier
+# Recommended models:
+#   - anthropic.claude-3-5-sonnet-20241022-v2:0  (Most capable, higher cost)
+#   - anthropic.claude-3-5-haiku-20241022-v1:0   (Fast, cost-effective)
+BEDROCK_MODEL_ID=anthropic.claude-3-5-haiku-20241022-v1:0
+
+# Bedrock guardrail configuration (OPTIONAL - for content filtering)
+# Leave empty if not using guardrails
+BEDROCK_GUARDRAIL_ID=
+BEDROCK_GUARDRAIL_VERSION=DRAFT
+
+# AWS credentials (OPTIONAL if using IAM roles, EC2 instance profiles, or AWS CLI profiles)
+# Only set these if you're not using AWS credential chain
+# AWS_ACCESS_KEY_ID=your-access-key-id
+# AWS_SECRET_ACCESS_KEY=your-secret-access-key
+
+# -----------------------------------------------------------------------------
+# Anthropic Claude API Configuration
+# -----------------------------------------------------------------------------
+# To use Claude API: Set CLAUDE_API_KEY + set AI_PROVIDER=claude
+# Get your API key from: https://console.anthropic.com/
+
+# Anthropic API key (REQUIRED for Claude API access)
+CLAUDE_API_KEY=
+
+# Claude model identifier
+# Recommended models:
+#   - claude-3-5-sonnet-20241022  (Most capable, balanced cost/performance)
+#   - claude-3-5-haiku-20241022   (Fast, cost-effective)
+CLAUDE_MODEL=claude-3-5-sonnet-20241022
+
+# -----------------------------------------------------------------------------
+# Google Gemini Configuration
+# -----------------------------------------------------------------------------
+# To use Gemini: Set GOOGLE_API_KEY + set AI_PROVIDER=gemini
+# Get your API key from: https://aistudio.google.com/app/apikey
+
+# Google API key (REQUIRED for Gemini access)
+GOOGLE_API_KEY=
+
+# Alternative: Use GEMINI_API_KEY if you prefer (both work)
+# GEMINI_API_KEY=
+
+# Gemini model identifier
+# Recommended models:
+#   - gemini-1.5-pro    (Most capable)
+#   - gemini-1.5-flash  (Fast, cost-effective)
+GEMINI_MODEL=gemini-1.5-pro
+
+# -----------------------------------------------------------------------------
+# Google OAuth Configuration
+# -----------------------------------------------------------------------------
+
+# Google OAuth credentials for authentication
+GOOGLE_CLIENT_ID=your_google_client_id
+GOOGLE_CLIENT_SECRET=your_google_client_secret
+
+# OAuth redirect URIs - Configure in Google Cloud Console:
+# → APIs & Services → Credentials → OAuth 2.0 Client ID
+#
+# For LOCALHOST development:   http://localhost:8501
+# For REPLIT deployment:        https://[repl-name].[username].repl.co
+# For STREAMLIT CLOUD:          https://[your-app-name].streamlit.app
+# For CUSTOM DOMAIN:            https://yourdomain.com
+
+# Session secret key for secure cookie signing
+# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
+SESSION_SECRET_KEY=your-session-secret-key-here
+
+# -----------------------------------------------------------------------------
+# Data Configuration
+# -----------------------------------------------------------------------------
+
+# Directory containing processed data files (Parquet format)
 PROCESSED_DATA_DIR=data/processed/
+
+# Default data file to load
+DEFAULT_DATA_FILE=data.parquet
+
+# Cache configuration
 CACHE_TTL=3600
 FORCE_DATA_REFRESH=false
 
-# ========================================
-# CLOUDFLARE R2 CONFIGURATION
-# ========================================
-# Your Cloudflare R2 bucket details
-R2_ENDPOINT_URL=https://50ee71713e4e8762d5eab0e8ec442f1e.r2.cloudflarestorage.com
+# -----------------------------------------------------------------------------
+# Cloudflare R2 Storage Configuration (Optional)
+# -----------------------------------------------------------------------------
+
+# R2 bucket name
 R2_BUCKET_NAME=single-family-loan
+
+# R2 access credentials
 R2_ACCESS_KEY_ID=your_r2_access_key_id
 R2_SECRET_ACCESS_KEY=your_r2_secret_access_key
-R2_ACCOUNT_ID=your_cloudflare_account_id
 
-# ========================================
-# AI PROVIDER CONFIGURATION
-# ========================================
-# Choose your AI provider: 'claude' or 'bedrock'
-AI_PROVIDER=claude
+# R2 endpoint URL
+R2_ENDPOINT_URL=https://50ee71713e4e8762d5eab0e8ec442f1e.r2.cloudflarestorage.com
 
-# Claude API Configuration (Recommended)
-CLAUDE_API_KEY=your_claude_api_key_here
-CLAUDE_MODEL=claude-3-5-sonnet-20241022
+# Cloudflare account ID
+R2_ACCOUNT_ID=your_cloudflare_account_id
 
-# Amazon Bedrock Configuration (Alternative)
-BEDROCK_GUARDRAIL_ID=your_actual_guardrail_id
-BEDROCK_MODEL_ID=anthropic.claude-3-5-haiku-20241022-v1:0
-AWS_DEFAULT_REGION=us-west-2
-ENABLE_BEDROCK=true
+# -----------------------------------------------------------------------------
+# Cloudflare D1 Database Configuration (Optional)
+# -----------------------------------------------------------------------------
 
-# AWS Authentication (for Bedrock - choose one method):
-# Option 1: AWS Access Keys (for development/testing)
-AWS_ACCESS_KEY_ID=your_aws_access_key
-AWS_SECRET_ACCESS_KEY=your_aws_secret_key
+# D1 database ID for user activity logging
+CLOUDFLARE_D1_DATABASE_ID=your_d1_database_id
 
-# Option 2: Use AWS CLI credentials or IAM roles (recommended for production)
-# Leave the above commented out to use AWS CLI or IAM role credentials
+# D1 API token
+CLOUDFLARE_API_TOKEN=your_cloudflare_api_token
 
-# ========================================
-# PERFORMANCE OPTIMIZATION
-# ========================================
-ENABLE_PROMPT_CACHE=true
-PROMPT_CACHE_TTL=3600
+# Cloudflare account ID
+CLOUDFLARE_ACCOUNT_ID=your_cloudflare_account_id
 
-# ========================================
-# AUTHENTICATION (GOOGLE OAUTH)
-# ========================================
-# Google OAuth Configuration (Direct - No Supabase needed)
-GOOGLE_CLIENT_ID=your_google_client_id
-GOOGLE_CLIENT_SECRET=your_google_client_secret
+# -----------------------------------------------------------------------------
+# Application Settings
+# -----------------------------------------------------------------------------
 
-# IMPORTANT: Configure these URLs in your Google OAuth settings:
-# Go to Google Cloud Console → APIs & Services → Credentials
-# Edit your OAuth 2.0 Client ID and add these redirect URIs:
-#
-# For LOCALHOST development:
-# - http://localhost:8501
-#
-# For REPLIT deployment:
-# - https://[repl-name].[username].repl.co
-#
-# For STREAMLIT CLOUD deployment:
-# - https://[your-app-name].streamlit.app
-#
-# For CUSTOM DOMAIN:
-# - https://yourdomain.com
+# Application environment
+# Options: development, staging, production
+ENVIRONMENT=development
 
-# ========================================
-# STREAMLIT CONFIGURATION
-# ========================================
-STREAMLIT_SERVER_PORT=8501
-STREAMLIT_SERVER_HEADLESS=true
+# Enable debug mode (detailed error messages and logging)
+# Options: true, false
+DEBUG=true
 
-# ========================================
-# APPLICATION FEATURES
-# ========================================
+# Application features
 DEMO_MODE=false
 ENABLE_AUTH=false
 
-# ========================================
-# CLOUDFLARE D1 DATABASE (OPTIONAL)
-# ========================================
-# For user activity logging (logins and queries)
-# Leave empty to disable database logging
-CLOUDFLARE_ACCOUNT_ID=your_cloudflare_account_id
-CLOUDFLARE_D1_DATABASE_ID=your_d1_database_id
-CLOUDFLARE_API_TOKEN=your_cloudflare_api_token
+# Streamlit configuration
+STREAMLIT_SERVER_PORT=8501
+STREAMLIT_SERVER_HEADLESS=true
+
+# =============================================================================
+# Setup Instructions
+# =============================================================================
+#
+# 1. Copy this file to .env:
+#    cp .env.example .env
+#
+# 2. CONFIGURE AI PROVIDERS (Choose one or more):
+#
+#    OPTION A - Single Provider Setup:
+#    --------------------------------
+#    Configure ONE provider and set AI_PROVIDER explicitly:
+#      
+#      For Claude API:
+#        - Set CLAUDE_API_KEY=sk-ant-...
+#        - Set AI_PROVIDER=claude
+#      
+#      For AWS Bedrock:
+#        - Configure AWS credentials (IAM role OR access keys)
+#        - Set AWS_DEFAULT_REGION and BEDROCK_MODEL_ID
+#        - Set AI_PROVIDER=bedrock
+#      
+#      For Google Gemini:
+#        - Set GOOGLE_API_KEY=AIza...
+#        - Set AI_PROVIDER=gemini
+#
+#    OPTION B - Multi-Provider Setup (Recommended):
+#    ----------------------------------------------
+#    Configure MULTIPLE providers for redundancy and flexibility:
+#      
+#      1. Set up credentials for all providers you want to use
+#         (e.g., both CLAUDE_API_KEY and GOOGLE_API_KEY)
+#      
+#      2. Set AI_PROVIDER to your preferred default provider
+#         OR leave empty for automatic selection
+#      
+#      3. The app will show a provider selector in the sidebar
+#         allowing you to switch between available providers
+#
+# 3. CONFIGURE AUTHENTICATION (Optional - for user management):
+#    - Set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET
+#    - Configure OAuth redirect URIs in Google Cloud Console
+#    - Set ENABLE_AUTH=true
+#
+# 4. CONFIGURE DATA PATHS (Optional - customize data location):
+#    - Review PROCESSED_DATA_DIR and DEFAULT_DATA_FILE
+#    - Configure R2 or D1 settings if using Cloudflare storage
+#
+# 5. REVIEW APPLICATION SETTINGS:
+#    - Set ENVIRONMENT (development/staging/production)
+#    - Enable/disable DEBUG mode
+#    - Configure DEMO_MODE if needed
+#
+# For detailed setup instructions, see docs/ENVIRONMENT_SETUP.md
+# =============================================================================