Message compressor + input folder

SimpleAgent/.env.example

@@ -48,8 +48,25 @@ DEBUG_MODE=False
 # Output directory for all file operations
 OUTPUT_DIR=output
 
+# Input directory for user files that the agent can access
+INPUT_DIR=input
+
 # Memory file location (relative to OUTPUT_DIR)
 MEMORY_FILE=memory.json
 
+# =============================================================================
+# INPUT FILE SYSTEM SETTINGS
+# =============================================================================
+
+# Maximum input file size in bytes (default: 10MB)
+MAX_INPUT_FILE_SIZE=10485760
+
+# Allowed input file extensions (comma-separated)
+ALLOWED_INPUT_EXTENSIONS=.txt,.json,.csv,.md,.py,.js,.html,.css,.xml,.yaml,.yml
+
+# =============================================================================
+# EXTERNAL INTEGRATIONS
+# =============================================================================
+
 # GitHub Token - Optional, for GitHub operations
-# GITHUB_TOKEN=
+# GITHUB_TOKEN=

SimpleAgent/commands/file_ops/access_input_file/__init__.py

@@ -0,0 +1,189 @@
+"""
+Access Input File Command
+
+This command allows the agent to securely access files from the input directory.
+It provides read-only access to user-placed files with proper security validation.
+"""
+
+import json
+from typing import Dict, Any, List
+
+from core.execution.tool_manager import register_command
+from core.utils.input_manager import InputManager
+
+
+def access_input_file(filename: str = "", operation: str = "read", search_term: str = None, encoding: str = "utf-8") -> str:
+    """
+    Access and read input files securely.
+
+    Args:
+        filename: Name of the file to access
+        operation: Operation to perform ("read", "info", "list", "search", "json", "csv")
+        search_term: Term to search for (only used with "search" operation)
+        encoding: Text encoding to use for reading (default: utf-8)
+
+    Returns:
+        String containing the requested information or file content
+    """
+    input_manager = InputManager()
+
+    try:
+        if operation == "list":
+            # List all available input files
+            files = input_manager.list_input_files()
+            if not files:
+                return "No input files found in the input directory."
+
+            result = "Available input files:\n"
+            for file_info in files:
+                size_kb = file_info.size / 1024
+                result += f"- {file_info.name} ({size_kb:.1f} KB, {file_info.extension}, modified: {file_info.modified_time.strftime('%Y-%m-%d %H:%M')})\n"
+
+            return result.strip()
+
+        elif operation == "info":
+            # Get detailed information about a specific file
+            if not filename:
+                return "Error: filename is required for 'info' operation"
+
+            file_info = input_manager.get_input_file_info(filename)
+            size_kb = file_info.size / 1024
+
+            return f"""File Information for '{filename}':
+- Size: {file_info.size} bytes ({size_kb:.1f} KB)
+- Type: {file_info.extension} ({file_info.mime_type})
+- Modified: {file_info.modified_time.strftime('%Y-%m-%d %H:%M:%S')}
+- Text file: {'Yes' if file_info.is_text else 'No'}
+- Encoding: {file_info.encoding or 'Unknown'}"""
+
+        elif operation == "read":
+            # Read the full content of a file
+            if not filename:
+                return "Error: filename is required for 'read' operation"
+
+            content = input_manager.read_input_file(filename, encoding)
+
+            # Add a header with file info
+            file_info = input_manager.get_input_file_info(filename)
+            size_kb = file_info.size / 1024
+
+            header = f"=== Content of '{filename}' ({size_kb:.1f} KB) ===\n"
+            return header + content
+
+        elif operation == "search":
+            # Search for a term within a file
+            if not filename:
+                return "Error: filename is required for 'search' operation"
+            if not search_term:
+                return "Error: search_term is required for 'search' operation"
+
+            results = input_manager.search_file_content(filename, search_term, case_sensitive=False)
+
+            if not results:
+                return f"No matches found for '{search_term}' in '{filename}'"
+
+            result = f"Found {len(results)} matches for '{search_term}' in '{filename}':\n\n"
+            for line_num, line_content in results:
+                result += f"Line {line_num}: {line_content.strip()}\n"
+
+            return result.strip()
+
+        elif operation == "json":
+            # Read and parse a JSON file
+            if not filename:
+                return "Error: filename is required for 'json' operation"
+
+            data = input_manager.read_json_file(filename)
+
+            # Format the JSON nicely
+            formatted_json = json.dumps(data, indent=2, ensure_ascii=False)
+
+            header = f"=== JSON Content of '{filename}' ===\n"
+            return header + formatted_json
+
+        elif operation == "csv":
+            # Read a CSV file and return lines
+            if not filename:
+                return "Error: filename is required for 'csv' operation"
+
+            lines = input_manager.read_csv_lines(filename)
+
+            if not lines:
+                return f"CSV file '{filename}' is empty or contains no valid lines"
+
+            result = f"=== CSV Content of '{filename}' ({len(lines)} lines) ===\n"
+            for i, line in enumerate(lines[:50], 1):  # Show first 50 lines
+                result += f"{i}: {line}\n"
+
+            if len(lines) > 50:
+                result += f"\n... and {len(lines) - 50} more lines"
+
+            return result.strip()
+
+        elif operation == "summary":
+            # Get a summary of all input files
+            summary = input_manager.get_file_summary()
+
+            result = f"Input Directory Summary:\n"
+            result += f"- Total files: {summary['total_files']}\n"
+            result += f"- Total size: {summary['total_size'] / 1024:.1f} KB\n"
+
+            if summary['file_types']:
+                result += f"- File types: {', '.join(f'{ext}({count})' for ext, count in summary['file_types'].items())}\n"
+
+            if summary['files']:
+                result += f"\nFiles:\n"
+                for file_data in summary['files']:
+                    result += f"  - {file_data['name']} ({file_data['size']/1024:.1f} KB)\n"
+
+            return result.strip()
+
+        else:
+            return f"Error: Unknown operation '{operation}'. Available operations: read, info, list, search, json, csv, summary"
+
+    except FileNotFoundError as e:
+        return f"Error: File not found - {str(e)}"
+    except PermissionError as e:
+        return f"Error: Access denied - {str(e)}"
+    except ValueError as e:
+        return f"Error: Invalid file or operation - {str(e)}"
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+
+# Command schema for the tool manager
+ACCESS_INPUT_FILE_SCHEMA = {
+    "type": "function",
+    "function": {
+        "name": "access_input_file",
+        "description": "Access and read files from the input directory. Supports various operations like reading content, getting file info, listing files, searching content, and parsing JSON/CSV files.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "filename": {
+                    "type": "string",
+                    "description": "Name of the file to access (required for most operations except 'list' and 'summary')"
+                },
+                "operation": {
+                    "type": "string",
+                    "enum": ["read", "info", "list", "search", "json", "csv", "summary"],
+                    "description": "Operation to perform: 'read' (read full content), 'info' (get file details), 'list' (list all files), 'search' (search for text), 'json' (parse JSON), 'csv' (read CSV lines), 'summary' (get directory summary)",
+                    "default": "read"
+                },
+                "search_term": {
+                    "type": "string",
+                    "description": "Term to search for (only used with 'search' operation)"
+                },
+                "encoding": {
+                    "type": "string",
+                    "description": "Text encoding to use for reading files",
+                    "default": "utf-8"
+                }
+            },
+            "required": []
+        }
+    }
+}
+
+# Register the command
+register_command("access_input_file", access_input_file, ACCESS_INPUT_FILE_SCHEMA)

SimpleAgent/core/__init__.py

@@ -11,6 +11,7 @@
 from core.conversation.memory import MemoryManager
 from core.agent.run_manager import RunManager
 from core.utils.security import get_secure_path
+from core.utils.input_manager import InputManager
 
 __all__ = [
     "SimpleAgent", 
@@ -19,5 +20,6 @@
     "ExecutionManager",
     "MemoryManager",
     "RunManager",
-    "get_secure_path"
-] 
+    "get_secure_path",
+    "InputManager"
+]

SimpleAgent/core/agent/agent.py

@@ -20,6 +20,7 @@
 from core.agent.run_manager import RunManager
 from core.utils.security import get_secure_path
 from core.utils.config import DEFAULT_MODEL, OUTPUT_DIR
+from core.utils.input_manager import InputManager
 
 
 class SimpleAgent:
@@ -47,6 +48,9 @@ def __init__(self, model: str = None, output_dir: str = None):
         self.execution_manager = self.run_manager.execution_manager
         self.memory_manager = self.run_manager.memory_manager
 
+        # Initialize input manager for accessing input files
+        self.input_manager = InputManager()
+
         # Initialize memory
         self.memory = self.memory_manager.get_memory()
 
@@ -134,4 +138,58 @@ def get_secure_path(self, file_path: str) -> str:
         Returns:
             Modified file path within output directory
         """
-        return get_secure_path(file_path, self.output_dir) 
+        return get_secure_path(file_path, self.output_dir)
+
+    def list_input_files(self) -> List[str]:
+        """
+        List all available input files.
+
+        Returns:
+            List of input file names
+        """
+        files = self.input_manager.list_input_files()
+        return [f.name for f in files]
+
+    def read_input_file(self, filename: str) -> str:
+        """
+        Read the contents of an input file.
+
+        Args:
+            filename: Name of the file to read
+
+        Returns:
+            File contents as string
+        """
+        return self.input_manager.read_input_file(filename)
+
+    def get_input_file_info(self, filename: str) -> Dict[str, Any]:
+        """
+        Get information about an input file.
+
+        Args:
+            filename: Name of the file
+
+        Returns:
+            Dictionary with file information
+        """
+        file_info = self.input_manager.get_input_file_info(filename)
+        return {
+            'name': file_info.name,
+            'size': file_info.size,
+            'extension': file_info.extension,
+            'mime_type': file_info.mime_type,
+            'modified_time': file_info.modified_time.isoformat(),
+            'is_text': file_info.is_text
+        }
+
+    def input_file_exists(self, filename: str) -> bool:
+        """
+        Check if an input file exists.
+
+        Args:
+            filename: Name of the file to check
+
+        Returns:
+            True if file exists and is accessible, False otherwise
+        """
+        return self.input_manager.file_exists(filename)

SimpleAgent/core/agent/run_manager.py

@@ -38,9 +38,9 @@ def __init__(self, model: str, output_dir: str = OUTPUT_DIR):
             output_dir: The output directory for file operations
         """
         self.output_dir = output_dir
-        self.conversation_manager = ConversationManager()
-        self.execution_manager = ExecutionManager(model=model, output_dir=output_dir)
         self.memory_manager = MemoryManager()
+        self.conversation_manager = ConversationManager(memory_manager=self.memory_manager)
+        self.execution_manager = ExecutionManager(model=model, output_dir=output_dir)
         self.summarizer = ChangeSummarizer()
         self.loop_detector = LoopDetector(window_size=5, similarity_threshold=0.7)
 
@@ -73,6 +73,9 @@ def run(self, user_instruction: str, max_steps: int = 10, auto_continue: int = 0
         print(f"🛠️ Requires Tools: {task_goal.requires_tools}")
         if task_goal.expected_deliverables:
             print(f"📦 Expected Deliverables: {', '.join(task_goal.expected_deliverables)}")
+
+        # Set task objective for context compression
+        self.conversation_manager.set_task_objective(task_goal.primary_objective)
 
         # Get current date and time information for the system message
         current_datetime = time.strftime("%Y-%m-%d %H:%M:%S")
@@ -173,6 +176,10 @@ def run(self, user_instruction: str, max_steps: int = 10, auto_continue: int = 0
 
                     self.conversation_manager.update_system_message(system_content)
 
+                    # Check if context compression is needed before processing
+                    if self.conversation_manager.compress_if_needed():
+                        print("🔄 Context compressed - continuing with optimized history")
+
                     print(f"\n--- Step {step}/{max_steps} ---")
 
                     try:
@@ -373,6 +380,14 @@ def run(self, user_instruction: str, max_steps: int = 10, auto_continue: int = 0
                     self.execution_manager.stop_requested = True
                     print("\n✅ Agent execution interrupted by user")
 
+            # Show context compression statistics if any compression occurred
+            compression_stats = self.conversation_manager.get_compression_stats()
+            if compression_stats["total_compressions"] > 0:
+                print(f"\n🗜️ Context Compression Statistics:")
+                print(f"   📊 Total compressions: {compression_stats['total_compressions']}")
+                print(f"   💾 Total tokens saved: {compression_stats['total_tokens_saved']}")
+                print(f"   ⚡ Enabled long-running task capability")
+
             # Generate a final summary of all changes
             if changes_made:
                 final_summary = self.summarizer.summarize_changes(changes_made)

SimpleAgent/core/config.py

@@ -38,6 +38,26 @@
 if not os.path.exists(OUTPUT_DIR):
     os.makedirs(OUTPUT_DIR)
 
+# Input directory - All input files should be placed here for agent access
+# Can be customized through environment variable
+# Use absolute path to ensure it works regardless of current working directory
+_input_dir_relative = os.getenv("INPUT_DIR", "input")
+if os.path.isabs(_input_dir_relative):
+    INPUT_DIR = _input_dir_relative
+else:
+    # Get the directory where this config file is located (core/)
+    _config_dir = os.path.dirname(os.path.abspath(__file__))
+    # Go up one level to get to the SimpleAgent root directory
+    _project_root = os.path.dirname(_config_dir)
+    INPUT_DIR = os.path.join(_project_root, _input_dir_relative)
+
+if not os.path.exists(INPUT_DIR):
+    os.makedirs(INPUT_DIR)
+
+# Input file settings
+MAX_INPUT_FILE_SIZE = int(os.getenv("MAX_INPUT_FILE_SIZE", str(10 * 1024 * 1024)))  # 10MB default
+ALLOWED_INPUT_EXTENSIONS = os.getenv("ALLOWED_INPUT_EXTENSIONS", ".txt,.json,.csv,.md,.py,.js,.html,.css,.xml,.yaml,.yml").split(",")
+
 # Memory settings
 MEMORY_FILE = os.path.join(OUTPUT_DIR, os.getenv("MEMORY_FILE", "memory.json"))
 
@@ -71,4 +91,4 @@ def create_client():
         if not OPENAI_API_KEY:
             raise ValueError("OPENAI_API_KEY must be set when using OpenAI provider")
         from openai import OpenAI
-        return OpenAI(api_key=OPENAI_API_KEY) 
+        return OpenAI(api_key=OPENAI_API_KEY)