diff --git a/converter/config.py b/converter/config.py index 130e66b..b9c1988 100644 --- a/converter/config.py +++ b/converter/config.py @@ -35,7 +35,7 @@ class ConverterConfig: memory_type_mapper: Optional[MemoryTypeMapper] = None # Tiering strategy - tiering_strategy_type: str = "step_based" # One of: "step_based", "importance_aware" + tiering_strategy_type: str = "simple" # One of: "simple", "step_based", "importance_aware" tiering_strategy: Optional[TieringStrategy] = None # Import settings @@ -81,7 +81,7 @@ def _validate_batch_size(self): def _validate_tiering_strategy(self): """Validate tiering strategy settings.""" - valid_types = ["step_based", "importance_aware"] + valid_types = ["simple", "step_based", "importance_aware"] if self.tiering_strategy_type not in valid_types: raise ValueError( f"Invalid tiering_strategy_type: {self.tiering_strategy_type}. " @@ -100,7 +100,7 @@ def _validate_tiering_strategy(self): 'ActionModel': 'action', 'SocialInteractionModel': 'interaction' }, - 'tiering_strategy_type': 'step_based', + 'tiering_strategy_type': 'simple', 'import_mode': 'full', 'selective_agents': None } \ No newline at end of file diff --git a/converter/converter.py b/converter/converter.py index 36f98a3..5d3b631 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -7,7 +7,7 @@ from sqlalchemy.exc import SQLAlchemyError -from memory.config import MemoryConfig +from memory.config import MemoryConfig, RedisSTMConfig from memory.core import AgentMemorySystem from .agent_import import AgentImporter, AgentMetadata @@ -102,7 +102,15 @@ def from_agent_farm(db_path: str, config: Optional[Dict] = None) -> AgentMemoryS logger.info(f"Successfully imported {len(all_memories)} total memories") # Create memory system configuration - memory_config = MemoryConfig(use_mock_redis=True, logging_level="INFO") + memory_config = MemoryConfig( + use_mock_redis=True, + logging_level="INFO", + stm_config=RedisSTMConfig( + memory_limit=10000, # Increase STM memory limit + ttl=86400, # 24 hours + namespace="agent-stm", + ), + ) # Create and configure memory system memory_system = AgentMemorySystem.get_instance(memory_config) diff --git a/converter/memory_import.py b/converter/memory_import.py index a402f2d..ffb4abd 100644 --- a/converter/memory_import.py +++ b/converter/memory_import.py @@ -265,14 +265,14 @@ def _import_memory( ) tier = self.tiering_strategy.determine_tier(tiering_context) - # Get the correct ID field based on model type + # Get the correct ID field based on model type and ensure uniqueness memory_id = None if model_name == 'ActionModel': - memory_id = getattr(memory, 'action_id', None) + memory_id = f"action_{getattr(memory, 'action_id', None)}_step_{getattr(memory, 'step_number', 0)}" elif model_name == 'SocialInteractionModel': - memory_id = getattr(memory, 'interaction_id', None) + memory_id = f"interaction_{getattr(memory, 'interaction_id', None)}_step_{getattr(memory, 'step_number', 0)}" elif model_name == 'AgentStateModel': - memory_id = getattr(memory, 'id', None) + memory_id = f"state_{getattr(memory, 'id', None)}_step_{getattr(memory, 'step_number', 0)}" if memory_id is None: logger.warning(f"Could not find ID for {model_name} memory") diff --git a/converter/tiering.py b/converter/tiering.py index e91375f..b24cee1 100644 --- a/converter/tiering.py +++ b/converter/tiering.py @@ -31,6 +31,15 @@ def determine_tier(self, context: TieringContext) -> str: """ pass +class SimpleTieringStrategy(TieringStrategy): + """ + Simple tiering strategy that puts all memories in STM. + """ + + def determine_tier(self, context: TieringContext) -> str: + """Always return STM tier.""" + return "stm" + class StepBasedTieringStrategy(TieringStrategy): """ Step-based time decay tiering strategy. @@ -86,12 +95,12 @@ def determine_tier(self, context: TieringContext) -> str: return base_tier -def create_tiering_strategy(strategy_type: str = "step_based") -> TieringStrategy: +def create_tiering_strategy(strategy_type: str = "simple") -> TieringStrategy: """ Factory function to create tiering strategies. Args: - strategy_type: Type of strategy to create ("step_based" or "importance_aware") + strategy_type: Type of strategy to create ("simple", "step_based", or "importance_aware") Returns: Configured TieringStrategy instance @@ -100,6 +109,7 @@ def create_tiering_strategy(strategy_type: str = "step_based") -> TieringStrateg ValueError: If strategy_type is invalid """ strategies = { + "simple": SimpleTieringStrategy, "step_based": StepBasedTieringStrategy, "importance_aware": ImportanceAwareTieringStrategy } diff --git a/debug_converter.py b/debug_converter.py new file mode 100644 index 0000000..45c7c0b --- /dev/null +++ b/debug_converter.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +""" +Script to run the AgentFarm to Memory System converter. +""" + +import logging +import sys +from datetime import datetime +from pathlib import Path + +from converter.config import DEFAULT_CONFIG +from converter.converter import from_agent_farm + +def main(): + db_path = "data/simulation.db" + output = "validation/memory_samples/agent_farm_memories.json" + validate = False + error_handling = "skip" + use_mock_redis = True + batch_size = 100 + tiering_strategy = "simple" + log_file = None + + # Create logs directory if it doesn't exist + log_dir = Path("logs") + log_dir.mkdir(exist_ok=True) + + # Generate default log filename if not provided + if not log_file: + timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + log_file = log_dir / f"converter_{timestamp}.log" + + # Configure logging + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler(sys.stdout), + ], + force=True, + ) + logger = logging.getLogger(__name__) + + # Set all SQLAlchemy loggers to WARNING level to suppress most logs + logging.getLogger("sqlalchemy").setLevel(logging.WARNING) + logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING) + logging.getLogger("sqlalchemy.orm").setLevel(logging.WARNING) + logging.getLogger("sqlalchemy.pool").setLevel(logging.WARNING) + + # Set all other loggers to DEBUG level + for name in logging.root.manager.loggerDict: + if not name.startswith("sqlalchemy"): + logging.getLogger(name).setLevel(logging.DEBUG) + + logger.info(f"Logging to file: {log_file}") + + # Prepare configuration + config = DEFAULT_CONFIG.copy() + config.update( + { + "validate": validate, + "error_handling": error_handling, + "use_mock_redis": True, + "batch_size": batch_size, + "tiering_strategy_type": tiering_strategy, + } + ) + + # Run converter + logger.info(f"Converting database: {db_path}") + memory_system = from_agent_farm(db_path, config) + + return memory_system + +if __name__ == "__main__": + memory_system = main() + agent = memory_system.agents['nWpvyFJReoFD5Fnq7AEggt'] + stats = agent.get_memory_statistics('nWpvyFJReoFD5Fnq7AEggt') + print(f"STM count: {stats['tiers']['stm']['count']}") + print(f"IM count: {stats['tiers']['im']['count']}") + print(f"LTM count: {stats['tiers']['ltm']['count']}") diff --git a/memory/agent_memory.py b/memory/agent_memory.py index 38a97d9..17e76c7 100644 --- a/memory/agent_memory.py +++ b/memory/agent_memory.py @@ -3,6 +3,7 @@ import logging import math import time +import uuid from typing import Any, Dict, List, Optional, Union from memory.config import MemoryConfig @@ -220,7 +221,7 @@ def _create_memory_entry( Returns: Formatted memory entry """ - # Generate unique memory ID + # Generate memory ID using agent_id, step number and timestamp timestamp = int(time.time()) memory_id = f"{self.agent_id}-{step_number}-{timestamp}" diff --git a/memory/core.py b/memory/core.py index 93115c5..88a427d 100644 --- a/memory/core.py +++ b/memory/core.py @@ -360,24 +360,27 @@ def add_memory(self, memory_data: Dict[str, Any]) -> str: step_number = memory_data.get("step_number", 0) priority = memory_data.get("metadata", {}).get("importance_score", 1.0) memory_type = memory_data.get("type", "generic") + tier = memory_data.get("metadata", {}).get( + "tier", "stm" + ) # Get tier from metadata # Choose appropriate method based on memory type if memory_type == "state": memory_agent.store_state( - memory_data.get("content", {}), step_number, priority + memory_data.get("content", {}), step_number, priority, tier ) elif memory_type == "interaction": memory_agent.store_interaction( - memory_data.get("content", {}), step_number, priority + memory_data.get("content", {}), step_number, priority, tier ) elif memory_type == "action": memory_agent.store_action( - memory_data.get("content", {}), step_number, priority + memory_data.get("content", {}), step_number, priority, tier ) else: # For generic types, use store_state as a fallback memory_agent.store_state( - memory_data.get("content", {}), step_number, priority + memory_data.get("content", {}), step_number, priority, tier ) return memory_id diff --git a/memory/search/strategies/similarity.py b/memory/search/strategies/similarity.py index 5146325..4f9d4ef 100644 --- a/memory/search/strategies/similarity.py +++ b/memory/search/strategies/similarity.py @@ -99,8 +99,11 @@ def search( tiers_to_search = ["stm", "im", "ltm"] if tier is None else [tier] logger.debug("Searching memory tiers: %s", tiers_to_search) - # Get the memory agent - memory_agent = self.memory_system.get_memory_agent(agent_id) + if isinstance(self.memory_system, AgentMemorySystem): + # Get the memory agent + memory_agent = self.memory_system.get_memory_agent(agent_id) + else: + memory_agent = self.memory_system for current_tier in tiers_to_search: # Skip if tier is not supported @@ -221,6 +224,10 @@ def search( ) continue + # Ensure memory has required fields + if "id" not in memory: + memory["id"] = memory_id + # Attach similarity score and tier information if "metadata" not in memory: memory["metadata"] = {} diff --git a/tests/converter/test_config.py b/tests/converter/test_config.py index 968ce40..2578a92 100644 --- a/tests/converter/test_config.py +++ b/tests/converter/test_config.py @@ -4,7 +4,7 @@ import pytest from converter.config import ConverterConfig -from converter.tiering import StepBasedTieringStrategy, ImportanceAwareTieringStrategy +from converter.tiering import SimpleTieringStrategy, StepBasedTieringStrategy, ImportanceAwareTieringStrategy from converter.mapping import MemoryTypeMapper def test_default_config(): @@ -17,8 +17,8 @@ def test_default_config(): assert config.show_progress is True assert config.import_mode == "full" assert config.selective_agents is None - assert config.tiering_strategy_type == "step_based" - assert isinstance(config.tiering_strategy, StepBasedTieringStrategy) + assert config.tiering_strategy_type == "simple" + assert isinstance(config.tiering_strategy, SimpleTieringStrategy) assert isinstance(config.memory_type_mapper, MemoryTypeMapper) assert config.memory_type_mapping == { 'AgentStateModel': 'state', diff --git a/tests/test_memory_agent.py b/tests/test_memory_agent.py index b4e308e..131717d 100644 --- a/tests/test_memory_agent.py +++ b/tests/test_memory_agent.py @@ -28,7 +28,7 @@ import pytest from memory.agent_memory import MemoryAgent -from memory.config import AutoencoderConfig, MemoryConfig +from memory.config import MemoryConfig @pytest.fixture @@ -98,24 +98,25 @@ def memory_agent( agent_id = "test-agent" config = MemoryConfig() config.autoencoder_config.use_neural_embeddings = True - config.text_model_name = "test-model" + config.text_model_name = ( + "sentence-transformers/all-MiniLM-L6-v2" # Use a real model + ) config.ltm_config.db_path = "test_memory.db" # Set a valid db path - # Mock store classes before instantiating the agent - with mock.patch("memory.agent_memory.RedisSTMStore") as mock_stm_class, mock.patch( + with mock.patch("memory.agent_memory.RedisSTMStore") as mock_stm, mock.patch( "memory.agent_memory.RedisIMStore" - ) as mock_im_class, mock.patch( + ) as mock_im, mock.patch( "memory.agent_memory.SQLiteLTMStore" - ) as mock_ltm_class, mock.patch( + ) as mock_ltm, mock.patch( "memory.agent_memory.CompressionEngine" - ), mock.patch( - "memory.embeddings.text_embeddings.TextEmbeddingEngine" - ): + ) as mock_ce, mock.patch( + "memory.agent_memory.TextEmbeddingEngine" + ) as mock_ae: # Configure the mock classes to return our mock instances - mock_stm_class.return_value = mock_stm_store - mock_im_class.return_value = mock_im_store - mock_ltm_class.return_value = mock_ltm_store + mock_stm.return_value = mock_stm_store + mock_im.return_value = mock_im_store + mock_ltm.return_value = mock_ltm_store agent = MemoryAgent(agent_id, config) @@ -147,7 +148,7 @@ def test_init(self): ) as mock_ltm, mock.patch( "memory.agent_memory.CompressionEngine" ) as mock_ce, mock.patch( - "memory.embeddings.text_embeddings.TextEmbeddingEngine" + "memory.agent_memory.TextEmbeddingEngine" ) as mock_ae: agent = MemoryAgent(agent_id, config) @@ -174,7 +175,7 @@ def test_init_without_neural_embeddings(self): ), mock.patch("memory.agent_memory.SQLiteLTMStore"), mock.patch( "memory.agent_memory.CompressionEngine" ), mock.patch( - "memory.embeddings.text_embeddings.TextEmbeddingEngine" + "memory.agent_memory.TextEmbeddingEngine" ) as mock_te: agent = MemoryAgent(agent_id, config)