refactor: replace config-based agents with lightweight registry

.claude/commands/commit.md

@@ -61,4 +61,4 @@ them manually.
 ```
 
 The commit message will be concise, meaningful, and follow your project's
-conventions if I can detect them from recent commits
+conventions if I can detect them from recent commits.

python/CLAUDE.md

@@ -62,8 +62,9 @@ Cairo Coder uses a three-stage RAG pipeline implemented with DSPy modules:
 
 ### Agent-Based Architecture
 
-- **Agent Factory** (`src/cairo_coder/core/agent_factory.py`): Creates specialized agents from TOML configs
-- **Agents**: General, Scarb-specific, or custom agents with filtered sources
+- **Agent Registry** (`src/cairo_coder/agents/registry.py`): Lightweight enum-based registry of available agents
+- **Agent Factory** (`src/cairo_coder/core/agent_factory.py`): Creates agents from the registry with caching
+- **Agents**: Two built-in agents - Cairo Coder (general purpose) and Scarb Assistant (Scarb-specific)
 - **Pipeline Factory**: Creates optimized RAG pipelines loading from `optimizers/results/`
 
 ### FastAPI Server
@@ -93,15 +94,14 @@ Cairo Coder uses a three-stage RAG pipeline implemented with DSPy modules:
 
 ### Adding New Features
 
-1. **New Agent**: Extend `AgentConfiguration` with default agent configurations
+1. **New Agent**: Add an entry to `agents/registry.py` with `AgentId` enum and `AgentSpec` in the registry
 2. **New DSPy Module**: Create signature, implement forward/aforward methods
 3. **New Optimizer**: Create Marimo notebook, define metrics, use MIPROv2
 
 ### Configuration Management
 
-- `ConfigManager` loads from environment variables only
 - All configuration comes from environment variables (see `.env.example` in root)
-- Default agents are hardcoded in `AgentConfiguration` class
+- Agents are configured in the registry in `agents/registry.py`
 
 ## Important Notes
 
@@ -159,7 +159,7 @@ Familiarize yourself with these core fixtures defined in `conftest.py`. Use them
 - `mock_agent_factory`: A mock of the `AgentFactory` used in server tests to control which agent is created.
 - `mock_vector_db`: A mock of `SourceFilteredPgVectorRM` for testing the document retrieval layer without a real database.
 - `mock_lm`: A mock of a `dspy` language model for testing DSPy programs (`QueryProcessorProgram`, `GenerationProgram`) without making real API calls.
-- `sample_documents`, `sample_agent_configs`, `sample_processed_query`: Consistent, reusable data fixtures for your tests.
+- `sample_documents`, `sample_processed_query`: Consistent, reusable data fixtures for your tests.
 
 ### 5. Guidelines for Adding & Modifying Tests
 

python/src/cairo_coder/agents/registry.py

@@ -0,0 +1,117 @@
+"""
+Agent Registry for Cairo Coder.
+
+A lightweight enum-based registry that replaces the configuration-based
+agent system with a simple, in-memory registry of available agents.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+
+from cairo_coder.core.config import VectorStoreConfig
+from cairo_coder.core.rag_pipeline import RagPipeline, RagPipelineFactory
+from cairo_coder.core.types import DocumentSource
+from cairo_coder.dspy.document_retriever import SourceFilteredPgVectorRM
+from cairo_coder.dspy.generation_program import (
+    create_generation_program,
+    create_mcp_generation_program,
+)
+from cairo_coder.dspy.query_processor import create_query_processor
+
+
+class AgentId(Enum):
+    """Available agent identifiers."""
+
+    CAIRO_CODER = "cairo-coder"
+    SCARB = "scarb-assistant"
+
+
+@dataclass
+class AgentSpec:
+    """Specification for an agent."""
+
+    name: str
+    description: str
+    sources: list[DocumentSource]
+    generation_program_type: AgentId
+    max_source_count: int = 5
+    similarity_threshold: float = 0.4
+
+    def build(self, vector_db: SourceFilteredPgVectorRM, vector_store_config: VectorStoreConfig) -> RagPipeline:
+        """
+        Build a RagPipeline instance from this specification.
+
+        Args:
+            vector_db: Pre-initialized vector database instance
+            vector_store_config: Vector store configuration
+
+        Returns:
+            Configured RagPipeline instance
+        """
+        match self.generation_program_type:
+            case AgentId.SCARB:
+                return RagPipelineFactory.create_pipeline(
+                    name=self.name,
+                    vector_store_config=vector_store_config,
+                    sources=self.sources,
+                    query_processor=create_query_processor(),
+                    generation_program=create_generation_program("scarb"),
+                    mcp_generation_program=create_mcp_generation_program(),
+                    max_source_count=self.max_source_count,
+                    similarity_threshold=self.similarity_threshold,
+                    vector_db=vector_db,
+                )
+            case AgentId.CAIRO_CODER:
+                return RagPipelineFactory.create_pipeline(
+                    name=self.name,
+                    vector_store_config=vector_store_config,
+                    sources=self.sources,
+                    query_processor=create_query_processor(),
+                    generation_program=create_generation_program(),
+                    mcp_generation_program=create_mcp_generation_program(),
+                    max_source_count=self.max_source_count,
+                    similarity_threshold=self.similarity_threshold,
+                    vector_db=vector_db,
+                )
+
+
+# The global registry of available agents
+registry: dict[AgentId, AgentSpec] = {
+    AgentId.CAIRO_CODER: AgentSpec(
+        name="Cairo Coder",
+        description="General Cairo programming assistant",
+        sources=list(DocumentSource),  # All sources
+        generation_program_type=AgentId.CAIRO_CODER,
+        max_source_count=5,
+        similarity_threshold=0.4,
+    ),
+    AgentId.SCARB: AgentSpec(
+        name="Scarb Assistant",
+        description="Specialized assistant for Scarb build tool",
+        sources=[DocumentSource.SCARB_DOCS],
+        generation_program_type=AgentId.SCARB,
+        max_source_count=5,
+        similarity_threshold=0.3,  # Lower threshold for Scarb-specific queries
+    ),
+}
+
+
+def get_agent_by_string_id(agent_id: str) -> tuple[AgentId, AgentSpec]:
+    """
+    Get agent by string ID.
+
+    Args:
+        agent_id: String agent ID (must match enum value)
+
+    Returns:
+        Tuple of (AgentId enum, AgentSpec)
+
+    Raises:
+        ValueError: If agent_id is not found
+    """
+    # Try to find matching enum by value
+    for enum_id in AgentId:
+        if enum_id.value == agent_id:
+            return enum_id, registry[enum_id]
+
+    raise ValueError(f"Agent not found: {agent_id}")

python/src/cairo_coder/config/manager.py

@@ -2,11 +2,7 @@
 
 import os
 
-from ..core.config import (
-    AgentConfiguration,
-    Config,
-    VectorStoreConfig,
-)
+from ..core.config import Config, VectorStoreConfig
 
 
 class ConfigManager:
@@ -50,31 +46,8 @@ def load_config() -> Config:
             host=host,
             port=port,
             debug=debug,
-            default_agent_id="cairo-coder",
         )
 
-    @staticmethod
-    def get_agent_config(config: Config, agent_id: str | None = None) -> AgentConfiguration:
-        """
-        Get agent configuration by ID.
-
-        Args:
-            config: Application configuration.
-            agent_id: Agent ID to retrieve. Defaults to default agent.
-
-        Returns:
-            Agent configuration.
-
-        Raises:
-            ValueError: If agent ID is not found.
-        """
-        if agent_id is None:
-            agent_id = config.default_agent_id
-
-        if agent_id not in config.agents:
-            raise ValueError(f"Agent '{agent_id}' not found in configuration")
-
-        return config.agents[agent_id]
 
     @staticmethod
     def validate_config(config: Config) -> None:
@@ -90,14 +63,3 @@ def validate_config(config: Config) -> None:
         # Check database configuration
         if not config.vector_store.password:
             raise ValueError("Database password is required")
-
-        # Check agents have valid sources
-        for agent_id, agent in config.agents.items():
-            if not agent.sources:
-                raise ValueError(f"Agent '{agent_id}' has no sources configured")
-
-        # Check default agent exists
-        if config.default_agent_id not in config.agents:
-            raise ValueError(
-                f"Default agent '{config.default_agent_id}' not found in configuration"
-            )