Logging

autogen/agentchat/assistant_agent.py

@@ -1,6 +1,7 @@
 from typing import Callable, Dict, Literal, Optional, Union
 
 from .conversable_agent import ConversableAgent
+from autogen.runtime_logging import logging_enabled, log_new_agent
 
 
 class AssistantAgent(ConversableAgent):
@@ -67,6 +68,8 @@ def __init__(
             description=description,
             **kwargs,
         )
+        if logging_enabled():
+            log_new_agent(self, locals())
 
         # Update the provided description if None, and we are using the default system_message,
         # then use the default description.

autogen/agentchat/conversable_agent.py

@@ -14,6 +14,7 @@
 from ..coding.factory import CodeExecutorFactory
 
 from ..oai.client import OpenAIWrapper, ModelClient
+from ..runtime_logging import logging_enabled, log_new_agent
 from ..cache.cache import Cache
 from ..code_utils import (
     UNKNOWN,
@@ -145,6 +146,9 @@ def __init__(
                 self.llm_config.update(llm_config)
             self.client = OpenAIWrapper(**self.llm_config)
 
+        if logging_enabled():
+            log_new_agent(self, locals())
+
         # Initialize standalone client cache object.
         self.client_cache = None
 

autogen/agentchat/groupchat.py

@@ -9,9 +9,9 @@
 from ..code_utils import content_str
 from .agent import Agent
 from .conversable_agent import ConversableAgent
+from ..runtime_logging import logging_enabled, log_new_agent
 from ..graph_utils import check_graph_validity, invert_disallowed_to_allowed
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -463,6 +463,8 @@ def __init__(
             system_message=system_message,
             **kwargs,
         )
+        if logging_enabled():
+            log_new_agent(self, locals())
         # Store groupchat
         self._groupchat = groupchat
 

autogen/agentchat/user_proxy_agent.py

@@ -1,6 +1,7 @@
 from typing import Callable, Dict, List, Literal, Optional, Union
 
 from .conversable_agent import ConversableAgent
+from ..runtime_logging import logging_enabled, log_new_agent
 
 
 class UserProxyAgent(ConversableAgent):
@@ -93,3 +94,6 @@ def __init__(
             if description is not None
             else self.DEFAULT_USER_PROXY_AGENT_DESCRIPTIONS[human_input_mode],
         )
+
+        if logging_enabled():
+            log_new_agent(self, locals())

autogen/logger/__init__.py

@@ -0,0 +1,4 @@
+from .logger_factory import LoggerFactory
+from .sqlite_logger import SqliteLogger
+
+__all__ = ("LoggerFactory", "SqliteLogger")

autogen/logger/base_logger.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Dict, TYPE_CHECKING, Union
+import sqlite3
+import uuid
+
+from openai import OpenAI, AzureOpenAI
+from openai.types.chat import ChatCompletion
+
+if TYPE_CHECKING:
+    from autogen import ConversableAgent, OpenAIWrapper
+
+
+class BaseLogger(ABC):
+    @abstractmethod
+    def start(self) -> str:
+        """
+        Open a connection to the logging database, and start recording.
+
+        Returns:
+            session_id (str):     a unique id for the logging session
+        """
+        ...
+
+    @abstractmethod
+    def log_chat_completion(
+        invocation_id: uuid.UUID,
+        client_id: int,
+        wrapper_id: int,
+        request: Dict,
+        response: Union[str, ChatCompletion],
+        is_cached: int,
+        cost: float,
+        start_time: str,
+    ) -> None:
+        """
+        Log a chat completion to database.
+
+        In AutoGen, chat completions are somewhat complicated because they are handled by the `autogen.oai.OpenAIWrapper` class.
+        One invocation to `create` can lead to multiple underlying OpenAI calls, depending on the llm_config list used, and
+        any errors or retries.
+
+        Args:
+            invocation_id (uuid):               A unique identifier for the invocation to the OpenAIWrapper.create method call
+            client_id (int):                    A unique identifier for the underlying OpenAI client instance
+            wrapper_id (int):                   A unique identifier for the OpenAIWrapper instance
+            request (dict):                     A dictionary representing the the request or call to the OpenAI client endpoint
+            response (str or ChatCompletion):   The response from OpenAI
+            is_chached (int):                   1 if the response was a cache hit, 0 otherwise
+            cost(float):                        The cost for OpenAI response
+            start_time (str):                   A string representing the moment the request was initiated
+        """
+        ...
+
+    @abstractmethod
+    def log_new_agent(agent: ConversableAgent, init_args: Dict) -> None:
+        """
+        Log the birth of a new agent.
+
+        Args:
+            agent (ConversableAgent):   The agent to log.
+            init_args (dict):           The arguments passed to the construct the conversable agent
+        """
+        ...
+
+    @abstractmethod
+    def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict) -> None:
+        """
+        Log the birth of a new OpenAIWrapper.
+
+        Args:
+            wrapper (OpenAIWrapper):    The wrapper to log.
+            init_args (dict):           The arguments passed to the construct the wrapper
+        """
+        ...
+
+    @abstractmethod
+    def log_new_client(client: Union[AzureOpenAI, OpenAI], wrapper: OpenAIWrapper, init_args: Dict) -> None:
+        """
+        Log the birth of a new OpenAIWrapper.
+
+        Args:
+            wrapper (OpenAI):           The OpenAI client to log.
+            init_args (dict):           The arguments passed to the construct the client
+        """
+        ...
+
+    @abstractmethod
+    def stop() -> None:
+        """
+        Close the connection to the logging database, and stop logging.
+        """
+        ...
+
+    @abstractmethod
+    def get_connection() -> Union[sqlite3.Connection]:
+        """
+        Return a connection to the logging database.
+        """
+        ...

autogen/logger/logger_factory.py

@@ -0,0 +1,17 @@
+from typing import Any, Dict, Optional
+from autogen.logger.base_logger import BaseLogger
+from autogen.logger.sqlite_logger import SqliteLogger
+
+__all__ = ("LoggerFactory",)
+
+
+class LoggerFactory:
+    @staticmethod
+    def get_logger(logger_type: str = "sqlite", config: Optional[Dict[str, Any]] = None) -> BaseLogger:
+        if config is None:
+            config = {}
+
+        if logger_type == "sqlite":
+            return SqliteLogger(config)
+        else:
+            raise ValueError(f"[logger_factory] Unknown logger type: {logger_type}")

autogen/logger/logger_utils.py

@@ -0,0 +1,36 @@
+import datetime
+import inspect
+from typing import Any, Dict, List, Tuple, Union
+
+__all__ = ("get_current_ts", "to_dict")
+
+
+def get_current_ts():
+    return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")
+
+
+def to_dict(
+    obj: Union[int, float, str, bool, Dict[Any, Any], List[Any], Tuple[Any, ...], Any],
+    exclude: Tuple[str] = (),
+    no_recursive: Tuple[str] = (),
+) -> Any:
+    if isinstance(obj, (int, float, str, bool)):
+        return obj
+    elif callable(obj):
+        return inspect.getsource(obj).strip()
+    elif isinstance(obj, dict):
+        return {
+            str(k): to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive)
+            for k, v in obj.items()
+            if k not in exclude
+        }
+    elif isinstance(obj, (list, tuple)):
+        return [to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive) for v in obj]
+    elif hasattr(obj, "__dict__"):
+        return {
+            str(k): to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive)
+            for k, v in vars(obj).items()
+            if k not in exclude
+        }
+    else:
+        return obj