Fix docstrings

scripts/docs/conf.py

@@ -152,6 +152,12 @@
 myst_heading_anchors = 5
 
 
+# allow annotation for __call__ methods
+autodoc_default_options = {
+    'special-members': '__call__',
+}
+
+
 def setup(app):
     # Add the gallery directive
     app.add_directive("gallery-grid", GalleryDirective)

src/promptflow-evals/promptflow/evals/evaluators/_chat/_chat.py

@@ -22,39 +22,40 @@
 
 
 class ChatEvaluator:
+    """
+    Initialize a chat evaluator configured for a specific Azure OpenAI model.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: AzureOpenAIModelConfiguration
+    :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
+        focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
+    :type eval_last_turn: bool
+    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
+        Default is True.
+    :type parallel: bool
+    :return: A function that evaluates and generates metrics for "chat" scenario.
+    :rtype: function
+
+    **Usage**
+
+    .. code-block:: python
+
+        chat_eval = ChatEvaluator(model_config)
+        conversation = [
+            {"role": "user", "content": "What is the value of 2 + 2?"},
+            {"role": "assistant", "content": "2 + 2 = 4", "context": {
+                "citations": [
+                        {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
+                        ]
+                }
+            }
+        ]
+        result = chat_eval(conversation=conversation)
+    """
+
     def __init__(
         self, model_config: AzureOpenAIModelConfiguration, eval_last_turn: bool = False, parallel: bool = True
     ):
-        """
-        Initialize an evaluator configured for a specific Azure OpenAI model.
-
-        :param model_config: Configuration for the Azure OpenAI model.
-        :type model_config: AzureOpenAIModelConfiguration
-        :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
-            focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
-        :type eval_last_turn: bool
-        :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
-            Default is True.
-        :type parallel: bool
-        :return: A function that evaluates and generates metrics for "chat" scenario.
-        :rtype: function
-
-        **Usage**
-
-        .. code-block:: python
-
-            chat_eval = ChatEvaluator(model_config)
-            conversation = [
-                {"role": "user", "content": "What is the value of 2 + 2?"},
-                {"role": "assistant", "content": "2 + 2 = 4", "context": {
-                    "citations": [
-                            {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
-                            ]
-                    }
-                }
-            ]
-            result = chat_eval(conversation=conversation)
-        """
         self._eval_last_turn = eval_last_turn
         self._parallel = parallel
 
@@ -73,7 +74,8 @@ def __init__(
         self._retrieval_chat_evaluator = RetrievalChatEvaluator(model_config)
 
     def __call__(self, *, conversation, **kwargs):
-        """Evaluates chat scenario.
+        """
+        Evaluates chat scenario.
 
         :param conversation: The conversation to be evaluated. Each turn should have "role" and "content" keys.
             "context" key is optional for assistant's turn and should have "citations" key with list of citations.

src/promptflow-evals/promptflow/evals/evaluators/_coherence/_coherence.py

@@ -12,22 +12,23 @@
 
 
 class CoherenceEvaluator:
-    def __init__(self, model_config: AzureOpenAIModelConfiguration):
-        """
-        Initialize an evaluator configured for a specific Azure OpenAI model.
+    """
+    Initialize a coherence evaluator configured for a specific Azure OpenAI model.
 
-        :param model_config: Configuration for the Azure OpenAI model.
-        :type model_config: AzureOpenAIModelConfiguration
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: AzureOpenAIModelConfiguration
 
-        **Usage**
+    **Usage**
 
-        .. code-block:: python
+    .. code-block:: python
 
-            eval_fn = CoherenceEvaluator(model_config)
-            result = eval_fn(
-                question="What is the capital of Japan?",
-                answer="The capital of Japan is Tokyo.")
-        """
+        eval_fn = CoherenceEvaluator(model_config)
+        result = eval_fn(
+            question="What is the capital of Japan?",
+            answer="The capital of Japan is Tokyo.")
+    """
+
+    def __init__(self, model_config: AzureOpenAIModelConfiguration):
         # TODO: Remove this block once the bug is fixed
         # https://msdata.visualstudio.com/Vienna/_workitems/edit/3151324
         if model_config.api_version is None:
@@ -39,7 +40,9 @@ def __init__(self, model_config: AzureOpenAIModelConfiguration):
         self._flow = load_flow(source=prompty_path, model=prompty_model_config)
 
     def __call__(self, *, question: str, answer: str, **kwargs):
-        """Evaluate coherence.
+        """
+        Evaluate coherence.
+
         :param question: The question to be evaluated.
         :type question: str
         :param answer: The answer to be evaluated.

src/promptflow-evals/promptflow/evals/evaluators/_content_safety/_content_safety.py

@@ -7,35 +7,36 @@
 
 
 class ContentSafetyEvaluator:
-    def __init__(self, project_scope: dict, parallel: bool = True, credential=None):
-        """
-        Initialize an evaluator configured to evaluate content safetry metrics for QA scenario.
+    """
+    Initialize a content safety evaluator configured to evaluate content safetry metrics for QA scenario.
 
-        :param project_scope: The scope of the Azure AI project.
-            It contains subscription id, resource group, and project name.
-        :type project_scope: dict
-        :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
-            Default is True.
-        :param credential: The credential for connecting to Azure AI project.
-        :type credential: TokenCredential
-        :return: A function that evaluates content-safety metrics for "question-answering" scenario.
-        :rtype: function
+    :param project_scope: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type project_scope: dict
+    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
+        Default is True.
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: TokenCredential
+    :return: A function that evaluates content-safety metrics for "question-answering" scenario.
+    :rtype: function
 
-        **Usage**
+    **Usage**
 
-        .. code-block:: python
+    .. code-block:: python
 
-            project_scope = {
-                "subscription_id": "<subscription_id>",
-                "resource_group_name": "<resource_group_name>",
-                "project_name": "<project_name>",
-            }
-            eval_fn = ContentSafetyEvaluator(project_scope)
-            result = eval_fn(
-                question="What is the capital of France?",
-                answer="Paris.",
-            )
-        """
+        project_scope = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ContentSafetyEvaluator(project_scope)
+        result = eval_fn(
+            question="What is the capital of France?",
+            answer="Paris.",
+        )
+    """
+
+    def __init__(self, project_scope: dict, parallel: bool = True, credential=None):
         self._parallel = parallel
         self._evaluators = [
             ViolenceEvaluator(project_scope, credential),
@@ -45,7 +46,8 @@ def __init__(self, project_scope: dict, parallel: bool = True, credential=None):
         ]
 
     def __call__(self, *, question: str, answer: str, **kwargs):
-        """Evaluates content-safety metrics for "question-answering" scenario.
+        """
+        Evaluates content-safety metrics for "question-answering" scenario.
 
         :param question: The question to be evaluated.
         :type question: str

src/promptflow-evals/promptflow/evals/evaluators/_content_safety/_content_safety_chat.py

@@ -13,35 +13,36 @@
 
 
 class ContentSafetyChatEvaluator:
+    """
+    Initialize a content safety chat evaluator configured to evaluate content safetry metrics for chat scenario.
+
+    :param project_scope: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type project_scope: dict
+    :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
+        focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
+    :type eval_last_turn: bool
+    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
+        Default is True.
+    :type parallel: bool
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: TokenCredential
+    :return: A function that evaluates and generates metrics for "chat" scenario.
+    :rtype: function
+
+    **Usage**
+
+    .. code-block:: python
+
+        eval_fn = ContentSafetyChatEvaluator(model_config)
+        conversation = [
+            {"role": "user", "content": "What is the value of 2 + 2?"},
+            {"role": "assistant", "content": "2 + 2 = 4"}
+        ]
+        result = ContentSafetyChatEvaluator(conversation=conversation)
+    """
+
     def __init__(self, project_scope: dict, eval_last_turn: bool = False, parallel: bool = True, credential=None):
-        """
-        Initialize an evaluator configured to evaluate content safetry metrics for chat scenario.
-
-        :param project_scope: The scope of the Azure AI project.
-            It contains subscription id, resource group, and project name.
-        :type project_scope: dict
-        :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
-            focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
-        :type eval_last_turn: bool
-        :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
-            Default is True.
-        :type parallel: bool
-        :param credential: The credential for connecting to Azure AI project.
-        :type credential: TokenCredential
-        :return: A function that evaluates and generates metrics for "chat" scenario.
-        :rtype: function
-
-        **Usage**
-
-        .. code-block:: python
-
-            eval_fn = ContentSafetyChatEvaluator(model_config)
-            conversation = [
-                {"role": "user", "content": "What is the value of 2 + 2?"},
-                {"role": "assistant", "content": "2 + 2 = 4"}
-            ]
-            result = ContentSafetyChatEvaluator(conversation=conversation)
-        """
         self._eval_last_turn = eval_last_turn
         self._parallel = parallel
         self._evaluators = [
@@ -52,7 +53,8 @@ def __init__(self, project_scope: dict, eval_last_turn: bool = False, parallel:
         ]
 
     def __call__(self, *, conversation, **kwargs):
-        """Evaluates content-safety metrics for "chat" scenario.
+        """
+        Evaluates content-safety metrics for "chat" scenario.
 
         :param conversation: The conversation to be evaluated. Each turn should have "role" and "content" keys.
         :type conversation: List[Dict]

src/promptflow-evals/promptflow/evals/evaluators/_content_safety/_hate_unfairness.py

@@ -6,29 +6,29 @@
 
 
 class HateUnfairnessEvaluator:
-    def __init__(self, project_scope: dict, credential=None):
-        """
-        Initialize an evaluator for hate unfairness score.
+    """
+    Initialize a hate-unfairness evaluator for hate unfairness score.
 
-        :param project_scope: The scope of the Azure AI project.
-            It contains subscription id, resource group, and project name.
-        :type project_scope: dict
-        :param credential: The credential for connecting to Azure AI project.
-        :type credential: TokenCredential
+    :param project_scope: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type project_scope: dict
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: TokenCredential
 
-        **Usage**
+    **Usage**
 
-        .. code-block:: python
+    .. code-block:: python
 
-            project_scope = {
-                "subscription_id": "<subscription_id>",
-                "resource_group_name": "<resource_group_name>",
-                "project_name": "<project_name>",
-            }
-            eval_fn = HateUnfairnessEvaluator(project_scope)
-            result = eval_fn(question="What is the capital of France?", answer="Paris.")
-        """
+        project_scope = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = HateUnfairnessEvaluator(project_scope)
+        result = eval_fn(question="What is the capital of France?", answer="Paris.")
+    """
 
+    def __init__(self, project_scope: dict, credential=None):
         self._project_scope = project_scope
         self._credential = credential
 
@@ -38,7 +38,8 @@ def __init__(self, project_scope: dict, credential=None):
         self._flow = load_flow(source=flow_dir)
 
     def __call__(self, *, question: str, answer: str, **kwargs):
-        """Evaluates hate unfairness content.
+        """
+        Evaluates hate unfairness content.
 
         :param question: The question to be evaluated.
         :type question: str