Skip to content

Commit

Permalink
style: adding progress bar to llm-based evaluators (#7726)
Browse files Browse the repository at this point in the history
* adding progress bar

* fixing typo

* fixing tests

* Update test_llm_evaluator.py

* fixing missing colon

* passing directly to parent

* adding docstrings
  • Loading branch information
davidsbatista committed May 23, 2024
1 parent 76224fc commit a4fc2b6
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 3 deletions.
5 changes: 4 additions & 1 deletion haystack/components/evaluators/context_relevance.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
def __init__(
self,
examples: Optional[List[Dict[str, Any]]] = None,
progress_bar: bool = True,
api: str = "openai",
api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
):
Expand All @@ -89,12 +90,13 @@ def __init__(
"statement_scores": [1],
},
}]
:param progress_bar:
Whether to show a progress bar during the evaluation.
:param api:
The API to use for calling an LLM through a Generator.
Supported APIs: "openai".
:param api_key:
The API key.
"""
self.instructions = (
"Your task is to judge how relevant the provided context is for answering a question. "
Expand All @@ -115,6 +117,7 @@ def __init__(
examples=self.examples,
api=self.api,
api_key=self.api_key,
progress_bar=progress_bar,
)

@component.output_types(individual_scores=List[int], score=float, results=List[Dict[str, Any]])
Expand Down
4 changes: 4 additions & 0 deletions haystack/components/evaluators/faithfulness.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
def __init__(
self,
examples: Optional[List[Dict[str, Any]]] = None,
progress_bar: bool = True,
api: str = "openai",
api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
):
Expand All @@ -104,6 +105,8 @@ def __init__(
"statement_scores": [1, 0],
},
}]
:param progress_bar:
Whether to show a progress bar during the evaluation.
:param api:
The API to use for calling an LLM through a Generator.
Supported APIs: "openai".
Expand Down Expand Up @@ -131,6 +134,7 @@ def __init__(
examples=self.examples,
api=self.api,
api_key=self.api_key,
progress_bar=progress_bar,
)

@component.output_types(individual_scores=List[int], score=float, results=List[Dict[str, Any]])
Expand Down
10 changes: 8 additions & 2 deletions haystack/components/evaluators/llm_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import json
from typing import Any, Dict, List, Tuple, Type

from tqdm import tqdm

from haystack import component, default_from_dict, default_to_dict
from haystack.components.builders import PromptBuilder
from haystack.components.generators import OpenAIGenerator
Expand Down Expand Up @@ -50,6 +52,7 @@ def __init__(
inputs: List[Tuple[str, Type[List]]],
outputs: List[str],
examples: List[Dict[str, Any]],
progress_bar: bool = True,
*,
api: str = "openai",
api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
Expand All @@ -70,6 +73,8 @@ def __init__(
`outputs` parameters.
Each example is a dictionary with keys "inputs" and "outputs"
They contain the input and output as dictionaries respectively.
:param progress_bar:
Whether to show a progress bar during the evaluation.
:param api:
The API to use for calling an LLM through a Generator.
Supported APIs: "openai".
Expand All @@ -78,13 +83,13 @@ def __init__(
"""
self.validate_init_parameters(inputs, outputs, examples)

self.instructions = instructions
self.inputs = inputs
self.outputs = outputs
self.examples = examples
self.api = api
self.api_key = api_key
self.progress_bar = progress_bar

if api == "openai":
self.generator = OpenAIGenerator(
Expand Down Expand Up @@ -173,7 +178,7 @@ def run(self, **inputs) -> Dict[str, Any]:
list_of_input_names_to_values = [dict(zip(input_names, v)) for v in values]

results = []
for input_names_to_values in list_of_input_names_to_values:
for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
prompt = self.builder.run(**input_names_to_values)
result = self.generator.run(prompt=prompt["prompt"])

Expand Down Expand Up @@ -243,6 +248,7 @@ def to_dict(self) -> Dict[str, Any]:
examples=self.examples,
api=self.api,
api_key=self.api_key.to_dict(),
progress_bar=self.progress_bar,
)

@classmethod
Expand Down
2 changes: 2 additions & 0 deletions test/components/evaluators/test_llm_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ def test_to_dict_default(self, monkeypatch):
"instructions": "test-instruction",
"inputs": [("predicted_answers", List[str])],
"outputs": ["score"],
"progress_bar": True,
"examples": [
{"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
],
Expand Down Expand Up @@ -266,6 +267,7 @@ def test_to_dict_with_parameters(self, monkeypatch):
"instructions": "test-instruction",
"inputs": [("predicted_answers", List[str])],
"outputs": ["custom_score"],
"progress_bar": True,
"examples": [
{
"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"},
Expand Down

0 comments on commit a4fc2b6

Please sign in to comment.