From 76224fc781cd194a6550d8f4f3420bd3c0f4c32b Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Wed, 22 May 2024 13:14:39 +0200 Subject: [PATCH 1/3] make SerperDevWebSearch more robust (#7725) --- haystack/components/websearch/serper_dev.py | 2 +- .../serperdev-more-robust-229ba25c8fc9306d.yaml | 4 ++++ test/components/websearch/test_serperdev.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/serperdev-more-robust-229ba25c8fc9306d.yaml diff --git a/haystack/components/websearch/serper_dev.py b/haystack/components/websearch/serper_dev.py index fa85a1c997..3256199c5c 100644 --- a/haystack/components/websearch/serper_dev.py +++ b/haystack/components/websearch/serper_dev.py @@ -124,7 +124,7 @@ def run(self, query: str) -> Dict[str, Union[List[Document], List[str]]]: # we get the snippet from the json result and put it in the content field of the document organic = [ - Document(meta={k: v for k, v in d.items() if k != "snippet"}, content=d["snippet"]) + Document(meta={k: v for k, v in d.items() if k != "snippet"}, content=d.get("snippet")) for d in json_result["organic"] ] diff --git a/releasenotes/notes/serperdev-more-robust-229ba25c8fc9306d.yaml b/releasenotes/notes/serperdev-more-robust-229ba25c8fc9306d.yaml new file mode 100644 index 0000000000..9717400095 --- /dev/null +++ b/releasenotes/notes/serperdev-more-robust-229ba25c8fc9306d.yaml @@ -0,0 +1,4 @@ +--- +enhancements: + - | + Make the `SerperDevWebSearch` more robust when `snippet` is not present in the request response. diff --git a/test/components/websearch/test_serperdev.py b/test/components/websearch/test_serperdev.py index 1df191af6f..a2fe33344c 100644 --- a/test/components/websearch/test_serperdev.py +++ b/test/components/websearch/test_serperdev.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: 2022-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 + +import json import os from unittest.mock import Mock, patch from haystack.utils.auth import Secret @@ -111,6 +113,15 @@ def mock_serper_dev_search_result(): yield mock_run +@pytest.fixture +def mock_serper_dev_search_result_no_snippet(): + resp = {**EXAMPLE_SERPERDEV_RESPONSE} + del resp["organic"][0]["snippet"] + with patch("haystack.components.websearch.serper_dev.requests") as mock_run: + mock_run.post.return_value = Mock(status_code=200, json=lambda: resp) + yield mock_run + + class TestSerperDevSearchAPI: def test_init_fail_wo_api_key(self, monkeypatch): monkeypatch.delenv("SERPERDEV_API_KEY", raising=False) @@ -142,6 +153,10 @@ def test_web_search_top_k(self, mock_serper_dev_search_result, top_k: int): assert all(isinstance(link, str) for link in links) assert all(link.startswith("http") for link in links) + def test_no_snippet(self, mock_serper_dev_search_result_no_snippet): + ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), top_k=1) + ws.run(query="Who is the boyfriend of Olivia Wilde?") + @patch("requests.post") def test_timeout_error(self, mock_post): mock_post.side_effect = Timeout From a4fc2b66e67eed22d7c90297359706445c30078c Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 23 May 2024 09:22:14 +0200 Subject: [PATCH 2/3] style: adding progress bar to llm-based evaluators (#7726) * adding progress bar * fixing typo * fixing tests * Update test_llm_evaluator.py * fixing missing colon * passing directly to parent * adding docstrings --- haystack/components/evaluators/context_relevance.py | 5 ++++- haystack/components/evaluators/faithfulness.py | 4 ++++ haystack/components/evaluators/llm_evaluator.py | 10 ++++++++-- test/components/evaluators/test_llm_evaluator.py | 2 ++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/haystack/components/evaluators/context_relevance.py b/haystack/components/evaluators/context_relevance.py index 9988bdeb02..9bd299bbc5 100644 --- a/haystack/components/evaluators/context_relevance.py +++ b/haystack/components/evaluators/context_relevance.py @@ -67,6 +67,7 @@ class ContextRelevanceEvaluator(LLMEvaluator): def __init__( self, examples: Optional[List[Dict[str, Any]]] = None, + progress_bar: bool = True, api: str = "openai", api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"), ): @@ -89,12 +90,13 @@ def __init__( "statement_scores": [1], }, }] + :param progress_bar: + Whether to show a progress bar during the evaluation. :param api: The API to use for calling an LLM through a Generator. Supported APIs: "openai". :param api_key: The API key. - """ self.instructions = ( "Your task is to judge how relevant the provided context is for answering a question. " @@ -115,6 +117,7 @@ def __init__( examples=self.examples, api=self.api, api_key=self.api_key, + progress_bar=progress_bar, ) @component.output_types(individual_scores=List[int], score=float, results=List[Dict[str, Any]]) diff --git a/haystack/components/evaluators/faithfulness.py b/haystack/components/evaluators/faithfulness.py index 2bcbb9b086..1e561f6693 100644 --- a/haystack/components/evaluators/faithfulness.py +++ b/haystack/components/evaluators/faithfulness.py @@ -81,6 +81,7 @@ class FaithfulnessEvaluator(LLMEvaluator): def __init__( self, examples: Optional[List[Dict[str, Any]]] = None, + progress_bar: bool = True, api: str = "openai", api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"), ): @@ -104,6 +105,8 @@ def __init__( "statement_scores": [1, 0], }, }] + :param progress_bar: + Whether to show a progress bar during the evaluation. :param api: The API to use for calling an LLM through a Generator. Supported APIs: "openai". @@ -131,6 +134,7 @@ def __init__( examples=self.examples, api=self.api, api_key=self.api_key, + progress_bar=progress_bar, ) @component.output_types(individual_scores=List[int], score=float, results=List[Dict[str, Any]]) diff --git a/haystack/components/evaluators/llm_evaluator.py b/haystack/components/evaluators/llm_evaluator.py index e4eebbd9ab..9766f236ad 100644 --- a/haystack/components/evaluators/llm_evaluator.py +++ b/haystack/components/evaluators/llm_evaluator.py @@ -5,6 +5,8 @@ import json from typing import Any, Dict, List, Tuple, Type +from tqdm import tqdm + from haystack import component, default_from_dict, default_to_dict from haystack.components.builders import PromptBuilder from haystack.components.generators import OpenAIGenerator @@ -50,6 +52,7 @@ def __init__( inputs: List[Tuple[str, Type[List]]], outputs: List[str], examples: List[Dict[str, Any]], + progress_bar: bool = True, *, api: str = "openai", api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"), @@ -70,6 +73,8 @@ def __init__( `outputs` parameters. Each example is a dictionary with keys "inputs" and "outputs" They contain the input and output as dictionaries respectively. + :param progress_bar: + Whether to show a progress bar during the evaluation. :param api: The API to use for calling an LLM through a Generator. Supported APIs: "openai". @@ -78,13 +83,13 @@ def __init__( """ self.validate_init_parameters(inputs, outputs, examples) - self.instructions = instructions self.inputs = inputs self.outputs = outputs self.examples = examples self.api = api self.api_key = api_key + self.progress_bar = progress_bar if api == "openai": self.generator = OpenAIGenerator( @@ -173,7 +178,7 @@ def run(self, **inputs) -> Dict[str, Any]: list_of_input_names_to_values = [dict(zip(input_names, v)) for v in values] results = [] - for input_names_to_values in list_of_input_names_to_values: + for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar): prompt = self.builder.run(**input_names_to_values) result = self.generator.run(prompt=prompt["prompt"]) @@ -243,6 +248,7 @@ def to_dict(self) -> Dict[str, Any]: examples=self.examples, api=self.api, api_key=self.api_key.to_dict(), + progress_bar=self.progress_bar, ) @classmethod diff --git a/test/components/evaluators/test_llm_evaluator.py b/test/components/evaluators/test_llm_evaluator.py index b1d41e000c..1b28dab84e 100644 --- a/test/components/evaluators/test_llm_evaluator.py +++ b/test/components/evaluators/test_llm_evaluator.py @@ -206,6 +206,7 @@ def test_to_dict_default(self, monkeypatch): "instructions": "test-instruction", "inputs": [("predicted_answers", List[str])], "outputs": ["score"], + "progress_bar": True, "examples": [ {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}} ], @@ -266,6 +267,7 @@ def test_to_dict_with_parameters(self, monkeypatch): "instructions": "test-instruction", "inputs": [("predicted_answers", List[str])], "outputs": ["custom_score"], + "progress_bar": True, "examples": [ { "inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, From 482f60ec99d5f13166c3f8f01d948fb4305cec28 Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Thu, 23 May 2024 09:35:10 +0200 Subject: [PATCH 3/3] fix: exit early if the component receives no documents (#7732) * exit early if the component receives no documents * relnote --- haystack/components/readers/extractive.py | 8 +++++++- .../notes/reader-crash-no-docs-53085ce48baaae81.yaml | 4 ++++ test/components/readers/test_extractive.py | 10 +++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/reader-crash-no-docs-53085ce48baaae81.yaml diff --git a/haystack/components/readers/extractive.py b/haystack/components/readers/extractive.py index 62e1ce0022..29bee01d3b 100644 --- a/haystack/components/readers/extractive.py +++ b/haystack/components/readers/extractive.py @@ -210,6 +210,7 @@ def _preprocess( """ texts = [] document_ids = [] + document_contents = [] for i, doc in enumerate(documents): if doc.content is None: warnings.warn( @@ -219,9 +220,11 @@ def _preprocess( continue texts.append(doc.content) document_ids.append(i) + document_contents.append(doc.content) + encodings_pt = self.tokenizer( # type: ignore queries, - [document.content for document in documents], + document_contents, padding=True, truncation=True, max_length=max_seq_length, @@ -571,6 +574,9 @@ def run( :raises ComponentError: If the component was not warmed up by calling 'warm_up()' before. """ + if not documents: + return {"answers": []} + queries = [query] # Temporary solution until we have decided what batching should look like in v2 nested_documents = [documents] if self.model is None: diff --git a/releasenotes/notes/reader-crash-no-docs-53085ce48baaae81.yaml b/releasenotes/notes/reader-crash-no-docs-53085ce48baaae81.yaml new file mode 100644 index 0000000000..cf0cd02a89 --- /dev/null +++ b/releasenotes/notes/reader-crash-no-docs-53085ce48baaae81.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Return an empty list of answers when `ExtractiveReader` receives an empty list of documents instead of raising an exception. diff --git a/test/components/readers/test_extractive.py b/test/components/readers/test_extractive.py index e6a0fca83f..f9a161e804 100644 --- a/test/components/readers/test_extractive.py +++ b/test/components/readers/test_extractive.py @@ -266,13 +266,17 @@ def test_from_dict_no_token(): assert component.token is None +def test_run_no_docs(mock_reader: ExtractiveReader): + mock_reader.warm_up() + assert mock_reader.run(query="hello", documents=[]) == {"answers": []} + + def test_output(mock_reader: ExtractiveReader): - answers = mock_reader.run(example_queries[0], example_documents[0], top_k=3)[ - "answers" - ] # [0] Uncomment and remove first two indices when batching support is reintroduced + answers = mock_reader.run(example_queries[0], example_documents[0], top_k=3)["answers"] doc_ids = set() no_answer_prob = 1 for doc, answer in zip(example_documents[0], answers[:3]): + assert answer.document_offset is not None assert answer.document_offset.start == 11 assert answer.document_offset.end == 16 assert doc.content is not None