From 6d9d11c0a5e52b82d0aa988fc99a83bbe00ee3c5 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 09:53:16 +0100
Subject: [PATCH 01/16] Updated DefaultEmbeddingEncoder to call model.eval()
 and added test.

---
 .../nodes/retriever/_embedding_encoder.py     |  1 +
 test/nodes/test_retriever.py                  | 25 +++++++++++++++----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/haystack/nodes/retriever/_embedding_encoder.py b/haystack/nodes/retriever/_embedding_encoder.py
index 775c813cbe..2d195a976c 100644
--- a/haystack/nodes/retriever/_embedding_encoder.py
+++ b/haystack/nodes/retriever/_embedding_encoder.py
@@ -138,6 +138,7 @@ def __init__(self, retriever: "EmbeddingRetriever"):
     def embed(self, texts: Union[List[List[str]], List[str], str]) -> np.ndarray:
         # TODO: FARM's `sample_to_features_text` need to fix following warning -
         # tokenization_utils.py:460: FutureWarning: `is_pretokenized` is deprecated and will be removed in a future version, use `is_split_into_words` instead.
+        self.embedding_model.model.eval()
         emb = self.embedding_model.inference_from_dicts(dicts=[{"text": t} for t in texts])
         emb = np.stack([r["vec"] for r in emb])
         return emb
diff --git a/test/nodes/test_retriever.py b/test/nodes/test_retriever.py
index fea647d49a..b122ad91a4 100644
--- a/test/nodes/test_retriever.py
+++ b/test/nodes/test_retriever.py
@@ -1,12 +1,10 @@
-from typing import List
-
-import os
 import logging
 import os
 from math import isclose
 from typing import Dict, List, Optional, Union
 
 import pytest
+import torch
 import numpy as np
 import pandas as pd
 from pandas.testing import assert_frame_equal
@@ -20,8 +18,6 @@
 from haystack.pipelines import DocumentSearchPipeline
 from haystack.schema import Document
 from haystack.document_stores.elasticsearch import ElasticsearchDocumentStore
-from haystack.document_stores.faiss import FAISSDocumentStore
-from haystack.document_stores import MilvusDocumentStore
 from haystack.nodes.retriever.dense import DensePassageRetriever, EmbeddingRetriever, TableTextRetriever
 from haystack.nodes.retriever.sparse import BM25Retriever, FilterRetriever, TfidfRetriever
 from haystack.nodes.retriever.multimodal import MultiModalRetriever
@@ -311,6 +307,25 @@ def test_dpr_embedding(document_store: BaseDocumentStore, retriever, docs_with_i
         assert isclose(embedding[0], expected_value, rel_tol=0.01)
 
 
+@pytest.mark.integration
+@pytest.mark.parametrize("document_store", ["memory"], indirect=True)
+@pytest.mark.parametrize("retriever", ["embedding"], indirect=True)
+def test_embedding_train_mode(retriever, docs_with_ids):
+    # Set to deterministic seed
+    old_seed = torch.seed()
+    torch.manual_seed(0)
+
+    embeddings_default_mode = retriever.embed_documents(documents=docs_with_ids)
+    retriever.embedding_encoder.embedding_model.model.train()
+    embeddings_train_mode = retriever.embed_documents(documents=docs_with_ids)
+
+    for emb1, emb2 in zip(embeddings_train_mode, embeddings_default_mode):
+        assert any(np.isclose(emb1, emb2, rtol=1.0e-5))
+
+    # Set back to old_seed
+    torch.manual_seed(old_seed)
+
+
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "document_store", ["elasticsearch", "faiss", "memory", "milvus", "weaviate", "pinecone"], indirect=True

From 01d22b5f7388dfc335a8a96426f123f266d2f749 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 10:06:49 +0100
Subject: [PATCH 02/16] Updated retribert retriever to set model.eval() and
 added test

---
 haystack/nodes/retriever/_embedding_encoder.py | 2 ++
 test/nodes/test_retriever.py                   | 8 ++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/haystack/nodes/retriever/_embedding_encoder.py b/haystack/nodes/retriever/_embedding_encoder.py
index 2d195a976c..611c5eea9f 100644
--- a/haystack/nodes/retriever/_embedding_encoder.py
+++ b/haystack/nodes/retriever/_embedding_encoder.py
@@ -297,6 +297,7 @@ def embed_queries(self, queries: List[str]) -> np.ndarray:
         :param queries: List of queries to embed.
         :return: Embeddings, one per input query, shape: (queries, embedding_dim)
         """
+        self.embedding_model.eval()
         query_text = [{"text": q} for q in queries]
         dataloader = self._create_dataloader(query_text)
 
@@ -324,6 +325,7 @@ def embed_documents(self, docs: List[Document]) -> np.ndarray:
         :param docs: List of documents to embed.
         :return: Embeddings, one per input document, shape: (documents, embedding_dim)
         """
+        self.embedding_model.eval()
         doc_text = [{"text": d.content} for d in docs]
         dataloader = self._create_dataloader(doc_text)
 
diff --git a/test/nodes/test_retriever.py b/test/nodes/test_retriever.py
index b122ad91a4..8a073ad5a4 100644
--- a/test/nodes/test_retriever.py
+++ b/test/nodes/test_retriever.py
@@ -21,6 +21,7 @@
 from haystack.nodes.retriever.dense import DensePassageRetriever, EmbeddingRetriever, TableTextRetriever
 from haystack.nodes.retriever.sparse import BM25Retriever, FilterRetriever, TfidfRetriever
 from haystack.nodes.retriever.multimodal import MultiModalRetriever
+from haystack.nodes.retriever._embedding_encoder import _RetribertEmbeddingEncoder
 
 from ..conftest import SAMPLES_PATH, MockRetriever
 
@@ -309,14 +310,17 @@ def test_dpr_embedding(document_store: BaseDocumentStore, retriever, docs_with_i
 
 @pytest.mark.integration
 @pytest.mark.parametrize("document_store", ["memory"], indirect=True)
-@pytest.mark.parametrize("retriever", ["embedding"], indirect=True)
+@pytest.mark.parametrize("retriever", ["embedding", "retribert"], indirect=True)
 def test_embedding_train_mode(retriever, docs_with_ids):
     # Set to deterministic seed
     old_seed = torch.seed()
     torch.manual_seed(0)
 
     embeddings_default_mode = retriever.embed_documents(documents=docs_with_ids)
-    retriever.embedding_encoder.embedding_model.model.train()
+    if isinstance(retriever.embedding_encoder, _RetribertEmbeddingEncoder):
+        retriever.embedding_encoder.embedding_model.train()
+    else:
+        retriever.embedding_encoder.embedding_model.model.train()
     embeddings_train_mode = retriever.embed_documents(documents=docs_with_ids)
 
     for emb1, emb2 in zip(embeddings_train_mode, embeddings_default_mode):

From 8505cb357d63e7cacfe2753d733b3601025db619 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 10:17:15 +0100
Subject: [PATCH 03/16] Updated test to test
 _SentenceTransformersEmbeddingEncoder

---
 test/nodes/test_retriever.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/nodes/test_retriever.py b/test/nodes/test_retriever.py
index 8a073ad5a4..c572855740 100644
--- a/test/nodes/test_retriever.py
+++ b/test/nodes/test_retriever.py
@@ -21,7 +21,7 @@
 from haystack.nodes.retriever.dense import DensePassageRetriever, EmbeddingRetriever, TableTextRetriever
 from haystack.nodes.retriever.sparse import BM25Retriever, FilterRetriever, TfidfRetriever
 from haystack.nodes.retriever.multimodal import MultiModalRetriever
-from haystack.nodes.retriever._embedding_encoder import _RetribertEmbeddingEncoder
+from haystack.nodes.retriever._embedding_encoder import _DefaultEmbeddingEncoder
 
 from ..conftest import SAMPLES_PATH, MockRetriever
 
@@ -310,17 +310,17 @@ def test_dpr_embedding(document_store: BaseDocumentStore, retriever, docs_with_i
 
 @pytest.mark.integration
 @pytest.mark.parametrize("document_store", ["memory"], indirect=True)
-@pytest.mark.parametrize("retriever", ["embedding", "retribert"], indirect=True)
+@pytest.mark.parametrize("retriever", ["embedding", "retribert", "embedding_sbert"], indirect=True)
 def test_embedding_train_mode(retriever, docs_with_ids):
     # Set to deterministic seed
     old_seed = torch.seed()
     torch.manual_seed(0)
 
     embeddings_default_mode = retriever.embed_documents(documents=docs_with_ids)
-    if isinstance(retriever.embedding_encoder, _RetribertEmbeddingEncoder):
-        retriever.embedding_encoder.embedding_model.train()
-    else:
+    if isinstance(retriever.embedding_encoder, _DefaultEmbeddingEncoder):
         retriever.embedding_encoder.embedding_model.model.train()
+    else:
+        retriever.embedding_encoder.embedding_model.train()
     embeddings_train_mode = retriever.embed_documents(documents=docs_with_ids)
 
     for emb1, emb2 in zip(embeddings_train_mode, embeddings_default_mode):

From 4da4c14393d906e97f128348c058a558f4bed7d1 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 12:59:37 +0100
Subject: [PATCH 04/16] Set model.eval() in FARMReader and TransformerReader
 when running predict or predict_batch. Added test that failed previously.

---
 haystack/nodes/reader/farm.py         |  2 ++
 haystack/nodes/reader/transformers.py |  2 ++
 test/nodes/test_reader.py             | 39 +++++++++++++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/haystack/nodes/reader/farm.py b/haystack/nodes/reader/farm.py
index b82c40205b..fcd712dfa7 100644
--- a/haystack/nodes/reader/farm.py
+++ b/haystack/nodes/reader/farm.py
@@ -815,6 +815,7 @@ def predict_batch(
         :param top_k: Number of returned answers per query.
         :param batch_size: Number of query-document pairs to be processed at a time.
         """
+        self.inferencer.model.eval()
         if top_k is None:
             top_k = self.top_k
 
@@ -881,6 +882,7 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
         :param top_k: The maximum number of answers to return
         :return: Dict containing query and answers
         """
+        self.inferencer.model.eval()
         if top_k is None:
             top_k = self.top_k
         # convert input to FARM format
diff --git a/haystack/nodes/reader/transformers.py b/haystack/nodes/reader/transformers.py
index e588caaee2..f89a10d07c 100644
--- a/haystack/nodes/reader/transformers.py
+++ b/haystack/nodes/reader/transformers.py
@@ -135,6 +135,7 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
         :param top_k: The maximum number of answers to return
         :return: Dict containing query and answers
         """
+        self.model.model.eval()
         if top_k is None:
             top_k = self.top_k
 
@@ -201,6 +202,7 @@ def predict_batch(
         :param top_k: Number of returned answers per query.
         :param batch_size: Number of query-document pairs to be processed at a time.
         """
+        self.model.model.eval()
         if top_k is None:
             top_k = self.top_k
 
diff --git a/test/nodes/test_reader.py b/test/nodes/test_reader.py
index ca7af5f44a..60b3efa20f 100644
--- a/test/nodes/test_reader.py
+++ b/test/nodes/test_reader.py
@@ -5,6 +5,7 @@
 
 import pytest
 
+import torch
 from huggingface_hub import snapshot_download
 from haystack.modeling.data_handler.inputs import QAInput, Question
 
@@ -56,6 +57,44 @@ def test_output(reader, docs):
     assert len(prediction["answers"]) == 5
 
 
+def test_output_train_mode(reader, docs):
+    # Set to deterministic seed
+    old_seed = torch.seed()
+    torch.manual_seed(0)
+
+    pred_default = reader.predict(query="Who lives in Berlin?", documents=docs, top_k=5)
+    if isinstance(reader, FARMReader):
+        reader.inferencer.model.train()
+    else:
+        reader.model.model.train()
+    pred_train_mode = reader.predict(query="Who lives in Berlin?", documents=docs, top_k=5)
+
+    assert pred_default["query"] == pred_train_mode["query"]
+    assert pred_default["answers"][0].answer == pred_train_mode["answers"][0].answer
+    assert (
+        pred_default["answers"][0].offsets_in_context[0].start
+        == pred_train_mode["answers"][0].offsets_in_context[0].start
+    )
+    assert (
+        pred_default["answers"][0].offsets_in_context[0].end == pred_train_mode["answers"][0].offsets_in_context[0].end
+    )
+    assert (
+        pred_default["answers"][0].offsets_in_document[0].start
+        == pred_train_mode["answers"][0].offsets_in_document[0].start
+    )
+    assert (
+        pred_default["answers"][0].offsets_in_document[0].end
+        == pred_train_mode["answers"][0].offsets_in_document[0].end
+    )
+    assert pred_default["answers"][0].type == pred_train_mode["answers"][0].type
+    assert math.isclose(pred_default["answers"][0].score, pred_train_mode["answers"][0].score, rel_tol=1e-4)
+    assert pred_default["answers"][0].context == pred_train_mode["answers"][0].context
+    assert len(pred_default["answers"]) == len(pred_train_mode["answers"])
+
+    # Set back to old_seed
+    torch.manual_seed(old_seed)
+
+
 def test_output_batch_single_query_single_doc_list(reader, docs):
     prediction = reader.predict_batch(queries=["Who lives in Berlin?"], documents=docs, top_k=5)
     assert prediction is not None

From 3ce1beff96c474f06a15dbb59b29b086830860b0 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 13:45:40 +0100
Subject: [PATCH 05/16] Added model.eval calls to _get_prediction methods of
 Inferencer

---
 haystack/modeling/infer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/haystack/modeling/infer.py b/haystack/modeling/infer.py
index dcc953ee6a..cd579c2d20 100644
--- a/haystack/modeling/infer.py
+++ b/haystack/modeling/infer.py
@@ -354,6 +354,7 @@ def _get_predictions(self, dataset: Dataset, tensor_names: List, baskets):
                         Example: QA - input string to convert the predicted answer from indices back to string space
         :return: list of predictions
         """
+        self.model.eval()
         samples = [s for b in baskets for s in b.samples]
 
         data_loader = NamedDataLoader(
@@ -390,6 +391,7 @@ def _get_predictions_and_aggregate(self, dataset: Dataset, tensor_names: List, b
                         Example: QA - input string to convert the predicted answer from indices back to string space
         :return: list of predictions
         """
+        self.model.eval()
         data_loader = NamedDataLoader(
             dataset=dataset, sampler=SequentialSampler(dataset), batch_size=self.batch_size, tensor_names=tensor_names
         )  # type ignore

From ebb58e4a0389979cf688d27ad19d1619f657a483 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 14:20:22 +0100
Subject: [PATCH 06/16] Added model.eval to PromptNode and DocumentClassifier.
 Also added new unit test for TextToSpeech.

---
 haystack/nodes/document_classifier/transformers.py |  1 +
 haystack/nodes/prompt/prompt_node.py               |  1 +
 test/nodes/test_audio.py                           | 11 +++++++++++
 3 files changed, 13 insertions(+)

diff --git a/haystack/nodes/document_classifier/transformers.py b/haystack/nodes/document_classifier/transformers.py
index 4d03af925f..9000b1872f 100644
--- a/haystack/nodes/document_classifier/transformers.py
+++ b/haystack/nodes/document_classifier/transformers.py
@@ -171,6 +171,7 @@ def predict(self, documents: List[Document], batch_size: Optional[int] = None) -
         :param batch_size: The number of Documents to classify at a time.
         :return: A list of Documents enriched with meta information.
         """
+        self.model.model.eval()
         if batch_size is None:
             batch_size = self.batch_size
 
diff --git a/haystack/nodes/prompt/prompt_node.py b/haystack/nodes/prompt/prompt_node.py
index d666775406..b9c6b3ea1d 100644
--- a/haystack/nodes/prompt/prompt_node.py
+++ b/haystack/nodes/prompt/prompt_node.py
@@ -296,6 +296,7 @@ def invoke(self, *args, **kwargs):
         It takes a prompt and returns a list of generated text using the local Hugging Face transformers model
         :return: A list of generated text.
         """
+        self.pipe.model.eval()
         output = []
         if kwargs and "prompt" in kwargs:
             prompt = kwargs.pop("prompt")
diff --git a/test/nodes/test_audio.py b/test/nodes/test_audio.py
index 1fdaf31b7f..af3aa2139d 100644
--- a/test/nodes/test_audio.py
+++ b/test/nodes/test_audio.py
@@ -19,6 +19,17 @@
 
 @pytest.mark.skipif(soundfile_not_found, reason="soundfile not found")
 class TestTextToSpeech:
+    def test_text_to_speech_train_mode(self):
+        text2speech = TextToSpeech(
+            model_name_or_path="espnet/kan-bayashi_ljspeech_vits",
+            transformers_params={"seed": 777, "always_fix_seed": True},
+        )
+        eval_mode_audio_data = text2speech.text_to_audio_data(text="answer")
+        text2speech.model.train()
+        train_mode_audio_data = text2speech.text_to_audio_data(text="answer")
+
+        assert np.allclose(train_mode_audio_data, eval_mode_audio_data)
+
     def test_text_to_speech_audio_data(self):
         text2speech = TextToSpeech(
             model_name_or_path="espnet/kan-bayashi_ljspeech_vits",

From 3c7299e8f14adafc471007b3af7cb965e4810e69 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 14:36:49 +0100
Subject: [PATCH 07/16] Added model.eval to Text2SparqlRetriever

---
 haystack/nodes/retriever/text2sparql.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/haystack/nodes/retriever/text2sparql.py b/haystack/nodes/retriever/text2sparql.py
index 5e96d1e091..528cc54dde 100644
--- a/haystack/nodes/retriever/text2sparql.py
+++ b/haystack/nodes/retriever/text2sparql.py
@@ -55,6 +55,7 @@ def retrieve(self, query: str, top_k: Optional[int] = None):
         :param query: Text query that shall be translated to SPARQL and then executed on the knowledge graph
         :param top_k: How many SPARQL queries to generate per text query.
         """
+        self.model.model.eval()
         if top_k is None:
             top_k = self.top_k
         inputs = self.tok([query], max_length=100, truncation=True, return_tensors="pt")

From a83eff1592239228e2a31587a8c1a78e716cddd3 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 14:42:59 +0100
Subject: [PATCH 08/16] Fix to unit test

---
 test/nodes/test_audio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/nodes/test_audio.py b/test/nodes/test_audio.py
index af3aa2139d..d82199724a 100644
--- a/test/nodes/test_audio.py
+++ b/test/nodes/test_audio.py
@@ -25,7 +25,7 @@ def test_text_to_speech_train_mode(self):
             transformers_params={"seed": 777, "always_fix_seed": True},
         )
         eval_mode_audio_data = text2speech.text_to_audio_data(text="answer")
-        text2speech.model.train()
+        text2speech.model.model.train()
         train_mode_audio_data = text2speech.text_to_audio_data(text="answer")
 
         assert np.allclose(train_mode_audio_data, eval_mode_audio_data)

From 71b5a4e581aa22c2a06dac9deca37025b9443c7a Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 14:53:51 +0100
Subject: [PATCH 09/16] Added model.eval to TransformersTranslator

---
 haystack/nodes/translator/transformers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/haystack/nodes/translator/transformers.py b/haystack/nodes/translator/transformers.py
index e896114749..8b7b22281f 100644
--- a/haystack/nodes/translator/transformers.py
+++ b/haystack/nodes/translator/transformers.py
@@ -151,6 +151,7 @@ def translate(
             truncation=True,
         ).to(self.devices[0])
 
+        self.model.model.eval()
         generated_output = self.model.generate(**batch)
         translated_texts = self.tokenizer.batch_decode(
             generated_output, skip_special_tokens=True, clean_up_tokenization_spaces=self.clean_up_tokenization_spaces

From ab6eab1d304c79d2b38bfc7b1191184949ad6c7a Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 15:01:32 +0100
Subject: [PATCH 10/16] Added model.eval() to EntityExtractor and Text2Speech

---
 haystack/nodes/audio/_text_to_speech.py | 1 +
 haystack/nodes/extractor/entity.py      | 1 +
 2 files changed, 2 insertions(+)

diff --git a/haystack/nodes/audio/_text_to_speech.py b/haystack/nodes/audio/_text_to_speech.py
index 4e8d721136..a301d46e30 100644
--- a/haystack/nodes/audio/_text_to_speech.py
+++ b/haystack/nodes/audio/_text_to_speech.py
@@ -126,6 +126,7 @@ def text_to_audio_data(self, text: str, _models_output_key: str = "wav") -> np.a
         :param _models_output_key: The key in the prediction dictionary that contains the audio data. Defaults to 'wav'.
         :return: A numpy array representing the audio generated by the model.
         """
+        self.model.model.eval()
         prediction = self.model(text)
         if not prediction:
             raise AudioNodeError(
diff --git a/haystack/nodes/extractor/entity.py b/haystack/nodes/extractor/entity.py
index 7f1ce3d8d1..2469ec6d80 100644
--- a/haystack/nodes/extractor/entity.py
+++ b/haystack/nodes/extractor/entity.py
@@ -430,6 +430,7 @@ def extract(self, text: Union[str, List[str]], batch_size: int = 1):
         dataloader = DataLoader(dataset, shuffle=False, batch_size=batch_size, num_workers=self.num_workers)
 
         # Forward
+        self.model.model.eval()
         predictions: List[Dict[str, Any]] = []
         for batch in tqdm(dataloader, disable=not self.progress_bar, total=len(dataloader), desc="Extracting entities"):
             batch = ensure_tensor_on_device(batch, device=self.devices[0])

From 5284ab36d9943c474d1c66e4b9aac1d6db7f4fa4 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 15:12:23 +0100
Subject: [PATCH 11/16] Added model.eval to TransformersQueryClassifier and
 QuestionGenerator

---
 haystack/nodes/query_classifier/transformers.py         | 2 ++
 haystack/nodes/question_generator/question_generator.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/haystack/nodes/query_classifier/transformers.py b/haystack/nodes/query_classifier/transformers.py
index fdaf43bd09..0b3457f750 100644
--- a/haystack/nodes/query_classifier/transformers.py
+++ b/haystack/nodes/query_classifier/transformers.py
@@ -142,6 +142,7 @@ def _get_edge_number_from_label(self, label):
         return self.labels.index(label) + 1
 
     def run(self, query: str):  # type: ignore
+        self.model.model.eval()
         if self.task == "zero-shot-classification":
             prediction = self.model([query], candidate_labels=self.labels, truncation=True)
             label = prediction[0]["labels"][0]
@@ -155,6 +156,7 @@ def run_batch(self, queries: List[str], batch_size: Optional[int] = None):  # ty
         queries_dataset = ListDataset(queries)
         if batch_size is None:
             batch_size = self.batch_size
+        self.model.model.eval()
         all_predictions = []
         if self.task == "zero-shot-classification":
             for predictions in tqdm(
diff --git a/haystack/nodes/question_generator/question_generator.py b/haystack/nodes/question_generator/question_generator.py
index 3338312d86..b21bd6c31e 100644
--- a/haystack/nodes/question_generator/question_generator.py
+++ b/haystack/nodes/question_generator/question_generator.py
@@ -153,6 +153,7 @@ def generate(self, text: str) -> List[str]:
         input_ids = tokenized["input_ids"].to(self.devices[0])
         # Necessary if padding is enabled so the model won't attend pad tokens
         attention_mask = tokenized["attention_mask"].to(self.devices[0])
+        self.model.model.eval()
         tokens_output = self.model.generate(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -222,6 +223,7 @@ def generate_batch(
             input_ids = tokenized["input_ids"].to(self.devices[0])
             # Necessary if padding is enabled so the model won't attend pad tokens
             attention_mask = tokenized["attention_mask"].to(self.devices[0])
+            self.model.model.eval()
             tokens_output = self.model.generate(
                 input_ids=input_ids,
                 attention_mask=attention_mask,

From 1335c3296742847df5523e9e7e0f903b48bd2adc Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 15:21:58 +0100
Subject: [PATCH 12/16] Added model eval to RAGenerator

---
 haystack/nodes/answer_generator/transformers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/haystack/nodes/answer_generator/transformers.py b/haystack/nodes/answer_generator/transformers.py
index 298113a095..a35c4aae15 100644
--- a/haystack/nodes/answer_generator/transformers.py
+++ b/haystack/nodes/answer_generator/transformers.py
@@ -231,6 +231,8 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
         ```
         """
         torch.set_grad_enabled(False)
+        self.model.question_encoder.eval()
+        self.model.generator.eval()
         if len(documents) == 0:
             raise AttributeError("generator need documents to predict the answer")
 

From 09e887080f3a52545f885620567cd6a029fd3e52 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Fri, 30 Dec 2022 15:25:49 +0100
Subject: [PATCH 13/16] Added model eval to Seq2SeqGenerator

---
 haystack/nodes/answer_generator/transformers.py         | 1 +
 haystack/nodes/extractor/entity.py                      | 2 +-
 haystack/nodes/question_generator/question_generator.py | 4 ++--
 haystack/nodes/retriever/text2sparql.py                 | 2 +-
 haystack/nodes/translator/transformers.py               | 2 +-
 5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/haystack/nodes/answer_generator/transformers.py b/haystack/nodes/answer_generator/transformers.py
index a35c4aae15..43708e2c98 100644
--- a/haystack/nodes/answer_generator/transformers.py
+++ b/haystack/nodes/answer_generator/transformers.py
@@ -429,6 +429,7 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
 
         """
         torch.set_grad_enabled(False)
+        self.model.eval()
         if len(documents) == 0:
             raise AttributeError("generator needs documents to predict the answer")
 
diff --git a/haystack/nodes/extractor/entity.py b/haystack/nodes/extractor/entity.py
index 2469ec6d80..d03bc74fa0 100644
--- a/haystack/nodes/extractor/entity.py
+++ b/haystack/nodes/extractor/entity.py
@@ -430,7 +430,7 @@ def extract(self, text: Union[str, List[str]], batch_size: int = 1):
         dataloader = DataLoader(dataset, shuffle=False, batch_size=batch_size, num_workers=self.num_workers)
 
         # Forward
-        self.model.model.eval()
+        self.model.eval()
         predictions: List[Dict[str, Any]] = []
         for batch in tqdm(dataloader, disable=not self.progress_bar, total=len(dataloader), desc="Extracting entities"):
             batch = ensure_tensor_on_device(batch, device=self.devices[0])
diff --git a/haystack/nodes/question_generator/question_generator.py b/haystack/nodes/question_generator/question_generator.py
index b21bd6c31e..4ccc56c2f7 100644
--- a/haystack/nodes/question_generator/question_generator.py
+++ b/haystack/nodes/question_generator/question_generator.py
@@ -153,7 +153,7 @@ def generate(self, text: str) -> List[str]:
         input_ids = tokenized["input_ids"].to(self.devices[0])
         # Necessary if padding is enabled so the model won't attend pad tokens
         attention_mask = tokenized["attention_mask"].to(self.devices[0])
-        self.model.model.eval()
+        self.model.eval()
         tokens_output = self.model.generate(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -223,7 +223,7 @@ def generate_batch(
             input_ids = tokenized["input_ids"].to(self.devices[0])
             # Necessary if padding is enabled so the model won't attend pad tokens
             attention_mask = tokenized["attention_mask"].to(self.devices[0])
-            self.model.model.eval()
+            self.model.eval()
             tokens_output = self.model.generate(
                 input_ids=input_ids,
                 attention_mask=attention_mask,
diff --git a/haystack/nodes/retriever/text2sparql.py b/haystack/nodes/retriever/text2sparql.py
index 528cc54dde..600ba653e4 100644
--- a/haystack/nodes/retriever/text2sparql.py
+++ b/haystack/nodes/retriever/text2sparql.py
@@ -55,7 +55,7 @@ def retrieve(self, query: str, top_k: Optional[int] = None):
         :param query: Text query that shall be translated to SPARQL and then executed on the knowledge graph
         :param top_k: How many SPARQL queries to generate per text query.
         """
-        self.model.model.eval()
+        self.model.eval()
         if top_k is None:
             top_k = self.top_k
         inputs = self.tok([query], max_length=100, truncation=True, return_tensors="pt")
diff --git a/haystack/nodes/translator/transformers.py b/haystack/nodes/translator/transformers.py
index 8b7b22281f..30d8febcbb 100644
--- a/haystack/nodes/translator/transformers.py
+++ b/haystack/nodes/translator/transformers.py
@@ -151,7 +151,7 @@ def translate(
             truncation=True,
         ).to(self.devices[0])
 
-        self.model.model.eval()
+        self.model.eval()
         generated_output = self.model.generate(**batch)
         translated_texts = self.tokenizer.batch_decode(
             generated_output, skip_special_tokens=True, clean_up_tokenization_spaces=self.clean_up_tokenization_spaces

From ad789f7c63f6e9a2d77258dfd0a5e0cbd0415ac3 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Mon, 27 Mar 2023 09:36:27 +0200
Subject: [PATCH 14/16] Undoing additions of eval as discussed in PR

---
 haystack/modeling/infer.py                    |  2 -
 .../nodes/answer_generator/transformers.py    |  3 --
 .../nodes/document_classifier/transformers.py |  1 -
 haystack/nodes/extractor/entity.py            |  1 -
 .../nodes/query_classifier/transformers.py    |  2 -
 .../question_generator/question_generator.py  |  2 -
 haystack/nodes/reader/farm.py                 |  2 -
 haystack/nodes/reader/transformers.py         |  2 -
 .../nodes/retriever/_embedding_encoder.py     |  3 --
 haystack/nodes/retriever/text2sparql.py       |  1 -
 haystack/nodes/translator/transformers.py     |  1 -
 test/nodes/test_reader.py                     | 39 -------------------
 test/nodes/test_retriever.py                  | 24 ------------
 13 files changed, 83 deletions(-)

diff --git a/haystack/modeling/infer.py b/haystack/modeling/infer.py
index 0ced623579..fa591904b2 100644
--- a/haystack/modeling/infer.py
+++ b/haystack/modeling/infer.py
@@ -358,7 +358,6 @@ def _get_predictions(self, dataset: Dataset, tensor_names: List, baskets):
                         Example: QA - input string to convert the predicted answer from indices back to string space
         :return: list of predictions
         """
-        self.model.eval()
         samples = [s for b in baskets for s in b.samples]
 
         data_loader = NamedDataLoader(
@@ -395,7 +394,6 @@ def _get_predictions_and_aggregate(self, dataset: Dataset, tensor_names: List, b
                         Example: QA - input string to convert the predicted answer from indices back to string space
         :return: list of predictions
         """
-        self.model.eval()
         data_loader = NamedDataLoader(
             dataset=dataset, sampler=SequentialSampler(dataset), batch_size=self.batch_size, tensor_names=tensor_names  # type: ignore [arg-type]
         )  # type ignore
diff --git a/haystack/nodes/answer_generator/transformers.py b/haystack/nodes/answer_generator/transformers.py
index d5688d2539..cc38b15377 100644
--- a/haystack/nodes/answer_generator/transformers.py
+++ b/haystack/nodes/answer_generator/transformers.py
@@ -230,8 +230,6 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
         ```
         """
         torch.set_grad_enabled(False)
-        self.model.question_encoder.eval()
-        self.model.generator.eval()
         if len(documents) == 0:
             raise AttributeError("generator need documents to predict the answer")
 
@@ -434,7 +432,6 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
 
         """
         torch.set_grad_enabled(False)
-        self.model.eval()
         if len(documents) == 0:
             raise AttributeError("generator needs documents to predict the answer")
 
diff --git a/haystack/nodes/document_classifier/transformers.py b/haystack/nodes/document_classifier/transformers.py
index a551d6c115..0436d138ae 100644
--- a/haystack/nodes/document_classifier/transformers.py
+++ b/haystack/nodes/document_classifier/transformers.py
@@ -173,7 +173,6 @@ def predict(self, documents: List[Document], batch_size: Optional[int] = None) -
         :param batch_size: The number of Documents to classify at a time.
         :return: A list of Documents enriched with meta information.
         """
-        self.model.model.eval()
         if batch_size is None:
             batch_size = self.batch_size
 
diff --git a/haystack/nodes/extractor/entity.py b/haystack/nodes/extractor/entity.py
index e80b8173fd..c99a345ca4 100644
--- a/haystack/nodes/extractor/entity.py
+++ b/haystack/nodes/extractor/entity.py
@@ -431,7 +431,6 @@ def extract(self, text: Union[str, List[str]], batch_size: int = 1):
         dataloader = DataLoader(dataset, shuffle=False, batch_size=batch_size, num_workers=self.num_workers)
 
         # Forward
-        self.model.eval()
         predictions: List[Dict[str, Any]] = []
         for batch in tqdm(dataloader, disable=not self.progress_bar, total=len(dataloader), desc="Extracting entities"):
             batch = ensure_tensor_on_device(batch, device=self.devices[0])
diff --git a/haystack/nodes/query_classifier/transformers.py b/haystack/nodes/query_classifier/transformers.py
index a8fca24b04..c769241c20 100644
--- a/haystack/nodes/query_classifier/transformers.py
+++ b/haystack/nodes/query_classifier/transformers.py
@@ -145,7 +145,6 @@ def _get_edge_number_from_label(self, label):
         return self.labels.index(label) + 1
 
     def run(self, query: str):  # type: ignore
-        self.model.model.eval()
         if self.task == "zero-shot-classification":
             prediction = self.model([query], candidate_labels=self.labels, truncation=True)
             label = prediction[0]["labels"][0]
@@ -159,7 +158,6 @@ def run_batch(self, queries: List[str], batch_size: Optional[int] = None):  # ty
         queries_dataset = ListDataset(queries)
         if batch_size is None:
             batch_size = self.batch_size
-        self.model.model.eval()
         all_predictions = []
         if self.task == "zero-shot-classification":
             for predictions in tqdm(
diff --git a/haystack/nodes/question_generator/question_generator.py b/haystack/nodes/question_generator/question_generator.py
index 903c9ca7dc..3d41567304 100644
--- a/haystack/nodes/question_generator/question_generator.py
+++ b/haystack/nodes/question_generator/question_generator.py
@@ -154,7 +154,6 @@ def generate(self, text: str) -> List[str]:
         input_ids = tokenized["input_ids"].to(self.devices[0])
         # Necessary if padding is enabled so the model won't attend pad tokens
         attention_mask = tokenized["attention_mask"].to(self.devices[0])
-        self.model.eval()
         tokens_output = self.model.generate(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -224,7 +223,6 @@ def generate_batch(
             input_ids = tokenized["input_ids"].to(self.devices[0])
             # Necessary if padding is enabled so the model won't attend pad tokens
             attention_mask = tokenized["attention_mask"].to(self.devices[0])
-            self.model.eval()
             tokens_output = self.model.generate(
                 input_ids=input_ids,
                 attention_mask=attention_mask,
diff --git a/haystack/nodes/reader/farm.py b/haystack/nodes/reader/farm.py
index 8eb69cb50c..14949f426b 100644
--- a/haystack/nodes/reader/farm.py
+++ b/haystack/nodes/reader/farm.py
@@ -834,7 +834,6 @@ def predict_batch(
         :param top_k: Number of returned answers per query.
         :param batch_size: Number of query-document pairs to be processed at a time.
         """
-        self.inferencer.model.eval()
         if top_k is None:
             top_k = self.top_k
 
@@ -901,7 +900,6 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
         :param top_k: The maximum number of answers to return
         :return: Dict containing query and answers
         """
-        self.inferencer.model.eval()
         if top_k is None:
             top_k = self.top_k
         # convert input to FARM format
diff --git a/haystack/nodes/reader/transformers.py b/haystack/nodes/reader/transformers.py
index de94e3296d..65e141c593 100644
--- a/haystack/nodes/reader/transformers.py
+++ b/haystack/nodes/reader/transformers.py
@@ -136,7 +136,6 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
         :param top_k: The maximum number of answers to return
         :return: Dict containing query and answers
         """
-        self.model.model.eval()
         if top_k is None:
             top_k = self.top_k
 
@@ -203,7 +202,6 @@ def predict_batch(
         :param top_k: Number of returned answers per query.
         :param batch_size: Number of query-document pairs to be processed at a time.
         """
-        self.model.model.eval()
         if top_k is None:
             top_k = self.top_k
 
diff --git a/haystack/nodes/retriever/_embedding_encoder.py b/haystack/nodes/retriever/_embedding_encoder.py
index b95af92566..61be9f975a 100644
--- a/haystack/nodes/retriever/_embedding_encoder.py
+++ b/haystack/nodes/retriever/_embedding_encoder.py
@@ -69,7 +69,6 @@ def __init__(self, retriever: "EmbeddingRetriever"):
     def embed(self, texts: Union[List[List[str]], List[str], str]) -> np.ndarray:
         # TODO: FARM's `sample_to_features_text` need to fix following warning -
         # tokenization_utils.py:460: FutureWarning: `is_pretokenized` is deprecated and will be removed in a future version, use `is_split_into_words` instead.
-        self.embedding_model.model.eval()
         emb = self.embedding_model.inference_from_dicts(dicts=[{"text": t} for t in texts])
         emb = np.stack([r["vec"] for r in emb])
         return emb
@@ -268,7 +267,6 @@ def embed_queries(self, queries: List[str]) -> np.ndarray:
         :param queries: List of queries to embed.
         :return: Embeddings, one per input query, shape: (queries, embedding_dim)
         """
-        self.embedding_model.eval()
         query_text = [{"text": q} for q in queries]
         dataloader = self._create_dataloader(query_text)
 
@@ -296,7 +294,6 @@ def embed_documents(self, docs: List[Document]) -> np.ndarray:
         :param docs: List of documents to embed.
         :return: Embeddings, one per input document, shape: (documents, embedding_dim)
         """
-        self.embedding_model.eval()
         doc_text = [{"text": d.content} for d in docs]
         dataloader = self._create_dataloader(doc_text)
 
diff --git a/haystack/nodes/retriever/text2sparql.py b/haystack/nodes/retriever/text2sparql.py
index 8f69bd7c58..5bb28e4abb 100644
--- a/haystack/nodes/retriever/text2sparql.py
+++ b/haystack/nodes/retriever/text2sparql.py
@@ -55,7 +55,6 @@ def retrieve(self, query: str, top_k: Optional[int] = None):
         :param query: Text query that shall be translated to SPARQL and then executed on the knowledge graph
         :param top_k: How many SPARQL queries to generate per text query.
         """
-        self.model.eval()
         if top_k is None:
             top_k = self.top_k
         inputs = self.tok([query], max_length=100, truncation=True, return_tensors="pt")
diff --git a/haystack/nodes/translator/transformers.py b/haystack/nodes/translator/transformers.py
index bec1aa7567..9cc7f1ec9e 100644
--- a/haystack/nodes/translator/transformers.py
+++ b/haystack/nodes/translator/transformers.py
@@ -152,7 +152,6 @@ def translate(
             truncation=True,
         ).to(self.devices[0])
 
-        self.model.eval()
         generated_output = self.model.generate(**batch)
         translated_texts = self.tokenizer.batch_decode(
             generated_output, skip_special_tokens=True, clean_up_tokenization_spaces=self.clean_up_tokenization_spaces
diff --git a/test/nodes/test_reader.py b/test/nodes/test_reader.py
index 10b48435b3..c21076978c 100644
--- a/test/nodes/test_reader.py
+++ b/test/nodes/test_reader.py
@@ -5,7 +5,6 @@
 
 import pytest
 
-import torch
 from huggingface_hub import snapshot_download
 from haystack.modeling.data_handler.inputs import QAInput, Question
 
@@ -63,44 +62,6 @@ def test_output(reader, docs):
     assert len(prediction["answers"]) == 5
 
 
-def test_output_train_mode(reader, docs):
-    # Set to deterministic seed
-    old_seed = torch.seed()
-    torch.manual_seed(0)
-
-    pred_default = reader.predict(query="Who lives in Berlin?", documents=docs, top_k=5)
-    if isinstance(reader, FARMReader):
-        reader.inferencer.model.train()
-    else:
-        reader.model.model.train()
-    pred_train_mode = reader.predict(query="Who lives in Berlin?", documents=docs, top_k=5)
-
-    assert pred_default["query"] == pred_train_mode["query"]
-    assert pred_default["answers"][0].answer == pred_train_mode["answers"][0].answer
-    assert (
-        pred_default["answers"][0].offsets_in_context[0].start
-        == pred_train_mode["answers"][0].offsets_in_context[0].start
-    )
-    assert (
-        pred_default["answers"][0].offsets_in_context[0].end == pred_train_mode["answers"][0].offsets_in_context[0].end
-    )
-    assert (
-        pred_default["answers"][0].offsets_in_document[0].start
-        == pred_train_mode["answers"][0].offsets_in_document[0].start
-    )
-    assert (
-        pred_default["answers"][0].offsets_in_document[0].end
-        == pred_train_mode["answers"][0].offsets_in_document[0].end
-    )
-    assert pred_default["answers"][0].type == pred_train_mode["answers"][0].type
-    assert math.isclose(pred_default["answers"][0].score, pred_train_mode["answers"][0].score, rel_tol=1e-4)
-    assert pred_default["answers"][0].context == pred_train_mode["answers"][0].context
-    assert len(pred_default["answers"]) == len(pred_train_mode["answers"])
-
-    # Set back to old_seed
-    torch.manual_seed(old_seed)
-
-
 def test_output_batch_single_query_single_doc_list(reader, docs):
     prediction = reader.predict_batch(queries=["Who lives in Berlin?"], documents=docs, top_k=5)
     assert prediction is not None
diff --git a/test/nodes/test_retriever.py b/test/nodes/test_retriever.py
index 0fc2c41274..e289a0ff5c 100644
--- a/test/nodes/test_retriever.py
+++ b/test/nodes/test_retriever.py
@@ -4,7 +4,6 @@
 from typing import Dict, List, Optional, Union
 
 import pytest
-import torch
 import numpy as np
 import pandas as pd
 from pandas.testing import assert_frame_equal
@@ -21,7 +20,6 @@
 from haystack.nodes.retriever.dense import DensePassageRetriever, EmbeddingRetriever, TableTextRetriever
 from haystack.nodes.retriever.sparse import BM25Retriever, FilterRetriever, TfidfRetriever
 from haystack.nodes.retriever.multimodal import MultiModalRetriever
-from haystack.nodes.retriever._embedding_encoder import _DefaultEmbeddingEncoder
 
 from ..conftest import SAMPLES_PATH, MockRetriever
 
@@ -320,28 +318,6 @@ def test_dpr_embedding(document_store: BaseDocumentStore, retriever, docs_with_i
         assert isclose(embedding[0], expected_value, rel_tol=0.01)
 
 
-@pytest.mark.integration
-@pytest.mark.parametrize("document_store", ["memory"], indirect=True)
-@pytest.mark.parametrize("retriever", ["embedding", "retribert", "embedding_sbert"], indirect=True)
-def test_embedding_train_mode(retriever, docs_with_ids):
-    # Set to deterministic seed
-    old_seed = torch.seed()
-    torch.manual_seed(0)
-
-    embeddings_default_mode = retriever.embed_documents(documents=docs_with_ids)
-    if isinstance(retriever.embedding_encoder, _DefaultEmbeddingEncoder):
-        retriever.embedding_encoder.embedding_model.model.train()
-    else:
-        retriever.embedding_encoder.embedding_model.train()
-    embeddings_train_mode = retriever.embed_documents(documents=docs_with_ids)
-
-    for emb1, emb2 in zip(embeddings_train_mode, embeddings_default_mode):
-        assert any(np.isclose(emb1, emb2, rtol=1.0e-5))
-
-    # Set back to old_seed
-    torch.manual_seed(old_seed)
-
-
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "document_store", ["elasticsearch", "faiss", "memory", "milvus", "weaviate", "pinecone"], indirect=True

From 68def3a8c0a33922894a1369ac72888c9407027b Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Mon, 27 Mar 2023 09:52:49 +0200
Subject: [PATCH 15/16] Added self.model.eval() to end of training loop in
 FARMReader

---
 haystack/modeling/training/base.py |  1 +
 test/nodes/test_reader.py          | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/haystack/modeling/training/base.py b/haystack/modeling/training/base.py
index 473034ac03..5b0b316bb4 100644
--- a/haystack/modeling/training/base.py
+++ b/haystack/modeling/training/base.py
@@ -291,6 +291,7 @@ def train(self):
                 )
                 self.test_result = evaluator_test.eval(self.model)
                 evaluator_test.log_results(self.test_result, "Test", self.global_step)
+        self.model.eval()
         return self.model
 
     def compute_loss(self, batch: dict, step: int) -> torch.Tensor:
diff --git a/test/nodes/test_reader.py b/test/nodes/test_reader.py
index c21076978c..2894671c99 100644
--- a/test/nodes/test_reader.py
+++ b/test/nodes/test_reader.py
@@ -416,6 +416,36 @@ def test_no_answer_reader_skips_empty_documents(no_answer_reader):
     assert predictions["answers"][1][1].answer == "Carla"  # answer given for 2nd query as usual
 
 
+@pytest.mark.integration
+def test_reader_training_returns_eval(tmp_path):
+    max_seq_len = 16
+    max_query_length = 8
+    reader = FARMReader(
+        model_name_or_path="deepset/tinyroberta-squad2",
+        use_gpu=False,
+        num_processes=0,
+        max_seq_len=max_seq_len,
+        doc_stride=2,
+        max_query_length=max_query_length,
+    )
+
+    save_dir = f"{tmp_path}/test_dpr_training"
+    reader.train(
+        data_dir=str(SAMPLES_PATH / "squad"),
+        train_filename="tiny.json",
+        dev_filename="tiny.json",
+        n_epochs=1,
+        batch_size=1,
+        grad_acc_steps=1,
+        evaluate_every=0,
+        save_dir=save_dir,
+        max_seq_len=max_seq_len,
+        max_query_length=max_query_length,
+    )
+    assert reader.inferencer.model.training is False
+
+
+@pytest.mark.integration
 def test_reader_training(tmp_path):
     max_seq_len = 16
     max_query_length = 8

From 1ce9e900b052097ad9f1b616d78cbd520698fce0 Mon Sep 17 00:00:00 2001
From: Sebastian Lee <sebastian.lee@deepset.ai>
Date: Mon, 27 Mar 2023 10:58:18 +0200
Subject: [PATCH 16/16] Try removing integration tags

---
 test/nodes/test_reader.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/nodes/test_reader.py b/test/nodes/test_reader.py
index 2894671c99..495c9bd6a9 100644
--- a/test/nodes/test_reader.py
+++ b/test/nodes/test_reader.py
@@ -416,7 +416,6 @@ def test_no_answer_reader_skips_empty_documents(no_answer_reader):
     assert predictions["answers"][1][1].answer == "Carla"  # answer given for 2nd query as usual
 
 
-@pytest.mark.integration
 def test_reader_training_returns_eval(tmp_path):
     max_seq_len = 16
     max_query_length = 8
@@ -445,7 +444,6 @@ def test_reader_training_returns_eval(tmp_path):
     assert reader.inferencer.model.training is False
 
 
-@pytest.mark.integration
 def test_reader_training(tmp_path):
     max_seq_len = 16
     max_query_length = 8