Deprecate haystack

.github/workflows/test-check.yaml

@@ -97,6 +97,6 @@ jobs:
       - name: "Clean sparsezoo directory"
         run: rm -r sparsezoo/
       - name: ⚙️ Install dependencies
-        run: pip install .[dev,haystack]
+        run: pip install .[dev]
       - name: Run integrations tests
         run: make test_integrations

.vscode/launch.json

@@ -0,0 +1,27 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "haystack init",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "src/deepsparse/transformers/haystack/__init__.py",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
+        {
+            "name": "benchmarking bug",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "/root/projects/deepsparse/src/deepsparse/benchmark/benchmark_pipeline.py",
+            "console": "integratedTerminal",
+            "args": [
+                "text-classification",
+                "zoo:nlp/sentiment_analysis/distilbert-none/pytorch/huggingface/sst2/pruned90-none"
+            ]
+        }
+    ]
+}

MANIFEST.in

@@ -1,6 +1,5 @@
 include LICENSE
 include utils/artifacts.py
-include src/deepsparse/transformers/haystack/haystack_reqs.txt
 recursive-include src/deepsparse/avx2 *
 recursive-include src/deepsparse/avx512 *
 recursive-include src/deepsparse/neon *

integrations/test_placeholder.py

@@ -0,0 +1,14 @@
+
+def test_placeholder():
+    """
+    Needed to make the test suite run and not throw
+    an error about no tests being found when
+    `make test_integrations` is used.
+    
+    The error would look like this:
+        make: *** [Makefile:61: test_integrations] Error 5
+    
+    More information can be found here:
+        https://github.com/pytest-dev/pytest/issues/2393
+    """
+    pass

setup.py

@@ -77,13 +77,6 @@
 ]
 
 
-def _parse_requirements_file(file_path):
-    with open(file_path, "r") as requirements_file:
-        lines = requirements_file.read().splitlines()
-
-    return [line for line in lines if len(line) > 0 and line[0] != "#"]
-
-
 _deps = [
     "numpy>=1.16.3",
     "onnx>=1.5.0,<1.15.0",
@@ -153,17 +146,6 @@ def _parse_requirements_file(file_path):
 ]
 _sentence_transformers_integration_deps = ["optimum-deepsparse"] + _torch_deps
 
-# haystack dependencies are installed from a requirements file to avoid
-# conflicting versions with NM's deepsparse/transformers
-_haystack_requirements_file_path = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    "src",
-    "deepsparse",
-    "transformers",
-    "haystack",
-    "haystack_reqs.txt",
-)
-_haystack_integration_deps = _parse_requirements_file(_haystack_requirements_file_path)
 _clip_deps = [
     "open_clip_torch==2.20.0",
     "transformers<4.40",
@@ -270,7 +252,6 @@ def _setup_extras() -> Dict:
         "image_classification": _computer_vision_deps,
         "yolo": _computer_vision_deps,
         "yolov5": _computer_vision_deps,
-        "haystack": _haystack_integration_deps,
         "openpifpaf": _openpifpaf_integration_deps,
         "yolov8": _yolov8_integration_deps,
         "transformers": _transformers_integration_deps,

src/deepsparse/legacy/tasks.py

@@ -206,7 +206,10 @@ def check_register_task(
         elif cls.is_haystack(task):
             # trigger haystack pipeline as well as transformers pipelines to
             # register with Pipeline.register
-            import deepsparse.transformers.haystack  # noqa: F401
+            raise DeprecationWarning(
+                "Haystack support with deepsparse has been deprecated, "
+                "kindly use deepsparse-nightly==1.8.20240404 or older"
+            )
 
         elif cls.is_embedding_extraction(task):
             # trigger embedding_extraction pipelines to register with

src/deepsparse/pipeline.py

@@ -800,8 +800,13 @@ def haystack_pipeline(*args, **kwargs) -> "Pipeline":
     Neural Magic pipeline for running Haystack DocumentSearchPipeline.
     Supports selected Haystack Nodes as well as Haystack nodes integrated
     with the Neural Magic DeepSparse Engine
+
+    Note: Deprecated due to lack of pydanticV2 support in Haystack v1
     """
-    return Pipeline.create("information_retrieval_haystack", *args, **kwargs)
+    raise DeprecationWarning(
+        "Haystack support with deepsparse has been deprecated, "
+        "kindly use deepsparse-nightly==1.8.20240404 or older"
+    )
 
 
 def embedding_extraction_pipeline(*args, **kwargs) -> "Pipeline":

src/deepsparse/transformers/haystack/__init__.py

@@ -19,102 +19,6 @@
 # flake8: noqa
 # isort: skip_file
 
-
-import logging as _logging
-import os as _os
-
-import deepsparse as _deepsparse
-
-
-_HAYSTACK_PREFERRED_VERSION = "1.4.0"
-_HAYSTACK_EXTRAS = "[all]"
-
-
-# check haystack installation
-try:
-    import haystack as _haystack
-
-    if _haystack.__version__ != _HAYSTACK_PREFERRED_VERSION:
-        raise ValueError(
-            f"Deepsparse requires farm-haystack=={_HAYSTACK_PREFERRED_VERSION}, "
-            f"but found {_haystack.__version__}"
-        )
-    _haystack_import_error = None
-except Exception as _haystack_import_err:
-    _haystack_import_error = _haystack_import_err
-
-_LOGGER = _logging.getLogger(__name__)
-
-
-def _install_haystack_and_deps():
-    import subprocess as _subprocess
-    import sys as _sys
-
-    try:
-        _subprocess.check_call(
-            [
-                _sys.executable,
-                "-m",
-                "pip",
-                "install",
-                f"farm-haystack{_HAYSTACK_EXTRAS}=={_HAYSTACK_PREFERRED_VERSION}",
-                "--no-dependencies",
-            ]
-        )
-
-        import haystack as _haystack
-
-        _LOGGER.info("haystack and dependencies successfully installed")
-    except Exception:
-        raise ValueError(
-            "Unable to install and import haystack dependencies. Check "
-            "that haystack is installed, if not, install via "
-            "`pip install deepsparse[haystack]` and `pip install "
-            f"farm-haystack{_HAYSTACK_EXTRAS}=={_HAYSTACK_PREFERRED_VERSION} "
-            "--no-dependencies`"
-        )
-
-
-def _check_haystack_install():
-    if _haystack_import_error is not None:
-        import os
-
-        if os.getenv("NM_NO_AUTOINSTALL_HAYSTACK", False):
-            _LOGGER.warning(
-                "Unable to import haystack, skipping auto installation "
-                "due to NM_NO_AUTOINSTALL_HAYSTACK"
-            )
-            # skip any further checks
-            return
-        else:
-            _LOGGER.warning(
-                "haystack installation not detected. Installing "
-                "haystack dependencies if haystack is already "
-                "installed in the environment, it will be overwritten. Set "
-                "environment variable NM_NO_AUTOINSTALL_HAYSTACK to disable"
-            )
-            _install_haystack_and_deps()
-
-    # re check import after potential install
-    try:
-        import haystack as _haystack
-
-        if _haystack.__version__ != _HAYSTACK_PREFERRED_VERSION:
-            raise ValueError(
-                f"Deepsparse requires farm-haystack=={_HAYSTACK_PREFERRED_VERSION}, "
-                f"but found {_haystack.__version__}"
-            )
-    except Exception:
-        _LOGGER.warning(
-            "haystack and its dependencies may not be installed. They can be installed "
-            "via `pip install deepsparse[haystack]` and `pip install "
-            f"farm-haystack{_HAYSTACK_EXTRAS}=={_HAYSTACK_PREFERRED_VERSION} "
-            "--no-dependencies`"
-        )
-
-
-_check_haystack_install()
-
-from .nodes import *
-from .pipeline import *
-from .helpers import *
+raise DeprecationWarning(
+    "Haystack support with deepsparse has been deprecated, kindly use deepsparse-nightly==1.8.20240404 or older"
+)