Merge branch 'main' into v2-whisper-local

.github/labeler.yml

@@ -95,12 +95,6 @@ topic:pinecone:
 topic:knowledge_graph:
 - haystack/document_stores/memory_knowledgegraph.py
 - test/document_stores/test_knowledge_graph.py
-topic:milvus:
-- haystack/document_stores/milvus.py
-- test/document_stores/test_milvus.py
 topic:dc_document_store:
 - haystack/document_stores/deepsetcloud.py
 - test/document_stores/test_deepsetcloud.py
-topic:graphdb:
-- haystack/document_stores/graphdb.py
-- test/document_stores/test_knowledge_graph.py

.github/release.yml

@@ -29,12 +29,10 @@ changelog:
         - topic:document_store
         - topic:elasticsearch
         - topic:faiss
-        - topic:milvus
         - topic:weaviate
         - topic:pinecone
         - topic:sql
         - topic:knowledge_graph
-        - topic:graphdb
     - title: REST API
       labels:
         - topic:api

.github/workflows/cml.yaml → .github/workflows/cml.yml

@@ -14,6 +14,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
           VPC: ${{ secrets.AWS_CI_VPC }}
+
         run: |
           echo "Deploying..."
           RUNNER_LABELS="cml,aws"
@@ -22,21 +23,21 @@ jobs:
           docker-machine create \
             --driver amazonec2 \
             --amazonec2-instance-type p3.2xlarge \
-            --amazonec2-vpc-id $VPC \
+            --amazonec2-vpc-id "$VPC" \
             --amazonec2-region us-east-1 \
             --amazonec2-zone c \
             --amazonec2-ssh-user ubuntu \
             --amazonec2-ami ami-06a25ee8966373068 \
             --amazonec2-root-size 150 \
-            $MACHINE
-          eval "$(docker-machine env --shell sh $MACHINE)"
+            "$MACHINE"
+          eval "$(docker-machine env --shell sh "$MACHINE")"
 
           (
-          docker-machine ssh $MACHINE "sudo mkdir -p \
+          docker-machine ssh "$MACHINE" "sudo mkdir -p \
             /docker_machine && \
           sudo chmod 777 /docker_machine" && \
           docker-machine scp -r -q ~/.docker/machine/ \
-            $MACHINE:/docker_machine && \
+            "$MACHINE:/docker_machine" && \
           docker run --name elasticsearch -d \
                      -p 9200:9200 \
                      -e "discovery.type=single-node" \
@@ -46,22 +47,22 @@ jobs:
                       --net host \
                       -e POSTGRES_PASSWORD=password \
                       -v /docker_machine/machine:/root/.docker/machine \
-                      -e DOCKER_MACHINE=$MACHINE \
+                      -e "DOCKER_MACHINE=$MACHINE" \
                       postgres && \
           sleep 4 && \
           docker exec -i postgres psql -U postgres -c "CREATE DATABASE haystack;" && \
           docker run --name runner -d \
             --gpus all \
             -v /docker_machine/machine:/root/.docker/machine \
             --net host \
-            -e DOCKER_MACHINE=$MACHINE \
-            -e repo_token=$repo_token \
-            -e RUNNER_LABELS=$RUNNER_LABELS \
-            -e RUNNER_REPO=$RUNNER_REPO \
+            -e "DOCKER_MACHINE=$MACHINE" \
+            -e "repo_token=$repo_token" \
+            -e "RUNNER_LABELS=$RUNNER_LABELS" \
+            -e "RUNNER_REPO=$RUNNER_REPO" \
             -e RUNNER_IDLE_TIMEOUT=120 \
             dvcorg/cml-py3:latest && \
           sleep 20 && echo "Deployed $MACHINE"
-          ) || (echo "Shut down machine" && docker-machine rm -y -f $MACHINE && exit 1)
+          ) || (echo "Shut down machine" && docker-machine rm -y -f "$MACHINE" && exit 1)
   run-benchmark:
     needs: deploy-cloud-runner
     runs-on: [self-hosted,cml]
@@ -75,12 +76,14 @@ jobs:
         run: |
           apt-get update -y
           apt-get install python3-dev -y
-          pip install .[elasticsearch,faiss,milvus,weaviate,graphdb,ray,rest,ui,dev]
+          pip install .[elasticsearch,faiss,weaviate,ray,rest,ui,dev]
           cd test/benchmarks && python run.py --retriever_index --retriever_query --reader --ci --save_markdown
-          echo -en "## Benchmarks: Retriever Indexing\n" >> report.md
-          cat retriever_index_results.md >> report.md
-          echo -en "\n\n## Benchmarks: Retriever Querying\n" >> report.md
-          cat retriever_query_results.md >> report.md
-          echo -en "\n\n## Benchmarks: Reader\n" >> report.md
-          cat reader_results.md >> report.md
+          {
+            echo -en "## Benchmarks: Retriever Indexing\n"
+            cat retriever_index_results.md
+            echo -en "\n\n## Benchmarks: Retriever Querying\n"
+            cat retriever_query_results.md
+            echo -en "\n\n## Benchmarks: Reader\n"
+            cat reader_results.md
+          } >> report.md
           cml-send-comment report.md

.github/workflows/e2e.yml

@@ -47,19 +47,9 @@ jobs:
       run: |
         docker run -d -p 9201:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch:1.3.5
 
-    - name: Run Milvus
-      run: |
-        cd ../../   # Avoid causing permission issues on hashFiles later by creating unreadable folders like "volumes"
-        wget https://github.com/milvus-io/milvus/releases/download/v2.2.2/milvus-standalone-docker-compose.yml -O docker-compose.yml
-        sudo docker-compose up -d
-        sudo docker-compose ps
-
     - name: Run Weaviate
       run: docker run -d -p 8080:8080 --name haystack_test_weaviate --env AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED='true' --env PERSISTENCE_DATA_PATH='/var/lib/weaviate' --env ENABLE_EXPERIMENTAL_BM25='true' --env DISK_USE_READONLY_PERCENTAGE='95' semitechnologies/weaviate:1.17.2
 
-    - name: Run GraphDB
-      run: docker run -d -p 7200:7200 --name haystack_test_graphdb deepset/graphdb-free:9.4.1-adoptopenjdk11
-
     - name: Install Haystack
       run: pip install .
 

.github/workflows/examples_tests.yml

@@ -0,0 +1,77 @@
+name: Examples tests
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - examples/**
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+  PYTHON_VERSION: "3.8"
+
+jobs:
+  tests:
+    name: Examples
+    runs-on: ubuntu-latest
+    services:
+      elasticsearch:
+        image: elasticsearch:7.17.6
+        env:
+          discovery.type: "single-node"
+          ES_JAVA_OPTS: "-Xms128m -Xmx256m"
+        ports:
+          - 9200:9200
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Haystack
+        run: pip install .[all,dev]
+
+      - name: Run
+        run: pytest examples/
+
+      - name: Calculate alert data
+        id: calculator
+        if: (success() || failure()) && github.ref_name == 'main'
+        shell: bash
+        run: |
+          if [ "${{ job.status }}" = "success" ]; then
+            echo "alert_type=success" >> "$GITHUB_OUTPUT";
+          else
+            echo "alert_type=error" >> "$GITHUB_OUTPUT";
+          fi
+
+      - name: Send event to Datadog
+        if: (success() || failure()) && github.ref_name == 'main'
+        uses: masci/datadog@v1
+        with:
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          api-url: https://api.datadoghq.eu
+          events: |
+            - title: "${{ github.workflow }} workflow"
+              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
+              alert_type: "${{ steps.calculator.outputs.alert_type }}"
+              source_type_name: "Github"
+              host: ${{ github.repository_owner }}
+              tags:
+                - "project:${{ github.repository }}"
+                - "job:${{ github.job }}"
+                - "run_id:${{ github.run_id }}"
+                - "workflow:${{ github.workflow }}"
+                - "branch:${{ github.ref_name }}"
+                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"