Merge branch 'main' into main

.github/workflows/benchmarks.yml

@@ -21,10 +21,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: iterative/setup-cml@v1
+      - uses: iterative/setup-cml@v2
 
       - name: AWS authentication
-        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
         with:
           aws-region: ${{ env.AWS_REGION }}
           role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
@@ -237,7 +237,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: AWS authentication
-        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
         with:
           aws-region: ${{ env.AWS_REGION }}
           role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}

.github/workflows/ci_metrics.yml

@@ -4,8 +4,10 @@ on:
   workflow_run:
     workflows:
       - "end-to-end"
+      - "end-to-end (Preview)"
       - "Linting"
       - "Tests"
+      - "Tests (Preview)"
       - "REST API Tests"
     types:
       - completed

.github/workflows/e2e.yml

@@ -13,6 +13,7 @@ on:
       - ready_for_review
     paths:
       - "e2e/**/*.py"
+      - "!e2e/preview/**/*.py"   # See e2e_preview.yml
       - ".github/workflows/e2e.yml"
 
 env:

.github/workflows/e2e_preview.yml

@@ -0,0 +1,42 @@
+# If you change this name also do it in ci_metrics.yml
+name: end-to-end (Preview)
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    paths:
+      - "e2e/preview/**/*.py"
+      - ".github/workflows/e2e_preview.yml"
+
+env:
+  PYTHON_VERSION: "3.8"
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+jobs:
+  run:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt install ffmpeg  # for local Whisper tests
+
+    - name: Install Haystack
+      run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+
+    - name: Run tests
+      run: pytest e2e/preview

.github/workflows/examples_tests.yml

@@ -15,8 +15,6 @@ on:
       - ready_for_review
 
 env:
-  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/license_compliance.yml

@@ -171,7 +171,7 @@ jobs:
     name: All extras
     env:
       REQUIREMENTS_FILE: requirements_all.txt
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-4-cores
     steps:
       - name: Checkout the code
         uses: actions/checkout@v4

.github/workflows/pypi_release.yml

@@ -33,5 +33,5 @@ jobs:
         uses: act10ns/slack@v2
         with:
           status: ${{ job.status }}
-          channel: "#haystack"
+          channel: "#haystack-notifications"
           config: .github/config/pypi-release-slack-notification.yml

.github/workflows/release_notes.yml

@@ -22,9 +22,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-            # With the default value of 1, there are corner cases where tj-actions/changed-files
-            # fails with a `no merge base` error
-            fetch-depth: 0
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
 
       - name: Get release note files
         id: changed-files
@@ -35,5 +35,10 @@ jobs:
       - name: Check release notes
         if: steps.changed-files.outputs.any_changed == 'false' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
         run: |
+          # Check if any of the commit messages contain tags ci/docs/test
+          if git log --pretty=%s origin/main..HEAD | grep -E '^(ci:|docs:|test:)' > /dev/null; then
+            echo "Skipping release note check for commits with 'ci:', 'docs:', or 'test:' tags."
+          else
             echo "::error::The release notes file is missing, please add one or attach the label 'ignore-for-release-notes' to this PR."
             exit 1
+          fi

.github/workflows/rest_api_tests.yml

@@ -18,8 +18,6 @@ on:
       - "rest_api/pyproject.toml"
 
 env:
-  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
   PYTHON_VERSION: "3.8"
 
 jobs:

.github/workflows/tests.yml

@@ -17,13 +17,13 @@ on:
     paths:
       - "**.py"
       - "pyproject.toml"
+      - "!haystack/preview/**/*.py"  # See tests_preview.yml
+      - "!test/preview/**/*.py"  # See tests_preview.yml
       - "!.github/**/*.py"
       - "!rest_api/**/*.py"
       - "!docs/**/*.py"
 
 env:
-  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
   PYTHON_VERSION: "3.8"