Merge branch 'master' into pr/raphaelauv/10752

Signed-off-by: harupy <17039389+harupy@users.noreply.github.com>

.circleci/config.yml

@@ -3,13 +3,26 @@ version: 2.1
 orbs:
   win: circleci/windows@5.0.0
 
+commands:
+  fetch_merge_ref:
+    description: "Fetch the merge ref for a pull request"
+    steps:
+      - run:
+          name: Fetch merge ref
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              git fetch origin refs/pull/$CIRCLE_PR_NUMBER/merge
+              git checkout FETCH_HEAD
+            fi
+
 jobs:
   build_doc_r:
     machine:
       image: ubuntu-2004:202111-01
 
     steps:
       - checkout
+      - fetch_merge_ref
       - run:
           name: Build documentation
           working_directory: docs
@@ -26,6 +39,7 @@ jobs:
 
     steps:
       - checkout
+      - fetch_merge_ref
       - run:
           name: Install Java
           command: |
@@ -52,6 +66,7 @@ jobs:
           command: |
             make rsthtml
             make javadocs
+            python changed_pages.py
       - run:
           name: Check for broken links
           working_directory: docs
@@ -71,6 +86,7 @@ jobs:
 
     steps:
       - checkout
+      - fetch_merge_ref
       - run:
           name: Pull submodule
           command: |

.github/actions/setup-java/action.yml

@@ -14,7 +14,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         java-version: ${{ inputs.java-version }}
         distribution: ${{ inputs.distribution }}

.github/actions/setup-pyenv/action.yml

@@ -18,9 +18,6 @@ runs:
       shell: bash
       run: |
         git clone https://github.com/pyenv/pyenv.git "$HOME/.pyenv"
-        cd $HOME/.pyenv
-        # Checkout cb5da5e831fb50ce0ac44fc13617965b2309a482 to avoid https://github.com/pyenv/pyenv/commit/ca1593c80eae0b84349ca163699b71b04ea8b9fa that broke tests for virtualenv project execution
-        git checkout cb5da5e831fb50ce0ac44fc13617965b2309a482
     - name: Setup environment variables
       if: runner.os == 'Linux'
       shell: bash

.github/actions/setup-python/action.yml

@@ -28,15 +28,15 @@ runs:
             # GitHub Actions does not have a Python 3.8.12 binary for Windows as of Oct. 27 2022
             python_version="3.8.10"
           else
-            python_version="3.8.12"
+            python_version="3.8.18"
           fi
         elif [[ "$python_version" == "3.9" ]]; then
-          python_version="3.9.13"
+          python_version="3.9.18"
         else
           echo "Invalid python version: '$python_version'. Must be '3.8'"
           exit 1
         fi
         echo "version=$python_version" >> $GITHUB_OUTPUT
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ steps.get-python-version.outputs.version }}

.github/actions/untracked/action.yml

@@ -1,6 +1,6 @@
 name: "untracked"
 description: "Detect untracked files"
 runs:
-  using: "node16"
+  using: "node20"
   main: "index.js"
   post: "post.js"

.github/workflows/advice.yml

@@ -16,10 +16,10 @@ jobs:
     permissions:
       pull-requests: write # advice.js comments on PRs
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         with:
           script: |
             const script = require(

.github/workflows/autoformat.yml

@@ -30,10 +30,10 @@ jobs:
       base_repo: ${{ fromJSON(steps.judge.outputs.result).base_repo }}
       pull_number: ${{ fromJSON(steps.judge.outputs.result).pull_number }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: judge
         id: judge
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -62,7 +62,7 @@ jobs:
     outputs:
       reformatted: ${{ steps.patch.outputs.reformatted }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ needs.check-comment.outputs.repository }}
           ref: ${{ needs.check-comment.outputs.head_ref }}
@@ -74,7 +74,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           changed_files="$(gh pr view --repo ${{ github.repository }} ${{ needs.check-comment.outputs.pull_number }} --json files --jq '.files.[].path')"
-          protos=$([[ -z $(echo "$changed_files" | grep '^mlflow/protos') ]] && echo "false" || echo "true")
+          protos=$([[ -z $(echo "$changed_files" | grep '^\(mlflow/protos\|tests/protos\)') ]] && echo "false" || echo "true")
           python=$([[ -z $(echo "$changed_files" | grep '\.py$\|^pyproject\.toml$') ]] && echo "false" || echo "true")
           js=$([[ -z $(echo "$changed_files" | grep '^mlflow/server/js') ]] && echo "false" || echo "true")
           r=$([[ -z $(echo "$changed_files" | grep '^mlflow/R/mlflow') ]] && echo "false" || echo "true")
@@ -115,8 +115,8 @@ jobs:
       # ************************************************************************
       - if: steps.diff.outputs.python == 'true'
         run: |
-          ruff --fix .
-          black .
+          ruff check --fix .
+          ruff format .
           blacken-docs $(git ls-files '*.py' '*.rst' '*.md') || true
       # ************************************************************************
       # js
@@ -155,7 +155,7 @@ jobs:
           echo "reformatted=$reformatted" >> $GITHUB_OUTPUT
 
       - name: Upload patch
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ github.run_id }}.diff
           path: ${{ github.run_id }}.diff
@@ -166,7 +166,7 @@ jobs:
     needs: [check-comment, format]
     if: ${{ needs.format.outputs.reformatted == 'true' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ needs.check-comment.outputs.repository }}
           ref: ${{ needs.check-comment.outputs.head_ref }}
@@ -183,7 +183,7 @@ jobs:
           git merge base/${{ needs.check-comment.outputs.base_ref }}
 
       - name: Download patch
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ github.run_id }}.diff
           path: /tmp
@@ -202,9 +202,9 @@ jobs:
     permissions:
       statuses: write # autoformat.updateStatus
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Update status
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/cancel.yml

@@ -16,8 +16,8 @@ jobs:
     permissions:
       actions: write # to cancel workflow runs
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/github-script@v6
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/closing-pr.yml

@@ -21,10 +21,10 @@ jobs:
       pull-requests: read # closing-pr.js reads the PR body
       issues: write # closing-pr.js labels issues
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         with:
           script: |
             const script = require(

.github/workflows/cross-version-tests.yml

@@ -60,7 +60,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       is_matrix_empty: ${{ steps.set-matrix.outputs.is_matrix_empty }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
           repository: ${{ github.event_name == 'schedule' && 'mlflow/mlflow' || github.event.inputs.repository }}
@@ -72,7 +72,7 @@ jobs:
           pip install -r dev/requirements.txt
           pip install pytest pytest-cov
       - name: Check labels
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         id: check-labels
         with:
           script: |
@@ -94,7 +94,7 @@ jobs:
             };
       - name: Test set_matrix.py
         run: |
-          python -m pytest --noconftest tests/dev/test_set_matrix.py
+          python -m pytest --noconftest dev/tests
       - id: set-matrix
         name: Set matrix
         env:
@@ -126,7 +126,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
           repository: ${{ github.event_name == 'schedule' && 'mlflow/mlflow' || github.event.inputs.repository }}
@@ -188,6 +188,13 @@ jobs:
       - name: Install ${{ matrix.package }} ${{ matrix.version }}
         run: |
           ${{ matrix.install }}
+          # temp workaround before langchain_community release the following PR in a new version
+          # https://github.com/langchain-ai/langchain/pull/15691
+          if [ "${{ matrix.package }}/${{ matrix.category }}" = "langchain/autologging" ]
+          then
+            pip uninstall -y langchain_community
+            pip install git+https://github.com/hwchase17/langchain#subdirectory=libs/community
+          fi
       - name: Install pydantic no-binary for langchain
         if: matrix.package == 'langchain' && ( matrix.version == 'dev' || matrix.version >= '0.0.267')
         run: |
@@ -199,12 +206,17 @@ jobs:
       - name: Check package versions
         run: |
           python dev/show_package_release_dates.py
+      - name: Prefetch Transformer models
+        if: matrix.package == 'transformers' && matrix.category == 'models'
+        run: |
+          python tests/transformers/helper.py
       - uses: ./.github/actions/pipdeptree
       - name: Run tests
         env:
           MLFLOW_CONDA_HOME: /usr/share/miniconda
           SPARK_LOCAL_IP: localhost
           PACKAGE_VERSION: ${{ matrix.version }}
           JOHNSNOWLABS_LICENSE_JSON: ${{ secrets.JOHNSNOWLABS_LICENSE_JSON }}
+          HF_HUB_ENABLE_HF_TRANSFER: 1
         run: |
           ${{ matrix.run }}

.github/workflows/deployments.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/untracked
       - uses: ./.github/actions/setup-python
       - name: Install dependencies

.github/workflows/dev-setup.yml

@@ -41,7 +41,7 @@ jobs:
     timeout-minutes: 30
     if: github.event_name != 'schedule' || github.repository == 'mlflow-automation/mlflow'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
           repository: ${{ github.event_name == 'schedule' && 'mlflow/mlflow' || github.event.inputs.repository }}

.github/workflows/devcontainer.yml

@@ -36,7 +36,7 @@ jobs:
     permissions:
       packages: write # to push to ghcr.io
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Get image name
         run: |
           IMAGE_NAME=$(yq '.services.mlflow.image' .devcontainer/docker-compose.yml)

.github/workflows/examples.yml

@@ -47,7 +47,7 @@ jobs:
     timeout-minutes: 120
     if: github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'mlflow-automation/mlflow') || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ github.event_name == 'schedule' && 'mlflow/mlflow' || github.event.inputs.repository }}
           ref: ${{ github.event.inputs.ref }}
@@ -96,7 +96,7 @@ jobs:
     timeout-minutes: 120
     if: github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'mlflow-automation/mlflow') || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ github.event_name == 'schedule' && 'mlflow/mlflow' || github.event.inputs.repository }}
           ref: ${{ github.event.inputs.ref }}

.github/workflows/gateway.yml

@@ -32,14 +32,14 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/untracked
       - uses: ./.github/actions/setup-python
       - name: Install dependencies
         run: |
           pip install .[gateway] \
             pytest pytest-timeout pytest-asyncio \
-            httpx psutil sentence-transformers transformers
+            httpx psutil sentence-transformers transformers torchvision
           # Ensure pydantic v2 is installed
           pip freeze | tail -n +1 | grep -q '^pydantic==2\.'
       - name: Run tests with pydantic v2

.github/workflows/js.yml

@@ -32,6 +32,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
+        option: [--testPathPattern, --testPathIgnorePatterns]
         include:
           - os: ubuntu-latest
             shell: bash
@@ -43,7 +44,7 @@ jobs:
         shell: ${{ matrix.shell }}
         working-directory: mlflow/server/js
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - uses: actions/setup-node@v3
@@ -73,7 +74,7 @@ jobs:
           yarn i18n:check
       - name: Run tests
         run: |
-          yarn test
+          yarn test --silent ${{ matrix.option }} src/experiment-tracking/components
       - name: Run build
         if: runner.os == 'Linux'
         env:

.github/workflows/labeling.yml

@@ -26,7 +26,7 @@ jobs:
       issues: write # harupy/auto-labeling
     timeout-minutes: 120
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - uses: harupy/auto-labeling@master

.github/workflows/lint.yml

@@ -32,14 +32,14 @@ jobs:
     timeout-minutes: 30
     if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/untracked
       - uses: ./.github/actions/setup-python
         id: setup-python
       - name: Add problem matchers
         run: |
           echo "::add-matcher::.github/workflows/matchers/pylint.json"
-          echo "::add-matcher::.github/workflows/matchers/black.json"
+          echo "::add-matcher::.github/workflows/matchers/format.json"
           echo "::add-matcher::.github/workflows/matchers/ruff.json"
       - uses: ./.github/actions/cache-pip
       - name: Install dependencies