Merge branch 'main' of https://github.com/deepchecks/MLChecks into 0.8.x

.github/scripts/benchmark-history.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+
+set -eu
+ENV_PATH=$(realpath venv)
+PIP_PATH=$ENV_PATH/bin/pip
+
+configure_asv () {
+    cat << EOF > asv.conf.json
+{
+    "version": 1,
+    "repo": ".",
+    "branches": ["HEAD"],
+    "environment_type": "virtualenv",
+}
+EOF
+}
+
+run_asv () {
+    $PIP_PATH install -e .
+    git show --no-patch --format="%H (%s)"
+    configure_asv
+    $ENV_PATH/bin/asv run -E existing --set-commit-hash $(git rev-parse HEAD)
+}
+
+$PIP_PATH install asv
+configure_asv
+$ENV_PATH/bin/asv  machine --yes
+
+run_asv
+
+git fetch origin gh-pages:gh-pages
+$ENV_PATH/bin/asv gh-pages

.github/scripts/benchmark.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+
+set -eu
+ENV_PATH=$(realpath venv)
+PIP_PATH=$ENV_PATH/bin/pip
+
+configure_asv () {
+    cat << EOF > asv.conf.json
+{
+    "version": 1,
+    "repo": ".",
+    "branches": ["HEAD"],
+    "environment_type": "virtualenv",
+}
+EOF
+}
+
+run_asv () {
+    $PIP_PATH install -e .
+    git show --no-patch --format="%H (%s)"
+    configure_asv
+    $ENV_PATH/bin/asv run -E existing --set-commit-hash $(git rev-parse HEAD)
+}
+
+$PIP_PATH install asv
+configure_asv
+$ENV_PATH/bin/asv  machine --yes
+
+git update-ref refs/bm/pr HEAD
+# We know this is a PR run. The branch is a GitHub refs/pull/*/merge ref, so
+# the current target that this PR will be merged into is HEAD^1.
+
+run_asv
+
+git fetch
+git checkout --force main
+
+run_asv
+
+$ENV_PATH/bin/asv compare main refs/bm/pr

.github/workflows/benchmark-history.yml

@@ -0,0 +1,43 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+name: Benchmark-History
+
+on:
+  push:
+    branches: [ main ]
+  # Manual run
+  workflow_dispatch:
+
+env:
+  DISABLE_DEEPCHECKS_ANONYMOUS_TELEMETRY: "true"
+
+jobs:
+  benchmark-history:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup git user
+      run: |
+        git config --global user.name "$(git --no-pager log --format=format:'%an' -n 1)"
+        git config --global user.email "$(git --no-pager log --format=format:'%ae' -n 1)"
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - name: Set Up Env
+      run: make env
+    - name: pip freeze
+      run: make freeze
+    - name: Run benchmarks
+      run: |
+        .github/scripts/benchmark-history.sh

.github/workflows/benchmark.yml

@@ -0,0 +1,37 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+name: benchmarks
+
+on:
+  pull_request:
+    branches: [ main ]
+
+env:
+  DISABLE_DEEPCHECKS_ANONYMOUS_TELEMETRY: "true"
+
+jobs:
+  benchmark:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - name: Set Up Env
+      run: make env
+    - name: pip freeze
+      run: make freeze
+    - name: Run benchmarks
+      run: |
+        .github/scripts/benchmark.sh

.github/workflows/build.yml

@@ -107,7 +107,7 @@ jobs:
       with:
         requirements: 'requirements-all.txt'
         fail: 'Copyleft,Other,Error'
-        exclude: '(pyzmq.*23\.1\.0|debugpy.*1\.6\.0|certifi.*2022\.6\.15|tqdm.*4\.64\.0|webencodings.*0\.5\.1|torch.*1\.10\.2.*|torchaudio.*0\.10\.2.*|torchvision.*0\.11\.3.*|terminado.*0\.15\.0)'
+        exclude: '(pyzmq.*23\.2\.0|debugpy.*1\.6\.0|certifi.*2022\.6\.15|tqdm.*4\.64\.0|webencodings.*0\.5\.1|torch.*1\.10\.2.*|torchaudio.*0\.10\.2.*|torchvision.*0\.11\.3.*|terminado.*0\.15\.0)'
         # pyzmq is Revised BSD https://github.com/zeromq/pyzmq/blob/main/examples/LICENSE
         # debugpy is MIT https://github.com/microsoft/debugpy/blob/main/LICENSE
         # certifi is MPL-2.0 https://github.com/certifi/python-certifi/blob/master/LICENSE

.github/workflows/label-new-issue.yml

@@ -20,4 +20,4 @@ jobs:
     steps:
       - uses: andymckay/labeler@1.0.4
         with:
-          add-labels: "Needs Triage"
+          add-labels: "needs triage"

.github/workflows/pr.yaml

@@ -26,7 +26,7 @@ jobs:
         with:
           mode: minimum
           count: 1
-          labels: "feature, bugfix, tests, ci, documentation, refactoring"
+          labels: "feature, bug, tests, ci, documentation, refactoring"
       - uses: mheap/github-action-required-labels@v1
         with:
           mode: exactly

.gitignore

@@ -112,6 +112,12 @@ docs/source/user-guide/vision/auto_tutorials
 docs/source/tutorials/vision/*.html
 docs/source/tutorials/vision/deepchecks_formatted_image*.jpg
 docs/source/tutorials/vision/tomato-detection/
+docs/source/user-guide/vision/tutorials/tomato-detection
+docs/source/user-guide/vision/tutorials/hymenoptera_data
 docs/source/tutorials/vision/hymenoptera_data/
+docs/source/user-guide/vision/tutorials/EuroSAT
 docs/source/user-guide/general/exporting_results/wandb
 deepchecks/vision/datasets/classification/models/
+
+# ignore benchmark results
+benchmarks/results

benchmarks/__init__.py

@@ -0,0 +1,10 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#