update 360-aggregated-feature before PR to main

.github/workflows/ci.yaml

@@ -7,30 +7,11 @@ on:
     branches:
 
 jobs:
-  security:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
-      - name: Perform gitleaks checks
-        run: |
-          # Download and check
-          curl -LJO https://github.com/gitleaks/gitleaks/releases/download/v8.18.2/gitleaks_8.18.2_linux_x64.tar.gz
-          curl -LJO https://github.com/gitleaks/gitleaks/releases/download/v8.18.2/gitleaks_8.18.2_checksums.txt
-          shasum -a 256 --ignore-missing --quiet -c gitleaks_8.18.2_checksums.txt
-          if [ $? != 0 ]; then exit 1; fi
-          # Extract gitleaks
-          tar -zxvf gitleaks_8.18.2_linux_x64.tar.gz gitleaks
-          # Run gitleaks
-          ./gitleaks detect \
-            --config .gitleaks.toml \
-            --gitleaks-ignore-path .gitleaksignore \
-            --no-git
   quality:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
       - name: Setting up PDM
         uses: pdm-project/setup-pdm@568ddd69406b30de1774ec0044b73ae06e716aa4 # v4
         with:
@@ -60,7 +41,7 @@ jobs:
           sudo add-apt-repository ppa:openslide/openslide
           sudo apt install -y openslide-tools
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
           lfs: true
       - name: Setting up PDM
@@ -74,3 +55,42 @@ jobs:
           python-versions: ${{ matrix.python-version }}
       - name: Executing unit tests
         run: nox -s test
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+      - name: Setting up PDM
+        uses: pdm-project/setup-pdm@568ddd69406b30de1774ec0044b73ae06e716aa4 # v4
+        with:
+          python-version: "3.10"
+          architecture: x64
+      - name: Setting up nox
+        uses: wntrblm/nox@5656fcedc31a1ea37d016e4d94d00185330cc528 # 2024.04.15
+        with:
+          python-versions: "3.10"
+      - name: Configure Git Credentials
+        run: |
+          git config user.email "action@github.com"
+          git config user.name "GitHub Action"
+      - name: Building docs
+        run: nox -s docs -- deploy --update-aliases dev
+  security:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+      - name: Perform gitleaks checks
+        run: |
+          # Download and check
+          curl -LJO https://github.com/gitleaks/gitleaks/releases/download/v8.18.2/gitleaks_8.18.2_linux_x64.tar.gz
+          curl -LJO https://github.com/gitleaks/gitleaks/releases/download/v8.18.2/gitleaks_8.18.2_checksums.txt
+          shasum -a 256 --ignore-missing --quiet -c gitleaks_8.18.2_checksums.txt
+          if [ $? != 0 ]; then exit 1; fi
+          # Extract gitleaks
+          tar -zxvf gitleaks_8.18.2_linux_x64.tar.gz gitleaks
+          # Run gitleaks
+          ./gitleaks detect \
+            --config .gitleaks.toml \
+            --gitleaks-ignore-path .gitleaksignore \
+            --no-git

.github/workflows/docs.yaml

@@ -13,7 +13,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
           fetch-depth: 0
       - name: Setting up PDM

.github/workflows/release.yaml

@@ -2,6 +2,7 @@
 name: Release
 
 on:
+  workflow_dispatch:
   push:
     tags:
       - "*"
@@ -13,7 +14,7 @@ jobs:
       id-token: write
       contents: write
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
       - name: Setting up PDM
         uses: pdm-project/setup-pdm@568ddd69406b30de1774ec0044b73ae06e716aa4 # v4
         with:

README.md

@@ -1,14 +1,18 @@
 <div align="center">
 
-<img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/eva-logo.png?raw=true" width="400">
+<br />
+
+<img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/eva-logo.png?raw=true" width="340">
 
+<br />
 <br />
 
 _Oncology FM Evaluation Framework by kaiko.ai_
 
 [![PyPI](https://img.shields.io/pypi/v/kaiko-eva.svg?logo=python)](https://pypi.python.org/pypi/kaiko-eva)
-[![docs](https://img.shields.io/badge/docs-latest-green)](https://kaiko-ai.github.io/eva/latest)
-[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg?labelColor=gray)](https://github.com/kaiko-ai/eva#license)
+[![docs](https://img.shields.io/badge/📚_docs-latest-green)](https://kaiko-ai.github.io/eva/latest)
+[![license](https://img.shields.io/badge/⚖️_License-Apache%202.0-blue.svg?labelColor=gray)](https://github.com/kaiko-ai/eva#license)<br>
+[![paper](http://img.shields.io/badge/OpenReview-MIDL_2024-B31B1B.svg)](https://openreview.net/forum?id=FNBQOPj18N&noteId=FNBQOPj18N)
 
 <p align="center">
   <a href="https://github.com/kaiko-ai/eva#installation">Installation</a> •
@@ -59,12 +63,12 @@ eva --version
 
 ## How To Use
 
-_eva_ can be used directly from the terminal as a CLI tool as follows:
+_`eva`_ can be used directly from the terminal as a CLI tool as follows:
 ```sh
 eva {fit,predict,predict_fit} --config url/or/path/to/the/config.yaml 
 ```
 
-When used as a CLI tool, `_eva_` supports configuration files (`.yaml`) as an argument to define its functionality.
+When used as a CLI tool, _`eva`_ supports configuration files (`.yaml`) as an argument to define its functionality.
 Native supported configs can be found at the [configs](https://github.com/kaiko-ai/eva/tree/main/configs) directory
 of the repo. Apart from cloning the repo, you can download the latest config folder as `.zip` from your browser from
 [here](https://download-directory.github.io/?url=https://github.com/kaiko-ai/eva/tree/main/configs). Alternatively,
@@ -98,7 +102,7 @@ and [tutorials](https://kaiko-ai.github.io/eva/dev/user-guide/advanced/replicate
 
 ## Benchmarks
 
-In this section you will find model benchmarks which were generated with `_eva_`.
+In this section you will find model benchmarks which were generated with _`eva`_.
 
 ### Table I: WSI patch-level benchmark
 
@@ -129,15 +133,15 @@ over 5 runs, with an average standard deviation of ±0.003._
 <br />
 
 _References_:
-1. _"Emerging properties in self-supervised vision transformers”_
-2. _"Benchmarking self-supervised learning on diverse pathology datasets”_
-3. _"Scaling self-supervised learning for histopathology with masked image modeling”_
-4. _"A General-Purpose Self-Supervised Model for Computational Pathology”_
-5. _"Towards Training Large-Scale Pathology Foundation Models: from TCGA to Hospital Scale”_
+1. _"Emerging properties in self-supervised vision transformers”_, [arXiv](https://arxiv.org/abs/2104.14294)
+2. _"Benchmarking self-supervised learning on diverse pathology datasets”_, [arXiv](https://arxiv.org/abs/2212.04690)
+3. _"Scaling self-supervised learning for histopathology with masked image modeling”_, [medRxiv](https://www.medrxiv.org/content/10.1101/2023.07.21.23292757v1)
+4. _"A General-Purpose Self-Supervised Model for Computational Pathology”_, [arXiv](https://arxiv.org/abs/2308.15474)
+5. _"Towards Training Large-Scale Pathology Foundation Models: from TCGA to Hospital Scale”_, [arXiv](https://arxiv.org/pdf/2404.15217)
 
 ## Contributing
 
-_eva_ is an open source project and welcomes contributions of all kinds. Please checkout the [developer](./docs/DEVELOPER_GUIDE.md)
+_`eva`_ is an open source project and welcomes contributions of all kinds. Please checkout the [developer](./docs/DEVELOPER_GUIDE.md)
 and [contributing guide](./docs/CONTRIBUTING.md) for help on how to do so.
 
 All contributors must follow the [code of conduct](./docs/CODE_OF_CONDUCT.md).
@@ -162,7 +166,23 @@ Our codebase is built using multiple opensource contributions
 
 </div>
 
----
+
+## Citation
+
+If you find this repository useful, please consider giving a star ⭐ and adding the following citation:
+
+```bibtex
+@inproceedings{kaiko.ai2024eva,
+    title={eva: Evaluation framework for pathology foundation models},
+    author={kaiko.ai and Ioannis Gatopoulos and Nicolas K{\"a}nzig and Roman Moser and Sebastian Ot{\'a}lora},
+    booktitle={Medical Imaging with Deep Learning},
+    year={2024},
+    url={https://openreview.net/forum?id=FNBQOPj18N}
+}
+```
+
+<br />
+
 <div align="center">
   <img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/kaiko-logo.png?raw=true" width="200">
 </div>

configs/vision/dino_vit/offline/bach.yaml

@@ -6,6 +6,7 @@ trainer:
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:DINO_BACKBONE, dino_vits16}/offline/bach}
     max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 12500}
     callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
       - class_path: lightning.pytorch.callbacks.LearningRateMonitor
         init_args:
           logging_interval: epoch
@@ -22,7 +23,7 @@ trainer:
           patience: 400
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
-      - class_path: eva.callbacks.EmbeddingsWriter
+      - class_path: eva.callbacks.ClassificationEmbeddingsWriter
         init_args:
           output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings}/${oc.env:DINO_BACKBONE, dino_vits16}/bach
           dataloader_idx_map:
@@ -89,12 +90,12 @@ data:
           init_args: &PREDICT_DATASET_ARGS
             root: ${oc.env:DATA_ROOT, ./data}/bach
             split: train
-            download: false
+            download: ${oc.env:DOWNLOAD, false}
             # Set `download: true` to download the dataset from https://zenodo.org/records/3632035
             # The BACH dataset is distributed under the following license
             # Attribution-NonCommercial-NoDerivs 4.0 International license
             # (see: https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode)
-            image_transforms:
+            transforms:
               class_path: eva.vision.data.transforms.common.ResizeAndCrop
               init_args:
                 size: ${oc.env:RESIZE_DIM, 224}  

configs/vision/dino_vit/offline/camelyon16.yaml

@@ -22,7 +22,7 @@ trainer:
           patience: ${oc.env:PATIENCE, 10}
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
-      - class_path: eva.callbacks.EmbeddingsWriter
+      - class_path: eva.callbacks.ClassificationEmbeddingsWriter
         init_args:
           output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings/${oc.env:DINO_BACKBONE, dino_vits16}/camelyon16}
           dataloader_idx_map:
@@ -98,7 +98,7 @@ data:
       predict:
         - class_path: eva.vision.datasets.Camelyon16
           init_args: &PREDICT_DATASET_ARGS
-            root: ${oc.env:DATA_ROOT, ./data}/camelyon16
+            root: ${oc.env:DATA_ROOT, ./data/camelyon16}
             sampler:
               class_path: eva.vision.data.wsi.patching.samplers.ForegroundGridSampler
               init_args:

configs/vision/dino_vit/offline/crc.yaml

@@ -6,6 +6,7 @@ trainer:
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:DINO_BACKBONE, dino_vits16}/offline/crc}
     max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 12500}
     callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
       - class_path: lightning.pytorch.callbacks.LearningRateMonitor
         init_args:
           logging_interval: epoch
@@ -22,7 +23,7 @@ trainer:
           patience: 24
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
-      - class_path: eva.callbacks.EmbeddingsWriter
+      - class_path: eva.callbacks.ClassificationEmbeddingsWriter
         init_args:
           output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings}/${oc.env:DINO_BACKBONE, dino_vits16}/crc
           dataloader_idx_map:
@@ -89,11 +90,11 @@ data:
           init_args: &PREDICT_DATASET_ARGS
             root: ${oc.env:DATA_ROOT, ./data}/crc
             split: train
-            download: false
+            download: ${oc.env:DOWNLOAD, false}
             # Set `download: true` to download the dataset from https://zenodo.org/records/1214456
             # The CRC dataset is distributed under the following license: "CC BY 4.0 LEGAL CODE"
             # (see: https://creativecommons.org/licenses/by/4.0/legalcode)
-            image_transforms:
+            transforms:
               class_path: eva.vision.data.transforms.common.ResizeAndCrop
               init_args:
                 mean: ${oc.env:NORMALIZE_MEAN, [0.485, 0.456, 0.406]}

configs/vision/dino_vit/offline/mhist.yaml

@@ -6,6 +6,7 @@ trainer:
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:DINO_BACKBONE, dino_vits16}/offline/mhist}
     max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 12500}
     callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
       - class_path: lightning.pytorch.callbacks.LearningRateMonitor
         init_args:
           logging_interval: epoch
@@ -22,7 +23,7 @@ trainer:
           patience: 51
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
-      - class_path: eva.callbacks.EmbeddingsWriter
+      - class_path: eva.callbacks.ClassificationEmbeddingsWriter
         init_args:
           output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings}/${oc.env:DINO_BACKBONE, dino_vits16}/mhist
           dataloader_idx_map:
@@ -78,7 +79,9 @@ data:
           manifest_file: manifest.csv
           split: train
           target_transforms:
-            class_path: eva.core.data.transforms.ArrayToFloatTensor
+            class_path: torchvision.transforms.v2.ToDtype
+            init_args:
+              dtype: torch.float32
       val:
         class_path: eva.datasets.EmbeddingsClassificationDataset
         init_args:
@@ -89,7 +92,7 @@ data:
           init_args: &PREDICT_DATASET_ARGS
             root: ${oc.env:DATA_ROOT, ./data}/mhist
             split: train
-            image_transforms:
+            transforms:
               class_path: eva.vision.data.transforms.common.ResizeAndCrop
               init_args:
                 size: ${oc.env:RESIZE_DIM, 224}

configs/vision/dino_vit/offline/panda.yaml

@@ -22,7 +22,7 @@ trainer:
           patience: ${oc.env:PATIENCE, 8}
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
-      - class_path: eva.callbacks.EmbeddingsWriter
+      - class_path: eva.callbacks.ClassificationEmbeddingsWriter
         init_args:
           output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings/${oc.env:DINO_BACKBONE, dino_vits16}/panda}
           dataloader_idx_map:
@@ -98,7 +98,7 @@ data:
       predict:
         - class_path: eva.vision.datasets.PANDA
           init_args: &PREDICT_DATASET_ARGS
-            root: ${oc.env:DATA_ROOT, ./data}/panda/prostate-cancer-grade-assessment
+            root: ${oc.env:DATA_ROOT, ./data/panda/prostate-cancer-grade-assessment}
             sampler:
               class_path: eva.vision.data.wsi.patching.samplers.ForegroundGridSampler
               init_args:

configs/vision/dino_vit/offline/patch_camelyon.yaml

@@ -6,6 +6,7 @@ trainer:
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:DINO_BACKBONE, dino_vits16}/offline/patch_camelyon}
     max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 12500}
     callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
       - class_path: lightning.pytorch.callbacks.LearningRateMonitor
         init_args:
           logging_interval: epoch
@@ -22,7 +23,7 @@ trainer:
           patience: 9
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
-      - class_path: eva.callbacks.EmbeddingsWriter
+      - class_path: eva.callbacks.ClassificationEmbeddingsWriter
         init_args:
           output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings}/${oc.env:DINO_BACKBONE, dino_vits16}/patch_camelyon
           dataloader_idx_map:
@@ -79,7 +80,9 @@ data:
           manifest_file: manifest.csv
           split: train
           target_transforms:
-            class_path: eva.core.data.transforms.ArrayToFloatTensor
+            class_path: torchvision.transforms.v2.ToDtype
+            init_args:
+              dtype: torch.float32
       val:
         class_path: eva.datasets.EmbeddingsClassificationDataset
         init_args:
@@ -95,12 +98,12 @@ data:
           init_args: &PREDICT_DATASET_ARGS
             root: ${oc.env:DATA_ROOT, ./data}/patch_camelyon
             split: train
-            download: false
+            download: ${oc.env:DOWNLOAD, false}
             # Set `download: true` to download the dataset from https://zenodo.org/records/1494286
             # The PatchCamelyon dataset is distributed under the following license: 
             # "Creative Commons Zero v1.0 Universal"
             # (see: https://choosealicense.com/licenses/cc0-1.0/)
-            image_transforms:
+            transforms:
               class_path: eva.vision.data.transforms.common.ResizeAndCrop
               init_args:
                 size: ${oc.env:RESIZE_DIM, 224}

configs/vision/dino_vit/online/bach.yaml

@@ -5,6 +5,7 @@ trainer:
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:DINO_BACKBONE, dino_vits16}/online/bach}
     max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 12500}
     callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
       - class_path: lightning.pytorch.callbacks.LearningRateMonitor
         init_args:
           logging_interval: epoch
@@ -76,7 +77,7 @@ data:
           # The BACH dataset is distributed under the following license
           # Attribution-NonCommercial-NoDerivs 4.0 International license
           # (see: https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode)
-          image_transforms:
+          transforms:
             class_path: eva.vision.data.transforms.common.ResizeAndCrop
             init_args:
               size: ${oc.env:RESIZE_DIM, 224}