Implement Data Preparation MLCube

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/fets-ai/fetstool_docker_dependencies
+FROM ghcr.io/fets-ai/fetstool_docker_dependencies AS fets_base
 
 LABEL authors="FeTS_Admin <admin@fets.ai>"
 
@@ -11,15 +11,43 @@ RUN pwd && ls -l
 
 WORKDIR /Front-End
 
-COPY . .
+# Download model checkpoints to torch checkpoint location
+# https://pytorch.org/docs/stable/hub.html#where-are-my-downloaded-models-saved
+ENV TORCH_HOME="/.pytorch_cache"
+RUN mkdir -p $TORCH_HOME/hub/checkpoints && \
+    wget -O $TORCH_HOME/hub/checkpoints/dpn98-722954780.pth http://data.lip6.fr/cadene/pretrainedmodels/dpn98-722954780.pth --no-check-certificate && \
+    wget -O $TORCH_HOME/hub/checkpoints/resnet50-19c8e357.pth https://download.pytorch.org/models/resnet50-19c8e357.pth
+
+COPY src src
+
+COPY CMakeLists.txt README.txt LICENSE .
+
+COPY cmake_modules cmake_modules
+
+COPY data data
+
+COPY docs_sources docs_sources
 
 RUN pwd && ls -l
 
 ## C++ build
 RUN mkdir bin && cd bin && cmake -DCMAKE_INSTALL_PREFIX="./install/appdir/usr" -DITK_DIR="/CaPTk/bin/ITK-build" -DDCMTK_DIR="/CaPTk/bin/DCMTK-build" -DBUILD_TESTING=OFF .. && make -j$(nproc) && make install/strip 
 
 ## Python package installation
-RUN cd bin/install/appdir/usr/bin/ && python3.8 -m venv ./venv && ./venv/bin/pip install --upgrade pip wheel && ./venv/bin/pip install torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cpu && ./venv/bin/pip install -e . && ./venv/bin/pip install setuptools-rust Cython scikit-build scikit-learn openvino-dev==2023.0.1 && ./venv/bin/pip install -e .
+RUN apt-get install software-properties-common curl -y && \
+    add-apt-repository ppa:deadsnakes/ppa -y && apt-get update && \
+    apt-get install python3.8 python3.8-distutils -y && \
+    apt-get remove --purge python3.6 -y && \
+    apt autoremove -y && \
+    apt-get install python3.8-distutils -y && \
+    rm -fr /usr/bin/python /usr/bin/python3 /usr/bin/pip /usr/bin/pip3 && \
+    ln -s /usr/bin/python3.8 /usr/bin/python && ln -s /usr/bin/python3.8 /usr/bin/python3 && \
+    ln -s /usr/bin/pip3.8 /usr/bin/pip && ln -s /usr/bin/pip3.8 /usr/bin/pip3
+
+RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3.8 get-pip.py &&     rm get-pip.py
+
+RUN cd bin/install/appdir/usr/bin/ && pip install --upgrade pip wheel && pip install torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cpu && pip install -e . && pip install setuptools-rust Cython scikit-build scikit-learn openvino==2023.0.1 openvino-dev==2023.0.1 && pip install -e .
 
 ### put together a data example that is already aligned and ready to invoke the brain extraction and tumor segmentation
 
@@ -31,5 +59,39 @@ ENV QT_GRAPHICSSYSTEM="native"
 
 RUN echo "Env paths\n" && echo $PATH && echo $LD_LIBRARY_PATH
 
-# define entry point as the preparedataset script
-ENTRYPOINT ["/Front-End/bin/install/appdir/usr/bin/venv/bin/python", "/Front-End/bin/install/appdir/usr/bin/PrepareDataset.py"]
+# define entry point
+ENTRYPOINT ["python", "/Front-End/bin/install/appdir/usr/bin/PrepareDataset.py"]
+
+FROM fets_base AS data_prep
+
+RUN find /Front-End/bin/install/appdir/usr/bin -type f \( -perm -u=x -o -type l \) -exec cp -P {} /usr/bin \;
+
+WORKDIR /
+
+COPY ./mlcubes/data_preparation/project/requirements.txt /project/requirements.txt 
+
+RUN pip install --upgrade pip
+
+RUN pip install -r /project/requirements.txt
+
+ENV LANG C.UTF-8
+
+RUN mkdir /project/stages
+
+RUN cp /Front-End/bin/install/appdir/usr/bin/*.py /project/stages/
+
+RUN cp -R /Front-End/bin/install/appdir/usr/bin/data_prep_models /project/stages/data_prep_models
+
+# Hotfix: install more recent version of GaNDLF for metrics generation
+RUN pip install git+https://github.com/mlcommons/GaNDLF@616b37bafad8f89d5c816a88f44fa30470601311
+
+RUN pip install torch torchvision
+
+RUN pip install git+https://github.com/MIC-DKFZ/nnUNet.git@nnunetv1
+
+ENV nnUNet_raw_data_base="/tmp/nnUNet_raw_data_base"
+ENV nnUNet_preprocessed="/tmp/nnUNet_preprocessed"
+
+COPY ./mlcubes/data_preparation/project /project
+
+ENTRYPOINT ["python", "/project/mlcube.py"]

docs/itcr_connectivity.md

@@ -4,27 +4,35 @@ This section includes a reference of all ongoing and existing connections betwee
 A connectivity map featuring all ITCR projects can be found [here](https://www.ndexbio.org/#/network/04c0a7e8-af92-11e7-94d3-0ac135e8bacf).
 
 - [Existing Connections](#existing-connections)
+	- [OpenFL](#openfl)
+	- [GaNDLF](#gandlf)
+	- [CaPTk](#captk)
 	- [DCMTK](#dcmtk)
 	- [DCMQI](#dcmqi)
-	- [CaPTk](#captk)
 	- [Synapse PACS](#synapse-pacs)
 	- [TCIA and IDC](#tcia-and-idc)
 - [Ongoing Development](#ongoing-development)
 	- [XNAT](#xnat)
 	- [FLAIM](#flaim)
 	- [PRISM](#prism)
 
-##  Existing Connections
+##  Existing Connections 
 
-### DCMTK
-FeTS uses [DCMTK - DICOM ToolKit (DCMTK)](https://dicom.offis.de/dcmtk.php.en) for DICOM file handling. 
+### OpenFL
+FeTS leverages the [Open Federated Learning library (OpenFL)](https://github.com/securefederatedai/openfl) for enabling federated machine learning of models.
 
-### DCMQI
-FeTS leverages [DICOM for Quantitative Imaging (DCMQI)](http://qiicr.org/dcmqi-guide/tutorials/intro.html) for generating DICOM-Seg files from NIfTI files. 
+### GaNDLF
+FeTS leverages the [Generally Nuanced Deep Learning Framework (GaNDLF)](https://gandlf.org/) for enabling advanced preprocessing and deep learing based training.
 
 ### CaPTk
 FeTS leverages the [Cancer Imaging Phenomics Toolkit (CaPTk)](https://www.med.upenn.edu/cbica/captk) for integrating CaPTk's current functionality into FeTS. 
 
+### DCMTK
+FeTS uses [DCMTK - DICOM ToolKit (DCMTK)](https://dicom.offis.de/dcmtk.php.en) for DICOM file handling. 
+
+### DCMQI
+FeTS leverages [DICOM for Quantitative Imaging (DCMQI)](http://qiicr.org/dcmqi-guide/tutorials/intro.html) for generating DICOM-Seg files from NIfTI files.
+
 ### Synapse PACS
 FeTS' performance evaluation metrics are used by [Synapse PACS](https://www.synapse.org/).  
 
@@ -42,4 +50,4 @@ Interoperability and privacy preservation algorithmic comparison.
 ### PRISM
 Federated learning for segmentation of PRISM data.
 
-Contact [contact [at] fets.ai](mailto:contact@fets.ai) with any questions.
+Contact [contact [at] fets.ai](mailto:contact@fets.ai) with any questions.

mlcubes/.gitignore

@@ -0,0 +1,10 @@
+*.csv
+*.txt
+*.nii.gz
+*.mat
+*.dcm
+*.png
+*/mlcube/workspace/*
+!requirements.txt
+!*/mlcube/workspace/parameters.yaml
+models

mlcubes/data_preparation/mlcube/mlcube.yaml

@@ -0,0 +1,50 @@
+name: Data Preparator MLCube with Manual preparation steps
+description: Data Preparator MLCube showcasing examples were automated and manual steps are required. Provided by MLCommons
+authors:
+ - {name: MLCommons}
+
+platform:
+  accelerator_count: 0
+
+docker:
+  # Image name
+  image: mlcommons/rano-data-prep-mlcube:latest
+  # Docker build context relative to $MLCUBE_ROOT. Default is `build`.
+  build_context: "../project"
+  # Docker file name within docker build context, default is `Dockerfile`.
+  build_file: "Dockerfile"
+
+tasks:
+  prepare:
+    parameters:
+      inputs: {
+        data_path: input_data,
+        labels_path: input_labels,
+        parameters_file: parameters.yaml,
+        models: additional_files/models,
+      }
+      outputs: {
+        output_path: data/,
+        output_labels_path: labels/,
+        report_file: {type: file, default: report.yaml},
+        metadata_path: metadata/,
+      }
+  sanity_check:
+    parameters:
+      inputs: {
+        data_path: data/,
+        labels_path: labels/, 
+        parameters_file: parameters.yaml,
+        metadata_path: metadata/,
+      }
+  statistics:
+    parameters:
+      inputs: {
+        data_path: data/,
+        labels_path: labels/,
+        parameters_file: parameters.yaml,
+        metadata_path: metadata/,
+      }
+      outputs: {
+        output_path: {type: file, default: statistics.yaml}
+      }

mlcubes/data_preparation/mlcube/workspace/parameters.yaml

@@ -0,0 +1,21 @@
+seed: 2784
+train_percent: 0.8
+medperf_report_stages:
+- "IDENTIFIED"
+- "VALIDATED"
+- "MISSING_MODALITIES"
+- "EXTRA_MODALITIES"
+- "VALIDATION_FAILED"
+- "CONVERTED_TO_NIfTI"
+- "NIfTI_CONVERSION_FAILED"
+- "BRAIN_EXTRACT_FINISHED"
+- "BRAIN_EXTRACT_FINISHED"
+- "TUMOR_EXTRACT_FAILED"
+- "MANUAL_REVIEW_COMPLETE"
+- "MANUAL_REVIEW_REQUIRED"
+- "MULTIPLE_ANNOTATIONS_ERROR"
+- "COMPARISON_COMPLETE"
+- "EXACT_MATCH_IDENTIFIED"
+- "ANNOTATION_COMPARISON_FAILED"
+- "ANNOTATION_CONFIRMED"
+- "DONE"

mlcubes/data_preparation/project/mlcube.py

@@ -0,0 +1,62 @@
+"""MLCube handler file"""
+import typer
+import subprocess
+
+
+app = typer.Typer()
+
+
+def exec_python(cmd: str) -> None:
+    """Execute a python script as a subprocess
+
+    Args:
+        cmd (str): command to run as would be written inside the terminal
+    """
+    splitted_cmd = cmd.split()
+    process = subprocess.Popen(splitted_cmd, cwd=".")
+    process.wait()
+    assert process.returncode == 0, f"command failed: {cmd}"
+
+
+@app.command("prepare")
+def prepare(
+    data_path: str = typer.Option(..., "--data_path"),
+    labels_path: str = typer.Option(..., "--labels_path"),
+    parameters_file: str = typer.Option(..., "--parameters_file"),
+    models_path: str = typer.Option(..., "--models"),
+    output_path: str = typer.Option(..., "--output_path"),
+    output_labels_path: str = typer.Option(..., "--output_labels_path"),
+    report_file: str = typer.Option(..., "--report_file"),
+    metadata_path: str = typer.Option(..., "--metadata_path"),
+):
+    cmd = f"python3 project/prepare.py --data_path={data_path} --labels_path={labels_path} --models_path={models_path} --data_out={output_path} --labels_out={output_labels_path} --report={report_file} --parameters={parameters_file} --metadata_path={metadata_path}"
+    exec_python(cmd)
+
+
+@app.command("sanity_check")
+def sanity_check(
+    data_path: str = typer.Option(..., "--data_path"),
+    labels_path: str = typer.Option(..., "--labels_path"),
+    parameters_file: str = typer.Option(..., "--parameters_file"),
+    metadata_path: str = typer.Option(..., "--metadata_path"),
+):
+    # Modify the sanity_check command as needed
+    cmd = f"python3 project/sanity_check.py --data_path={data_path} --labels_path={labels_path} --metadata={metadata_path}"
+    exec_python(cmd)
+
+
+@app.command("statistics")
+def sanity_check(
+    data_path: str = typer.Option(..., "--data_path"),
+    labels_path: str = typer.Option(..., "--labels_path"),
+    parameters_file: str = typer.Option(..., "--parameters_file"),
+    metadata_path: str = typer.Option(..., "--metadata_path"),
+    out_path: str = typer.Option(..., "--output_path"),
+):
+    # Modify the statistics command as needed
+    cmd = f"python3 project/statistics.py --data_path={data_path} --labels_path={labels_path} --out_file={out_path} --metadata={metadata_path}"
+    exec_python(cmd)
+
+
+if __name__ == "__main__":
+    app()