Run and test batch_processing.sh using GitHub Actions

.github/workflows/test-batch-processing.yml

@@ -0,0 +1,54 @@
+name: Test batch_processing.sh
+
+on:
+  push:
+    branches:
+      - master
+      - release
+  pull_request:
+    branches:
+      - '*'
+  schedule:
+    # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule
+    # > Scheduled workflows run on the latest commit on the default or base branch
+    # i.e. this can only run on master
+    - cron:  '0 11 * * *'
+
+env:
+  # Turns on color output for pytest. See: https://github.com/pytest-dev/pytest/issues/7443#issuecomment-656642591
+  PY_COLORS: "1"
+  # Used in 'pytest.mark.skipif()' for test, which prevents the test from being run during the usual test suite
+  BATCH_PROCESSING_CI_JOB: "1"
+
+jobs:
+  batch_processing:
+    name: Test batch_processing.sh
+    # Both Ubuntu and macOS are relevant because of https://github.com/neuropoly/spinalcordtoolbox/issues/3194
+    strategy:
+      matrix:
+        os: [ ubuntu-18.04, macos-10.15 ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout SCT
+        uses: actions/checkout@v2
+
+      # install_sct edits ~/.bashrc, but those environment changes don't get passed to subsequent steps in GH Actions.
+      # So, we filter through the .bashrc and pass the values to $GITHUB_ENV and $GITHUB_PATH.
+      # Relevant documentation: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#environment-files
+      # This workaround should be replaced by https://github.com/neuropoly/spinalcordtoolbox/pull/3198#discussion_r568225392
+      - name: Install SCT
+        run: |
+          PIP_PROGRESS_BAR=off ./install_sct -y
+          cat ~/.bashrc | grep "export SCT_DIR" | cut -d " " -f 2 >> $GITHUB_ENV
+          cat ~/.bashrc | grep "export MPLBACKEND" | cut -d " " -f 2 >> $GITHUB_ENV
+          cat ~/.bashrc | grep "export PATH" | grep -o "/.*" | cut -d ':' -f 1 >> $GITHUB_PATH
+
+      - name: Run batch_processing.sh
+        run: |
+          ./batch_processing.sh
+
+      - name: Validate new results
+        run: |
+          source python/etc/profile.d/conda.sh
+          conda activate venv_sct
+          pytest unit_testing/batch_processing/test_batch_processing.py

unit_testing/batch_processing/cached_results/dmri/fa_in_cst.csv

@@ -0,0 +1,3 @@
+Timestamp,SCT Version,Filename,Slice (I->S),VertLevel,Label,Size [vox],WA(),STD()
+2021-01-25 00:50:28,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/dmri/dti_FA.nii.gz,2:14,,WM left lateral corticospinal tract,70.42203159235883,0.7800112141091362,0.1347296394611586
+2021-01-25 00:50:28,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/dmri/dti_FA.nii.gz,2:14,,WM right lateral corticospinal tract,71.18283749436773,0.7756265423313133,0.13633855479631649

unit_testing/batch_processing/cached_results/mt/mtr_in_wm.csv

@@ -0,0 +1,2 @@
+Timestamp,SCT Version,Filename,Slice (I->S),VertLevel,Label,Size [vox],MAP(),STD()
+2021-01-25 00:48:42,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/mt/mtr.nii.gz,4:17,2:5,white matter,1866.8310447890826,54.38001541207312,4.851424830774496

unit_testing/batch_processing/cached_results/t2/csa_c2c3.csv

@@ -0,0 +1,2 @@
+Timestamp,SCT Version,Filename,Slice (I->S),VertLevel,MEAN(area),STD(area),MEAN(angle_AP),STD(angle_AP),MEAN(angle_RL),STD(angle_RL),MEAN(diameter_AP),STD(diameter_AP),MEAN(diameter_RL),STD(diameter_RL),MEAN(eccentricity),STD(eccentricity),MEAN(orientation),STD(orientation),MEAN(solidity),STD(solidity),SUM(length)
+2021-01-25 00:37:12,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2/t2_seg.nii.gz,165:206,2:3,73.87711295363036,2.1351936742115987,0.6673174474772581,0.7973927575393721,6.786671980488889,2.2621696689009263,8.053078470024571,0.23920823068134595,11.6593364778757,0.25063568766537575,0.7216111974687509,0.02688762930825781,1.0974060397999306,0.9530528124223906,0.9738182447561956,0.010076830066330646,33.86993755180465

unit_testing/batch_processing/cached_results/t2s/csa_gm.csv

@@ -0,0 +1,5 @@
+Timestamp,SCT Version,Filename,Slice (I->S),VertLevel,MEAN(area),STD(area),MEAN(angle_AP),STD(angle_AP),MEAN(angle_RL),STD(angle_RL),MEAN(diameter_AP),STD(diameter_AP),MEAN(diameter_RL),STD(diameter_RL),MEAN(eccentricity),STD(eccentricity),MEAN(orientation),STD(orientation),MEAN(solidity),STD(solidity),SUM(length)
+2021-01-25 00:42:53,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_gmseg.nii.gz,5:10,5,17.078360250811315,0.9084472127191356,-6.755007971709035,1.2335211978417981,1.751738511637433,0.47485444480379757,4.786884200359135,0.3236340775908888,9.87050231340318,0.2519451704042565,0.8737301623216268,0.017539556764557967,9.82976774327435,2.1981212152813105,0.4487854572022603,0.0207194232303928,18.13926581795193
+2021-01-25 00:42:53,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_gmseg.nii.gz,11:16,4,16.415897543642277,1.1730361726481107,-4.005208959836534,0.3505147760960962,-0.39470896286425,0.748844177787039,5.200553168346998,0.39895175040406605,9.61956141148015,0.3654364431556718,0.8393486442952378,0.02751068978192369,6.518418631799988,4.182446319821502,0.41422091716797355,0.022780208265528044,18.04638221167759
+2021-01-25 00:42:53,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_gmseg.nii.gz,17:22,3,13.434297485782237,1.1834226156886096,-4.430807483424496,0.5849768303875148,-3.4994586118038224,1.0187670084366434,4.948753981122941,0.21242116858989743,8.195350314236558,0.42291123793398894,0.7955551981490846,0.022755695405770722,5.76531542356631,4.701322239111205,0.4269358746820224,0.018844000020263537,18.09153940993177
+2021-01-25 00:42:53,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_gmseg.nii.gz,23:27,2,12.487834828856176,0.4387725147038539,-7.560940468665102,1.152762815522508,-7.1498333648225145,1.0381903250952684,5.036671773972914,0.2797833462488226,7.925101605375119,0.192853523285372,0.7699493392381844,0.033497004831276773,12.836047628178534,4.162462819427864,0.4164871259830594,0.01988776179084211,15.256025299272746

unit_testing/batch_processing/cached_results/t2s/csa_wm.csv

@@ -0,0 +1,5 @@
+Timestamp,SCT Version,Filename,Slice (I->S),VertLevel,MEAN(area),STD(area),MEAN(angle_AP),STD(angle_AP),MEAN(angle_RL),STD(angle_RL),MEAN(diameter_AP),STD(diameter_AP),MEAN(diameter_RL),STD(diameter_RL),MEAN(eccentricity),STD(eccentricity),MEAN(orientation),STD(orientation),MEAN(solidity),STD(solidity),SUM(length)
+2021-01-25 00:42:48,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_wmseg.nii.gz,5:10,5,62.20960156107798,1.9994597782445047,-5.271669125950196,0.45806092689245265,1.5198641077368478,0.36426950182155915,8.0532945005971,0.18492872615833,14.281000236338372,0.16048063681072225,0.8254809946550461,0.011399351009829514,10.931680168141158,1.5920600641271945,0.7598215883538946,0.010829780205579547,18.083779595203534
+2021-01-25 00:42:48,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_wmseg.nii.gz,11:16,4,67.00371416619858,2.167866889330727,-4.400031031809612,0.06481672547897807,-0.3185506728631316,0.6854633529271242,8.615391054649963,0.12374018536804623,13.69862658893521,0.31307962629286223,0.7768677471953961,0.013942364524165198,4.396542163964256,2.0650417949545954,0.7822886450305937,0.018336930054822072,18.054793830863936
+2021-01-25 00:42:48,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_wmseg.nii.gz,17:22,3,63.80620897053496,1.0907016227281148,-4.988105919819013,0.37831128994988594,-3.284207421699947,1.0036276238132715,8.558311238256719,0.06498798161760432,12.663566301702842,0.2962581700996951,0.7363469449541583,0.01578189405831279,2.042613066331261,0.7838278722033172,0.8105067725797909,0.015090295670461884,18.101363288039686
+2021-01-25 00:42:48,git-HEAD-42ef57b3a6241a127ed0d7a7eebc909dc46c9d07,/home/runner/work/spinalcordtoolbox/spinalcordtoolbox/sct_example_data/t2s/t2s_wmseg.nii.gz,23:27,2,64.93830702088249,0.9618643658181062,-6.784235399804603,0.6225732536223008,-6.954192876194104,1.0620189864418308,8.86722102246955,0.040518678031165566,12.244140674946596,0.12624101563206694,0.689478722401728,0.006186592217914069,2.7667983037061328,0.9073250972148679,0.8211771625059046,0.010467210617137552,15.221387659765337

unit_testing/batch_processing/test_batch_processing.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding: utf-8
+# pytest unit tests to validate the reults of the batch_processing.sh script
+
+import os
+import pathlib
+import csv
+
+import pytest
+
+from spinalcordtoolbox.utils.sys import sct_dir_local_path
+
+
+@pytest.mark.skipif(
+    not os.getenv('BATCH_PROCESSING_CI_JOB'),  # This environment variable should be set by the CI workflow file
+    reason="Run only for batch processing CI job"
+)
+# The parametrization below checks only 6 values (one from each csv file -- same as actual batch_processing.sh)
+# TODO: We can and should be verifying more results produced by this pipeline, but which values?
+@pytest.mark.parametrize("csv_filepath,row,pos,rel_tolerance",
+                         [("t2/csa_c2c3.csv", -1, 5, 1e-14),
+                          ("t2s/csa_gm.csv", -1, 5, 1e-14),
+                          ("t2s/csa_wm.csv", -1, 5, 1e-14),
+                          ("mt/mtr_in_wm.csv", -1, 7, 1e-8),
+                          ("dmri/fa_in_cst.csv", -1, 7, 1e-8),
+                          ("dmri/fa_in_cst.csv", -2, 7, 1e-7)])
+def test_batch_processing_results(csv_filepath, row, pos, rel_tolerance):
+    """Ensure that new batch_processing.sh results are within a certain tolerance of the cached baseline results."""
+    sct_dir = pathlib.Path(sct_dir_local_path())
+    csv_filepath_old = sct_dir / "unit_testing/batch_processing/cached_results" / csv_filepath
+    csv_filepath_new = sct_dir / "sct_example_data" / csv_filepath
+    assert csv_filepath_old.is_file(), f"{csv_filepath_old} not present. Please check the SCT installation."
+    assert csv_filepath_new.is_file(), f"{csv_filepath_new} not present. Was batch_processing.sh run beforehand?"
+
+    with open(csv_filepath_old, newline='') as csvfile:
+        reader = csv.reader(csvfile, delimiter=',')
+        metric_value_old = float([row for row in reader][row][pos])  # Row/position varies depending on metric
+
+    with open(csv_filepath_new, newline='') as csvfile:
+        reader = csv.reader(csvfile, delimiter=',')
+        metric_value_new = float([row for row in reader][row][pos])  # Row/position varies depending on metric
+
+    assert metric_value_new == pytest.approx(metric_value_old, rel=rel_tolerance)