[INFRA] run matlab CI on windows and macOS

.github/workflows/tests_matlab.yml

@@ -0,0 +1,170 @@
+---
+name: tests matlab
+
+# Uses the cron schedule for github actions
+#
+# https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#scheduled-events
+#
+# ┌───────────── minute (0 - 59)
+# │ ┌───────────── hour (0 - 23)
+# │ │ ┌───────────── day of the month (1 - 31)
+# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
+# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
+# │ │ │ │ │
+# │ │ │ │ │
+# │ │ │ │ │
+# * * * * *
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: ['*']
+  pull_request:
+    branches: ['*']
+  schedule:
+  - cron: 0 0 1,15 * *
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+
+  test:
+
+    if: github.repository_owner == 'cpp-lln-lab'
+
+    strategy:
+      matrix:
+        test_type: [system]
+        script: [moae, facerep, fmriprep]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+        - test_type: unit
+          os: ubuntu-latest
+        - test_type: unit
+          os: macos-latest
+        - test_type: unit
+          os: windows-latest
+      fail-fast: false
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+
+    - name: ${{ matrix.test_type }} test
+      if: matrix.test_type == 'system'
+      run: echo ${{ matrix.test_type }} test ${{ matrix.script }}
+
+    - name: Install dependencies
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get -y -qq update
+        sudo apt-get -y install unzip wget
+
+    - name: Install Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: 18
+
+    - name: Install python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Clone bidspm
+      uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    - name: Install validators
+      run: make install
+
+    - name: Install SPM
+      run: |
+        git clone https://github.com/spm/spm12.git --depth 1
+
+    - name: Copy Macs toolbox to SPM inputs_folder on unix
+      if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
+      run: cp -rv lib/MACS spm12/toolbox/MACS
+    - name: Copy Macs toolbox to SPM inputs_folder on windows
+      if: matrix.os == 'windows-latest'
+      run: Copy-Item -Recurse -Verbose -Path ".\lib\MACS" -Destination ".\spm12\toolbox\MACS"
+
+    - name: Get moae fmriprep data from OSF on unix
+      if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
+      run: |
+        mkdir -p demos/MoAE/inputs/
+        cd demos/MoAE/inputs/
+        wget https://osf.io/vufjs/download
+        unzip download
+        mv moae_fmriprep fmriprep
+    - name: Get moae fmriprep data from OSF on windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        New-Item -ItemType Directory -Path ".\demos\MoAE\inputs\" -Force
+        Set-Location -Path ".\demos\MoAE\inputs\"
+        Invoke-WebRequest -Uri "https://osf.io/vufjs/download" -OutFile "download"
+        Expand-Archive -Path ".\download" -DestinationPath ".\"
+        Rename-Item -Path ".\moae_fmriprep" -NewName "fmriprep"
+
+    - name: Prepare test data unix
+      if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
+      run: |
+        cd tests
+        make data
+    - name: Prepare test data windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        cd tests
+        make create_dummy_dataset
+        make bids_examples
+        Invoke-WebRequest -Uri "https://files.de-1.osf.io/v1/resources/3vufp/providers/osfstorage/62ab741be6f3ec09a7e48d13/?zip=" -OutFile "moae_spm12.zip"
+        New-Item -ItemType Directory -Path "data/MoAE/derivatives/spm12" -Force
+        Expand-Archive -Path "./moae_spm12.zip" -DestinationPath "./data/MoAE/derivatives/spm12"
+
+    - name: Install Moxunit and MOcov
+      run: |
+        git clone https://github.com/MOxUnit/MOxUnit.git --depth 1
+        git clone https://github.com/MOcov/MOcov.git --depth 1
+
+    - name: Install MATLAB
+      uses: matlab-actions/setup-matlab@v1.2.4
+      with:
+        release: R2022b
+
+    - name: Run unit tests MATLAB
+      if: matrix.test_type == 'unit'
+      uses: matlab-actions/run-command@v1.1.3
+      with:
+        command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run tests_matlab;
+
+    - name: Check unit tests
+      if: matrix.test_type == 'unit' && ( matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' )
+      run: grep -q 0 test_report.log || { echo "Some tests failed. Check the 'Run tests' step to know which ones." >&2; exit 1; }
+    - name: Check logs windows
+      if: matrix.test_type == 'unit' && matrix.os == 'windows-latest'
+      run: |
+        if (-not (Get-Content test_report.log | Select-String -Pattern "0")) {
+          throw "Some tests failed.  Check the 'Run tests' step to know which ones."
+        }
+
+    - name: Code coverage matlab
+      if: matrix.test_type == 'unit' && ( matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' )
+      uses: codecov/codecov-action@v3
+      with:
+        file: coverage.xml
+        flags: ${{ matrix.os }}_matlab-${{ matrix.version }}
+        name: codecov-matlab
+        fail_ci_if_error: false
+      # token: ${{ secrets.CODECOV_TOKEN }}  # not required but might help API rate limits
+
+
+    - name: Run system tests MATLAB ${{ matrix.script }}
+      if: matrix.test_type == 'system'
+      uses: matlab-actions/run-command@v1.1.3
+      with:
+        command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run system_tests_${{ matrix.script }};

.github/workflows/tests.yml → .github/workflows/tests_octave.yml

@@ -1,5 +1,5 @@
 ---
-name: tests
+name: tests octave
 
 # Uses the cron schedule for github actions
 #
@@ -43,21 +43,17 @@ jobs:
 
     strategy:
       matrix:
-        platform: [matlab, octave]
         test_type: [system]
         script: [moae, facerep, fmriprep]
         include:
         - test_type: unit
-          platform: matlab
-        - test_type: unit
-          platform: octave
       fail-fast: false
 
     steps:
 
-    - name: ${{ matrix.platform }} - ${{ matrix.test_type }} test
+    - name: ${{ matrix.test_type }} test
       if: matrix.test_type == 'system'
-      run: echo ${{ matrix.platform }} ${{ matrix.test_type }} test ${{ matrix.script }}
+      run: echo ${{ matrix.test_type }} test ${{ matrix.script }}
 
     - name: Install dependencies
       run: |
@@ -109,7 +105,6 @@ jobs:
         git clone https://github.com/MOcov/MOcov.git --depth 1
 
     - name: Install octave
-      if: matrix.platform == 'octave'
       run: |
         sudo apt-get -y -qq update
         sudo apt-get -y install \
@@ -124,56 +119,37 @@ jobs:
         make -C MOcov install
 
     - name: Compile SPM
-      if: matrix.platform == 'octave'
       run: |
         make -C spm12/src PLATFORM=octave distclean
         make -C spm12/src PLATFORM=octave
         make -C spm12/src PLATFORM=octave install
         octave $OCTFLAGS --eval "addpath(fullfile(pwd, 'spm12')); savepath();"
 
     - name: Run unit tests Octave
-      if: matrix.platform == 'octave' && matrix.test_type == 'unit'
+      if: matrix.test_type == 'unit'
       run: |
         octave $OCTFLAGS --eval "addpath(fullfile(pwd, 'tests', 'utils')); savepath();"
         octave $OCTFLAGS --eval "bidspm(); cd demos/MoAE; download_moae_ds(true);"
         cd .github/workflows
         octave $OCTFLAGS --eval "tests_octave;"
 
-    - name: Install MATLAB
-      if: matrix.platform == 'matlab'
-      uses: matlab-actions/setup-matlab@v1.2.4
-      with:
-        release: R2020a
-
-    - name: Run unit tests MATLAB
-      if: matrix.platform == 'matlab' && matrix.test_type == 'unit'
-      uses: matlab-actions/run-command@v1.1.3
-      with:
-        command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run tests_matlab;
-
     - name: Check unit tests
       if: matrix.test_type == 'unit'
       run: |
         cat test_report.log | grep 0
 
-    - name: Code coverage ${{ matrix.platform }}
+    - name: Code coverage
       if: matrix.test_type == 'unit'
       uses: codecov/codecov-action@v3
       with:
         file: coverage.xml
-        flags: ${{ matrix.platform }}
-        name: codecov-${{ matrix.platform }}
+        flags: octave
+        name: codecov-octave
         fail_ci_if_error: false
       # token: ${{ secrets.CODECOV_TOKEN }}  # not required but might help API rate limits
 
     - name: Run system tests octave ${{ matrix.script }}
-      if: matrix.platform == 'octave' && matrix.test_type == 'system'
+      if: matrix.test_type == 'system'
       run: |
         cd /home/runner/work/bidspm/bidspm/.github/workflows
         octave $OCTFLAGS --eval "run system_tests_${{ matrix.script }};"
-
-    - name: Run system tests MATLAB ${{ matrix.script }}
-      if: matrix.platform == 'matlab' && matrix.test_type == 'system'
-      uses: matlab-actions/run-command@v1.1.3
-      with:
-        command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run system_tests_${{ matrix.script }};

.github/workflows/validation.yml

@@ -31,9 +31,3 @@ jobs:
         use-quiet-mode: yes
         use-verbose-mode: yes
         config-file: .github/workflows/mlc_config.json
-
-  codespell:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: codespell-project/actions-codespell@master

bidspm.m

@@ -586,11 +586,16 @@ function run_tests()
   folderToCover = fullfile(pwd, 'src');
   testFolder = fullfile(pwd, 'tests', subfolder);
 
-  success = moxunit_runtests(testFolder, ...
-                             '-verbose', '-recursive', '-with_coverage', ...
-                             '-cover', folderToCover, ...
-                             '-cover_xml_file', 'coverage.xml', ...
-                             '-cover_html_dir', fullfile(pwd, 'coverage_html'));
+  if ispc
+    success = moxunit_runtests(testFolder, ...
+                               '-verbose', '-recursive');
+  else
+    success = moxunit_runtests(testFolder, ...
+                               '-verbose', '-recursive', '-with_coverage', ...
+                               '-cover', folderToCover, ...
+                               '-cover_xml_file', 'coverage.xml', ...
+                               '-cover_html_dir', fullfile(pwd, 'coverage_html'));
+  end
 
   if success
     system('echo 0 > test_report.log');

demos/MoAE/test_moae.m

@@ -51,7 +51,8 @@
 
   %% preproc
   bids_dir = fullfile(WD, 'inputs', 'raw');
-  output_dir = fullfile(WD, 'outputs', 'derivatives');
+  output_dir = fullfile(tempname, 'outputs', 'derivatives');
+  mkdir(output_dir);
 
   bidspm(bids_dir, output_dir, 'subject', ...
          'participant_label', {'01'}, ...
@@ -83,8 +84,6 @@
 
   cd(WD);
 
-  rmdir(fullfile(WD, 'outputs', 'derivatives'), 's');
-
   % with Octave running more n-1 loop in CI is fine
   % but not running crashes with a segmentation fault
   % /home/runner/work/_temp/fb8e9d58-fa9f-4f93-8c96-387973f3632e.sh: line 2:

demos/openneuro/Makefile

@@ -43,8 +43,7 @@ data_ds001734:
 data_ds002799:
 	mkdir -p inputs
 	cd inputs && datalad install ///openneuro/ds002799
-	cd inputs/ds002799/derivatives/fmriprep
-	datalad get sub-292/*/func/*MNI152NLin2009cAsym*
-	datalad get sub-292/*/func/*tsv
-	datalad get sub-30[27]/*/func/*MNI152NLin2009cAsym*
-	datalad get sub-30[27]/*/func/*tsv
+	datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-292/*/func/*MNI152NLin2009cAsym*
+	datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-292/*/func/*tsv
+	datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-30[27]/*/func/*MNI152NLin2009cAsym*
+	datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-30[27]/*/func/*tsv

requirements_doc.txt

@@ -8,3 +8,4 @@ sphinx_rtd_theme
 sphinxcontrib-bibtex
 sphinxcontrib-matlabdomain
 sphinxcontrib-mermaid
+urllib3==1.26.15

src/IO/loadAndCheckOptions.m

@@ -42,7 +42,7 @@
   % finds most recent option file
   if size(optionJsonFile, 1) > 1
     containsDate = cellfun(@any, regexp(cellstr(optionJsonFile), ...
-                                        '^.*/[0-9]{4}-[0-9]{2}-[0-9]{2}T.*\.json'));
+                                        '^.*[0-9]{4}-[0-9]{2}-[0-9]{2}T.*\.json'));
     if any(containsDate)
       optionJsonFile = optionJsonFile(containsDate, :);
       optionJsonFile = sortrows(optionJsonFile);

src/QA/anatQA.m

@@ -148,10 +148,11 @@
   Bmax = sqrt(sum(data(:).^2));
   try
     % should work most of the time, possibly throwing warnings (cf Joost Kuijer)
-    json.EFC = real(nansum((data(:) ./ Bmax) .* log(data(:) ./ Bmax)));
+    tmp = (data(:) ./ Bmax) .* log(data(:) ./ Bmax);
   catch
-    json.EFC = real(nansum((data(:) ./ Bmax) .* abs(log(data(:) ./ Bmax))));
+    tmp = (data(:) ./ Bmax) .* abs(log(data(:) ./ Bmax));
   end
+  json.EFC = real(bids.internal.nansum(tmp));
 
 end
 

src/QA/computeRobustOutliers.m

@@ -72,6 +72,15 @@
   end
 
   if strcmpi(outlierType, 's-outliers')
+
+    if isempty(which('chi2inv'))
+      logger('WARNING', 'chi2inv function required to compute s-outliers.', ...
+             'filename', mfilename, ...
+             'id', 'missingChi2inv');
+      outliers = false(size(timeSeries));
+      return
+    end
+
     k = sqrt(chi2inv(0.975, 1));
     for p = size(timeSeries, 2):-1:1
       tmp = timeSeries(:, p);