From e75aff86ee74509d53ca1690070ce9ae2f7c83f1 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 29 Apr 2023 17:55:31 +0200 Subject: [PATCH 01/22] split octave and matlab tests --- .github/workflows/tests_matlab.yml | 142 ++++++++++++++++++ .../workflows/{tests.yml => tests_octave.yml} | 40 +---- 2 files changed, 150 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/tests_matlab.yml rename .github/workflows/{tests.yml => tests_octave.yml} (74%) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml new file mode 100644 index 00000000..76fc0abf --- /dev/null +++ b/.github/workflows/tests_matlab.yml @@ -0,0 +1,142 @@ +--- +name: tests matlab + +# Uses the cron schedule for github actions +# +# https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#scheduled-events +# +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) +# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) +# │ │ │ │ │ +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * + +env: + OCTFLAGS: --no-gui --no-window-system --silent + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ['*'] + pull_request: + branches: ['*'] + schedule: + - cron: 0 0 1,15 * * + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + + test: + + if: github.repository_owner == 'cpp-lln-lab' + + strategy: + matrix: + test_type: [system] + script: [moae, facerep, fmriprep] + version: [R2022a, R2022b] + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - test_type: unit + platform: matlab + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + + - name: ${{ matrix.test_type }} test + if: matrix.test_type == 'system' + run: echo ${{ matrix.test_type }} test ${{ matrix.script }} + + - name: Install dependencies + run: | + sudo apt-get -y -qq update + sudo apt-get -y install unzip wget + + - name: Install Node + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Clone bidspm + uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + + - name: Install validators + run: make install + + - name: Install SPM + run: | + git clone https://github.com/spm/spm12.git --depth 1 + + - name: Copy Macs toolbox to SPM inputs_folder + run: cp -rv lib/MACS spm12/toolbox/MACS + + - name: Get moae fmriprep data from OSF + run: | + mkdir -p demos/MoAE/inputs/ + cd demos/MoAE/inputs/ + wget https://osf.io/vufjs/download + unzip download + mv moae_fmriprep fmriprep + + - name: Prepare test data + run: | + cd tests + make data + + - name: Install Moxunit and MOcov + run: | + git clone https://github.com/MOxUnit/MOxUnit.git --depth 1 + git clone https://github.com/MOcov/MOcov.git --depth 1 + + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1.2.4 + with: + # MATLAB release to set up R2020a + release: ${{matrix.version}} + + - name: Run unit tests MATLAB + if: matrix.test_type == 'unit' + uses: matlab-actions/run-command@v1.1.3 + with: + command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run tests_matlab; + + - name: Check unit tests + if: matrix.test_type == 'unit' + run: | + cat test_report.log | grep 0 + + - name: Code coverage matlab + if: matrix.test_type == 'unit' + uses: codecov/codecov-action@v3 + with: + file: coverage.xml + flags: matlab + name: codecov-matlab + fail_ci_if_error: false + # token: ${{ secrets.CODECOV_TOKEN }} # not required but might help API rate limits + + + - name: Run system tests MATLAB ${{ matrix.script }} + if: matrix.test_type == 'system' + uses: matlab-actions/run-command@v1.1.3 + with: + command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run system_tests_${{ matrix.script }}; diff --git a/.github/workflows/tests.yml b/.github/workflows/tests_octave.yml similarity index 74% rename from .github/workflows/tests.yml rename to .github/workflows/tests_octave.yml index c2c0bf1e..3ebd6688 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests_octave.yml @@ -1,5 +1,5 @@ --- -name: tests +name: tests octave # Uses the cron schedule for github actions # @@ -43,21 +43,17 @@ jobs: strategy: matrix: - platform: [matlab, octave] test_type: [system] script: [moae, facerep, fmriprep] include: - test_type: unit - platform: matlab - - test_type: unit - platform: octave fail-fast: false steps: - - name: ${{ matrix.platform }} - ${{ matrix.test_type }} test + - name: ${{ matrix.test_type }} test if: matrix.test_type == 'system' - run: echo ${{ matrix.platform }} ${{ matrix.test_type }} test ${{ matrix.script }} + run: echo ${{ matrix.test_type }} test ${{ matrix.script }} - name: Install dependencies run: | @@ -109,7 +105,6 @@ jobs: git clone https://github.com/MOcov/MOcov.git --depth 1 - name: Install octave - if: matrix.platform == 'octave' run: | sudo apt-get -y -qq update sudo apt-get -y install \ @@ -124,7 +119,6 @@ jobs: make -C MOcov install - name: Compile SPM - if: matrix.platform == 'octave' run: | make -C spm12/src PLATFORM=octave distclean make -C spm12/src PLATFORM=octave @@ -132,48 +126,30 @@ jobs: octave $OCTFLAGS --eval "addpath(fullfile(pwd, 'spm12')); savepath();" - name: Run unit tests Octave - if: matrix.platform == 'octave' && matrix.test_type == 'unit' + if: matrix.test_type == 'unit' run: | octave $OCTFLAGS --eval "addpath(fullfile(pwd, 'tests', 'utils')); savepath();" octave $OCTFLAGS --eval "bidspm(); cd demos/MoAE; download_moae_ds(true);" cd .github/workflows octave $OCTFLAGS --eval "tests_octave;" - - name: Install MATLAB - if: matrix.platform == 'matlab' - uses: matlab-actions/setup-matlab@v1.2.4 - with: - release: R2020a - - - name: Run unit tests MATLAB - if: matrix.platform == 'matlab' && matrix.test_type == 'unit' - uses: matlab-actions/run-command@v1.1.3 - with: - command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run tests_matlab; - - name: Check unit tests if: matrix.test_type == 'unit' run: | cat test_report.log | grep 0 - - name: Code coverage ${{ matrix.platform }} + - name: Code coverage if: matrix.test_type == 'unit' uses: codecov/codecov-action@v3 with: file: coverage.xml - flags: ${{ matrix.platform }} - name: codecov-${{ matrix.platform }} + flags: octave + name: codecov-octave fail_ci_if_error: false # token: ${{ secrets.CODECOV_TOKEN }} # not required but might help API rate limits - name: Run system tests octave ${{ matrix.script }} - if: matrix.platform == 'octave' && matrix.test_type == 'system' + if: matrix.test_type == 'system' run: | cd /home/runner/work/bidspm/bidspm/.github/workflows octave $OCTFLAGS --eval "run system_tests_${{ matrix.script }};" - - - name: Run system tests MATLAB ${{ matrix.script }} - if: matrix.platform == 'matlab' && matrix.test_type == 'system' - uses: matlab-actions/run-command@v1.1.3 - with: - command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run system_tests_${{ matrix.script }}; From e69c478315323b9988f39d9bbaaaac49ce57b8fe Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 29 Apr 2023 18:29:44 +0200 Subject: [PATCH 02/22] FCI --- .github/workflows/tests_matlab.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 76fc0abf..b2c392ac 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -15,9 +15,6 @@ name: tests matlab # │ │ │ │ │ # * * * * * -env: - OCTFLAGS: --no-gui --no-window-system --silent - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -43,11 +40,14 @@ jobs: matrix: test_type: [system] script: [moae, facerep, fmriprep] - version: [R2022a, R2022b] os: [ubuntu-latest, macos-latest, windows-latest] include: - test_type: unit - platform: matlab + os: ubuntu-latest + - test_type: unit + os: macos-latest + - test_type: unit + os: windows-latest fail-fast: false runs-on: ${{ matrix.os }} @@ -110,8 +110,7 @@ jobs: - name: Install MATLAB uses: matlab-actions/setup-matlab@v1.2.4 with: - # MATLAB release to set up R2020a - release: ${{matrix.version}} + release: R2022b - name: Run unit tests MATLAB if: matrix.test_type == 'unit' From 0cd98d2c23b4fdaac470f77ee0912de809a54fcf Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Mon, 1 May 2023 22:42:54 +0200 Subject: [PATCH 03/22] [DATALAD] Recorded changes --- .github/workflows/tests_matlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index b2c392ac..46e5180d 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -59,6 +59,7 @@ jobs: run: echo ${{ matrix.test_type }} test ${{ matrix.script }} - name: Install dependencies + if: matrix.os == ubuntu-latest run: | sudo apt-get -y -qq update sudo apt-get -y install unzip wget From 3987b46979ee9d63380ee3d7b815f0d5f334291d Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Wed, 3 May 2023 09:41:21 +0200 Subject: [PATCH 04/22] update submodules --- .github/workflows/validation.yml | 6 ------ lib/CPP_ROI | 2 +- lib/octache | 2 +- lib/spm_2_bids | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 633148a1..674cba9d 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -31,9 +31,3 @@ jobs: use-quiet-mode: yes use-verbose-mode: yes config-file: .github/workflows/mlc_config.json - - codespell: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: codespell-project/actions-codespell@master diff --git a/lib/CPP_ROI b/lib/CPP_ROI index 364a9ff1..ba75ffb4 160000 --- a/lib/CPP_ROI +++ b/lib/CPP_ROI @@ -1 +1 @@ -Subproject commit 364a9ff139b5380d6326ba8808f90bf0f98fb4b7 +Subproject commit ba75ffb42242a061bbe43c29fc5ebecaccfa9c4e diff --git a/lib/octache b/lib/octache index fe1564a0..12eca8da 160000 --- a/lib/octache +++ b/lib/octache @@ -1 +1 @@ -Subproject commit fe1564a0fc8e6a8ead09707ff436fc62c8719bdf +Subproject commit 12eca8da76633aae6b547ca399c60ed984bd3b4c diff --git a/lib/spm_2_bids b/lib/spm_2_bids index 41a863c8..a1ca105b 160000 --- a/lib/spm_2_bids +++ b/lib/spm_2_bids @@ -1 +1 @@ -Subproject commit 41a863c8a725073a43af31f31c027137ec190af4 +Subproject commit a1ca105b6fab5917a3c210374db4e0d38250fd17 From 15086fa51fd75aac6cd79ef5930ba13c57e16dfd Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Wed, 3 May 2023 09:49:51 +0200 Subject: [PATCH 05/22] skip coverage on windows --- .github/workflows/tests_matlab.yml | 13 +++++++++---- bidspm.m | 15 ++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 46e5180d..2e3f8676 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -120,16 +120,21 @@ jobs: command: cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); run tests_matlab; - name: Check unit tests - if: matrix.test_type == 'unit' + if: matrix.test_type == 'unit' && ( matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' ) + run: grep -q 0 test_report.log || { echo "Some tests failed. Check the 'Run tests' step to know which ones." >&2; exit 1; } + - name: Check logs windows + if: matrix.test_type == 'unit' && matrix.os == 'windows-latest' run: | - cat test_report.log | grep 0 + if (-not (Get-Content test_report.log | Select-String -Pattern "0")) { + throw "Some tests failed. Check the 'Run tests' step to know which ones." + } - name: Code coverage matlab - if: matrix.test_type == 'unit' + if: matrix.test_type == 'unit' && ( matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' ) uses: codecov/codecov-action@v3 with: file: coverage.xml - flags: matlab + flags: ${{ matrix.os }}_matlab-${{ matrix.version }} name: codecov-matlab fail_ci_if_error: false # token: ${{ secrets.CODECOV_TOKEN }} # not required but might help API rate limits diff --git a/bidspm.m b/bidspm.m index a7669700..6c47d138 100644 --- a/bidspm.m +++ b/bidspm.m @@ -588,11 +588,16 @@ function run_tests() folderToCover = fullfile(pwd, 'src'); testFolder = fullfile(pwd, 'tests', subfolder); - success = moxunit_runtests(testFolder, ... - '-verbose', '-recursive', '-with_coverage', ... - '-cover', folderToCover, ... - '-cover_xml_file', 'coverage.xml', ... - '-cover_html_dir', fullfile(pwd, 'coverage_html')); + if ispc + success = moxunit_runtests(testFolder, ... + '-verbose', '-recursive'); + else + success = moxunit_runtests(testFolder, ... + '-verbose', '-recursive', '-with_coverage', ... + '-cover', folderToCover, ... + '-cover_xml_file', 'coverage.xml', ... + '-cover_html_dir', fullfile(pwd, 'coverage_html')); + end if success system('echo 0 > test_report.log'); From d381ae7cf180c266a184145ff7a2b8c86794b8c2 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 4 May 2023 13:44:08 +0200 Subject: [PATCH 06/22] update CPP ROI --- lib/CPP_ROI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/CPP_ROI b/lib/CPP_ROI index ba75ffb4..86d6fac8 160000 --- a/lib/CPP_ROI +++ b/lib/CPP_ROI @@ -1 +1 @@ -Subproject commit ba75ffb42242a061bbe43c29fc5ebecaccfa9c4e +Subproject commit 86d6fac8899c63d82a99087f5bedc79360bce07c From 5cfeafcc1fae3c5505748b495716795dbc709dfd Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 11:48:49 +0200 Subject: [PATCH 07/22] misc --- lib/CPP_ROI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/CPP_ROI b/lib/CPP_ROI index 86d6fac8..b92c5337 160000 --- a/lib/CPP_ROI +++ b/lib/CPP_ROI @@ -1 +1 @@ -Subproject commit 86d6fac8899c63d82a99087f5bedc79360bce07c +Subproject commit b92c5337ba6803068688675618cbad91c0241cfe From b66fbda54753a3b34daef6b01b955f00b550318a Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 11:52:28 +0200 Subject: [PATCH 08/22] update octache and spm_2_bids --- lib/octache | 2 +- lib/spm_2_bids | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/octache b/lib/octache index 77aea2df..12eca8da 160000 --- a/lib/octache +++ b/lib/octache @@ -1 +1 @@ -Subproject commit 77aea2df3ee75e80d9272bf5a3a101a7af92b9d5 +Subproject commit 12eca8da76633aae6b547ca399c60ed984bd3b4c diff --git a/lib/spm_2_bids b/lib/spm_2_bids index d645f38f..5a9bf746 160000 --- a/lib/spm_2_bids +++ b/lib/spm_2_bids @@ -1 +1 @@ -Subproject commit d645f38f61236bb1ee7397562001f50257be1d95 +Subproject commit 5a9bf746248a264cbb486b4eb6885a94cb0df900 From 911914eb0a8729b189def7dbbe7db5d0b28996dc Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 12:01:19 +0200 Subject: [PATCH 09/22] update cpp_roi --- lib/CPP_ROI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/CPP_ROI b/lib/CPP_ROI index 1946d0d6..d7726c09 160000 --- a/lib/CPP_ROI +++ b/lib/CPP_ROI @@ -1 +1 @@ -Subproject commit 1946d0d65e1c8886efbde09d4684035e77325cfc +Subproject commit d7726c096586ad822579799c87463c9cfc17f432 From 932ecc0cf838484b25fe46e930f9bb0a7045f765 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 12:14:01 +0200 Subject: [PATCH 10/22] fix doc install --- requirements_doc.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_doc.txt b/requirements_doc.txt index 5b3709f1..72f54ba8 100644 --- a/requirements_doc.txt +++ b/requirements_doc.txt @@ -8,3 +8,4 @@ sphinx_rtd_theme sphinxcontrib-bibtex sphinxcontrib-matlabdomain sphinxcontrib-mermaid +urllib3==1.26.15 From 8eb687b483c92725501217225c908e095291603a Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 12:54:04 +0200 Subject: [PATCH 11/22] fix typo --- .github/workflows/tests_matlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 2e3f8676..7131b739 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -59,7 +59,7 @@ jobs: run: echo ${{ matrix.test_type }} test ${{ matrix.script }} - name: Install dependencies - if: matrix.os == ubuntu-latest + if: matrix.os == 'ubuntu-latest' run: | sudo apt-get -y -qq update sudo apt-get -y install unzip wget From d1398d1e8cb6c89e1214f07ad96bfa79fc8556b2 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 13:19:11 +0200 Subject: [PATCH 12/22] adapt for windows --- .github/workflows/tests_matlab.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 7131b739..3b3648b6 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -87,8 +87,12 @@ jobs: run: | git clone https://github.com/spm/spm12.git --depth 1 - - name: Copy Macs toolbox to SPM inputs_folder + - name: Copy Macs toolbox to SPM inputs_folder on unix + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: cp -rv lib/MACS spm12/toolbox/MACS + - name: Copy Macs toolbox to SPM inputs_folder on windows + if: matrix.os == 'windows-latest' + run: Copy-Item -Recurse -Verbose -Path ".\lib\MACS" -Destination ".\spm12\toolbox\MACS" - name: Get moae fmriprep data from OSF run: | From db87720429931d14e55babab3af1f0e4c43dcea6 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 13:33:01 +0200 Subject: [PATCH 13/22] adapt for windows --- .github/workflows/tests_matlab.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 3b3648b6..863fa203 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -94,13 +94,22 @@ jobs: if: matrix.os == 'windows-latest' run: Copy-Item -Recurse -Verbose -Path ".\lib\MACS" -Destination ".\spm12\toolbox\MACS" - - name: Get moae fmriprep data from OSF + - name: Get moae fmriprep data from OSF on unix + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: | mkdir -p demos/MoAE/inputs/ cd demos/MoAE/inputs/ wget https://osf.io/vufjs/download unzip download mv moae_fmriprep fmriprep + - name: Get moae fmriprep data from OSF on windows + if: matrix.os == 'windows-latest' + run: | + New-Item -ItemType Directory -Path ".\demos\MoAE\inputs\" -Force + Set-Location -Path ".\demos\MoAE\inputs\" + Invoke-WebRequest -Uri "https://osf.io/vufjs/download" -OutFile "download" + Expand-Archive -Path ".\download" -DestinationPath ".\" + Rename-Item -Path ".\moae_fmriprep" -NewName "fmriprep" - name: Prepare test data run: | From 7348cf8add8a53a17d02ba0997a56ff1bccaee73 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 5 May 2023 14:33:26 +0200 Subject: [PATCH 14/22] update bids matlab --- lib/bids-matlab | 2 +- src/QA/anatQA.m | 5 +++-- src/QA/computeRobustOutliers.m | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/bids-matlab b/lib/bids-matlab index bbd2b4f5..287109a8 160000 --- a/lib/bids-matlab +++ b/lib/bids-matlab @@ -1 +1 @@ -Subproject commit bbd2b4f53efe9952172fb49f611eecab5fb3520e +Subproject commit 287109a8981ec913de32c295c31ff2bcd063ea3c diff --git a/src/QA/anatQA.m b/src/QA/anatQA.m index 8d64319c..404e087c 100644 --- a/src/QA/anatQA.m +++ b/src/QA/anatQA.m @@ -148,10 +148,11 @@ Bmax = sqrt(sum(data(:).^2)); try % should work most of the time, possibly throwing warnings (cf Joost Kuijer) - json.EFC = real(nansum((data(:) ./ Bmax) .* log(data(:) ./ Bmax))); + tmp = (data(:) ./ Bmax) .* log(data(:) ./ Bmax); catch - json.EFC = real(nansum((data(:) ./ Bmax) .* abs(log(data(:) ./ Bmax)))); + tmp = (data(:) ./ Bmax) .* abs(log(data(:) ./ Bmax)); end + json.EFC = real(bids.internal.nansum(tmp)); end diff --git a/src/QA/computeRobustOutliers.m b/src/QA/computeRobustOutliers.m index 9754f9d4..f4d87ad7 100644 --- a/src/QA/computeRobustOutliers.m +++ b/src/QA/computeRobustOutliers.m @@ -72,6 +72,15 @@ end if strcmpi(outlierType, 's-outliers') + + if isempty(which('chi2inv')) + logger('WARNING', 'chi2inv function required to compute s-outliers.', ... + 'filename', mfilename, ... + 'id', 'missingChi2inv'); + outliers = false(size(timeSeries)); + return + end + k = sqrt(chi2inv(0.975, 1)); for p = size(timeSeries, 2):-1:1 tmp = timeSeries(:, p); From 5d00994b7768e0c305038961dc094741bb86eeb9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 08:34:39 -0400 Subject: [PATCH 15/22] fix tests and set up --- .github/workflows/tests_matlab.yml | 8 +++++- src/utils/computeTsnr.m | 15 +++++++++- tests/Makefile | 11 ++++++- tests/tests_unit/IO/test_addGitIgnore.m | 38 +++++++++---------------- tests/utils/tempName.m | 11 +++++++ 5 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 tests/utils/tempName.m diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 863fa203..4263c3c5 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -111,10 +111,16 @@ jobs: Expand-Archive -Path ".\download" -DestinationPath ".\" Rename-Item -Path ".\moae_fmriprep" -NewName "fmriprep" - - name: Prepare test data + - name: Prepare test data unix + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: | cd tests make data + - name: Prepare test data windows + if: matrix.os == 'windows-latest' + run: | + cd tests + make data_win - name: Install Moxunit and MOcov run: | diff --git a/src/utils/computeTsnr.m b/src/utils/computeTsnr.m index b761315c..e42d894a 100755 --- a/src/utils/computeTsnr.m +++ b/src/utils/computeTsnr.m @@ -88,7 +88,7 @@ % Copyright (C) Stephan Heunis 2018 % Define variables - [r, c] = size(data); + r = size(data); % Create design matrix with model regressors if order == 1 @@ -109,3 +109,16 @@ outData = data - X(:, 2:end) * b(2:end, :); end + +function y = nanmean(varargin) + + if bids.internal.is_octave() + data = varargin{1}(:); + nanIdx = isnan(data); + y = mean(data(~nanIdx)); + return + end + + narginchk(1, 2); + y = mean(varargin{:}, 'omitnan'); +end diff --git a/tests/Makefile b/tests/Makefile index 5b2e2bce..938af6a8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -9,16 +9,25 @@ clean_local: rm -rf data/derivatives/bidspm-stats/group rm -rf data/derivatives/bidspm-*/jobs -data: clean +create_dummy_dataset: sh createDummyDataSet.sh + +bids_examples: rm -rf bids-examples/ git clone https://github.com/bids-standard/bids-examples.git --depth 1 cp bids-examples/synthetic/dataset_description.json bids-examples/synthetic/derivatives/fmriprep touch bids-examples/.gitkeep + +data: clean create_dummy_dataset bids_examples wget --quiet -O moae_spm12.zip https://files.de-1.osf.io/v1/resources/3vufp/providers/osfstorage/62ab741be6f3ec09a7e48d13/?zip= mkdir -p data/MoAE/derivatives/spm12 unzip -d ./data/MoAE/derivatives/spm12 moae_spm12.zip +data_win: clean create_dummy_dataset bids_examples + Invoke-WebRequest -Uri "https://files.de-1.osf.io/v1/resources/3vufp/providers/osfstorage/62ab741be6f3ec09a7e48d13/?zip=" -OutFile "moae_spm12.zip" + New-Item -ItemType Directory -Path "data/MoAE/derivatives/spm12" -Force + Expand-Archive -Path "./moae_spm12.zip" -DestinationPath "./data/MoAE/derivatives/spm12" + clean_openneuro: rm -rf openneuro/* diff --git a/tests/tests_unit/IO/test_addGitIgnore.m b/tests/tests_unit/IO/test_addGitIgnore.m index 7744bd5f..6522f114 100644 --- a/tests/tests_unit/IO/test_addGitIgnore.m +++ b/tests/tests_unit/IO/test_addGitIgnore.m @@ -1,6 +1,5 @@ -% (C) Copyright 2020 bidspm developers - function test_suite = test_addGitIgnore %#ok<*STOUT> + % (C) Copyright 2020 bidspm developers try % assignment of 'localfunctions' is necessary in Matlab >= 2016 test_functions = localfunctions(); %#ok<*NASGU> catch % no problem; early Matlab versions can use initTestSuite fine @@ -10,44 +9,38 @@ function test_addGitIgnore_basic() - if exist(fullfile(pwd, '.gitignore'), 'file') - delete(fullfile(pwd, '.gitignore')); - end + pth = tempName(); - addGitIgnore(pwd); + addGitIgnore(pth); - assertEqual(exist(fullfile(pwd, '.gitignore'), 'file'), 2); + assertEqual(exist(fullfile(pth, '.gitignore'), 'file'), 2); - fid = fopen(fullfile(pwd, '.gitignore'), 'r'); + fid = fopen(fullfile(pth, '.gitignore'), 'r'); c = fread(fid, Inf, 'uint8=>char')'; assertEqual(strfind(c, '.DS_store'), 1); fclose(fid); - delete(fullfile(pwd, '.gitignore')); - end function test_addGitIgnore_already_present() - if exist(fullfile(pwd, '.gitignore'), 'file') - delete(fullfile(pwd, '.gitignore')); - end - fid = fopen(fullfile(pwd, '.gitignore'), 'w'); + pth = tempName(); + + fid = fopen(fullfile(pth, '.gitignore'), 'w'); fprintf(fid, 'foo\n'); fprintf(fid, '.DS_store\n'); fclose(fid); - addGitIgnore(pwd); + addGitIgnore(pth); - fid = fopen(fullfile(pwd, '.gitignore'), 'r'); + fid = fopen(fullfile(pth, '.gitignore'), 'r'); c = fread(fid, Inf, 'uint8=>char')'; assertEqual(numel(strfind(c, '.DS_store')), 1); fclose(fid); - delete(fullfile(pwd, '.gitignore')); end function test_addGitIgnore_append() @@ -56,21 +49,18 @@ function test_addGitIgnore_append() return end - if exist(fullfile(pwd, '.gitignore'), 'file') - delete(fullfile(pwd, '.gitignore')); - end + pth = tempName(); - fid = fopen(fullfile(pwd, '.gitignore'), 'w'); + fid = fopen(fullfile(pth, '.gitignore'), 'w'); fprintf(fid, 'foo\n'); fprintf(fid, 'bar\n'); fclose(fid); - addGitIgnore(pwd); + addGitIgnore(pth); - fid = fopen(fullfile(pwd, '.gitignore'), 'r'); + fid = fopen(fullfile(pth, '.gitignore'), 'r'); c = fread(fid, Inf, 'uint8=>char')'; assert(strfind(c, '.DS_store') > 0); fclose(fid); - delete(fullfile(pwd, '.gitignore')); end diff --git a/tests/utils/tempName.m b/tests/utils/tempName.m new file mode 100644 index 00000000..3b250840 --- /dev/null +++ b/tests/utils/tempName.m @@ -0,0 +1,11 @@ +function pth = tempName() + % + % Creates a temporary directory and returns its fullpath + % + + % (C) Copyright 2023 bidspm developers + + pth = tempname(); + mkdir(pth); + +end From c1d98344c9cc608118d0dfa4c887cbe8d5413477 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 08:43:33 -0400 Subject: [PATCH 16/22] update submodules --- lib/CPP_ROI | 2 +- lib/octache | 2 +- lib/spm_2_bids | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/CPP_ROI b/lib/CPP_ROI index d7726c09..3e5dbad2 160000 --- a/lib/CPP_ROI +++ b/lib/CPP_ROI @@ -1 +1 @@ -Subproject commit d7726c096586ad822579799c87463c9cfc17f432 +Subproject commit 3e5dbad29cdef05e68d6793278b8e86b3f2a2718 diff --git a/lib/octache b/lib/octache index 12eca8da..99c970e3 160000 --- a/lib/octache +++ b/lib/octache @@ -1 +1 @@ -Subproject commit 12eca8da76633aae6b547ca399c60ed984bd3b4c +Subproject commit 99c970e362392c77eef10c2e917c7d90e8ad78f8 diff --git a/lib/spm_2_bids b/lib/spm_2_bids index 5a9bf746..64edd37d 160000 --- a/lib/spm_2_bids +++ b/lib/spm_2_bids @@ -1 +1 @@ -Subproject commit 5a9bf746248a264cbb486b4eb6885a94cb0df900 +Subproject commit 64edd37de4a70ed554cc6f57d3c32e64efa97244 From f4d573e43571d67ab15b99914abe65e3238f0c17 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 08:56:24 -0400 Subject: [PATCH 17/22] change prepare data on windows --- .github/workflows/tests_matlab.yml | 6 +++++- tests/Makefile | 9 ++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests_matlab.yml b/.github/workflows/tests_matlab.yml index 4263c3c5..636fa742 100644 --- a/.github/workflows/tests_matlab.yml +++ b/.github/workflows/tests_matlab.yml @@ -120,7 +120,11 @@ jobs: if: matrix.os == 'windows-latest' run: | cd tests - make data_win + make create_dummy_dataset + make bids_examples + Invoke-WebRequest -Uri "https://files.de-1.osf.io/v1/resources/3vufp/providers/osfstorage/62ab741be6f3ec09a7e48d13/?zip=" -OutFile "moae_spm12.zip" + New-Item -ItemType Directory -Path "data/MoAE/derivatives/spm12" -Force + Expand-Archive -Path "./moae_spm12.zip" -DestinationPath "./data/MoAE/derivatives/spm12" - name: Install Moxunit and MOcov run: | diff --git a/tests/Makefile b/tests/Makefile index 938af6a8..f642a785 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -9,7 +9,7 @@ clean_local: rm -rf data/derivatives/bidspm-stats/group rm -rf data/derivatives/bidspm-*/jobs -create_dummy_dataset: +create_dummy_dataset: clean sh createDummyDataSet.sh bids_examples: @@ -18,16 +18,11 @@ bids_examples: cp bids-examples/synthetic/dataset_description.json bids-examples/synthetic/derivatives/fmriprep touch bids-examples/.gitkeep -data: clean create_dummy_dataset bids_examples +data: create_dummy_dataset bids_examples wget --quiet -O moae_spm12.zip https://files.de-1.osf.io/v1/resources/3vufp/providers/osfstorage/62ab741be6f3ec09a7e48d13/?zip= mkdir -p data/MoAE/derivatives/spm12 unzip -d ./data/MoAE/derivatives/spm12 moae_spm12.zip -data_win: clean create_dummy_dataset bids_examples - Invoke-WebRequest -Uri "https://files.de-1.osf.io/v1/resources/3vufp/providers/osfstorage/62ab741be6f3ec09a7e48d13/?zip=" -OutFile "moae_spm12.zip" - New-Item -ItemType Directory -Path "data/MoAE/derivatives/spm12" -Force - Expand-Archive -Path "./moae_spm12.zip" -DestinationPath "./data/MoAE/derivatives/spm12" - clean_openneuro: rm -rf openneuro/* From 3c4f6d94f9ba7b19aa8601443e6f03b7a16294ef Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 10:50:47 -0400 Subject: [PATCH 18/22] add more informative errors --- demos/openneuro/Makefile | 9 ++++----- src/IO/loadAndCheckOptions.m | 2 +- src/stats/subject_level/convertOnsetTsvToMat.m | 15 +++++++++++++++ src/stats/subject_level/getEventsData.m | 10 ++++++++++ src/utils/computeTsnr.m | 2 +- tests/tests_unit/IO/test_saveSpmScript.m | 3 +++ 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/demos/openneuro/Makefile b/demos/openneuro/Makefile index 4573cdf6..cb4417bd 100644 --- a/demos/openneuro/Makefile +++ b/demos/openneuro/Makefile @@ -43,8 +43,7 @@ data_ds001734: data_ds002799: mkdir -p inputs cd inputs && datalad install ///openneuro/ds002799 - cd inputs/ds002799/derivatives/fmriprep - datalad get sub-292/*/func/*MNI152NLin2009cAsym* - datalad get sub-292/*/func/*tsv - datalad get sub-30[27]/*/func/*MNI152NLin2009cAsym* - datalad get sub-30[27]/*/func/*tsv + datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-292/*/func/*MNI152NLin2009cAsym* + datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-292/*/func/*tsv + datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-30[27]/*/func/*MNI152NLin2009cAsym* + datalad get -d inputs/ds002799 inputs/ds002799/derivatives/fmriprep/sub-30[27]/*/func/*tsv diff --git a/src/IO/loadAndCheckOptions.m b/src/IO/loadAndCheckOptions.m index 628db9a8..4b62c5d7 100644 --- a/src/IO/loadAndCheckOptions.m +++ b/src/IO/loadAndCheckOptions.m @@ -42,7 +42,7 @@ % finds most recent option file if size(optionJsonFile, 1) > 1 containsDate = cellfun(@any, regexp(cellstr(optionJsonFile), ... - '^.*/[0-9]{4}-[0-9]{2}-[0-9]{2}T.*\.json')); + '^.*[0-9]{4}-[0-9]{2}-[0-9]{2}T.*\.json')); if any(containsDate) optionJsonFile = optionJsonFile(containsDate, :); optionJsonFile = sortrows(optionJsonFile); diff --git a/src/stats/subject_level/convertOnsetTsvToMat.m b/src/stats/subject_level/convertOnsetTsvToMat.m index a0dc5df5..774a7c81 100644 --- a/src/stats/subject_level/convertOnsetTsvToMat.m +++ b/src/stats/subject_level/convertOnsetTsvToMat.m @@ -51,10 +51,25 @@ % (C) Copyright 2019 bidspm developers + REQUIRED_COLUMNS = {'onset', 'duration'}; + [pth, file, ext] = spm_fileparts(tsvFile); tsv.file = validationInputFile(pth, [file, ext]); tsv.content = bids.util.tsvread(tsv.file); + for i = 1:numel(REQUIRED_COLUMNS) + if ~isfield(content, REQUIRED_COLUMNS{i}) + msg = sprintf(['''%s'' column is missing from file: %s', ... + '\nAvailable columns are: %s'], ... + REQUIRED_COLUMNS{i}, ... + tsv.file, ... + bids.internal.create_unordered_list(fieldnames(tsv.content))); + logger('ERROR', msg, ... + 'filename', filename, ... + 'id', 'missingTrialypeColumn'); + end + end + if ~all(isnumeric(tsv.content.onset)) msg = sprintf('%s\n%s', 'Onset column contains non numeric values in file:', tsv.file); diff --git a/src/stats/subject_level/getEventsData.m b/src/stats/subject_level/getEventsData.m index d32fd03a..cebd1693 100644 --- a/src/stats/subject_level/getEventsData.m +++ b/src/stats/subject_level/getEventsData.m @@ -11,6 +11,16 @@ content = bids.util.tsvread(tsvFile); + if ~isfield(content, 'trial_type') + msg = sprintf(['''trial_type'' column is missing from file: %s', ... + '\nAvailable columns are: %s'], ... + tsvFile, ... + bids.internal.create_unordered_list(fieldnames(content))); + logger('ERROR', msg, ... + 'filename', filename, ... + 'id', 'missingTrialypeColumn'); + end + conditions = unique(content.trial_type); if ~isempty(modelFile) diff --git a/src/utils/computeTsnr.m b/src/utils/computeTsnr.m index e42d894a..022a75b7 100755 --- a/src/utils/computeTsnr.m +++ b/src/utils/computeTsnr.m @@ -97,7 +97,7 @@ X = [(1:r)' ((1:r)').^2]; end % Remove mean from design matrix - X = X - repmat(mean(X, 1), r, 1); + X = X - repmat(mean(X, 1), [r, 1]); % Add constant regressor X = [ones(r, 1) X]; diff --git a/tests/tests_unit/IO/test_saveSpmScript.m b/tests/tests_unit/IO/test_saveSpmScript.m index 046386d2..bde5470b 100644 --- a/tests/tests_unit/IO/test_saveSpmScript.m +++ b/tests/tests_unit/IO/test_saveSpmScript.m @@ -30,6 +30,9 @@ function test_saveSpmScript_basic() fid1 = fopen(expectedContent, 'r'); expectedContent = fscanf(fid1, '%s'); fclose(fid1); + if ispc + expectedContent = strrep('/', '\'); + end fid2 = fopen(outputFilename, 'r'); actualContent = fscanf(fid2, '%s'); From 99d74b4cb3ce7639b819ed64ffece8628092d46a Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 11:17:21 -0400 Subject: [PATCH 19/22] use temp dir --- demos/MoAE/test_moae.m | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/demos/MoAE/test_moae.m b/demos/MoAE/test_moae.m index 54b22c36..a8b0bc53 100644 --- a/demos/MoAE/test_moae.m +++ b/demos/MoAE/test_moae.m @@ -51,7 +51,8 @@ %% preproc bids_dir = fullfile(WD, 'inputs', 'raw'); - output_dir = fullfile(WD, 'outputs', 'derivatives'); + output_dir = fullfile(tempname, 'outputs', 'derivatives'); + mkdir(output_dir); bidspm(bids_dir, output_dir, 'subject', ... 'participant_label', {'01'}, ... @@ -83,8 +84,6 @@ cd(WD); - rmdir(fullfile(WD, 'outputs', 'derivatives'), 's'); - % with Octave running more n-1 loop in CI is fine % but not running crashes with a segmentation fault % /home/runner/work/_temp/fb8e9d58-fa9f-4f93-8c96-387973f3632e.sh: line 2: From e1dd1451539cef744affd8aa7afdd866866350f2 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 11:23:49 -0400 Subject: [PATCH 20/22] fix typo --- src/stats/subject_level/convertOnsetTsvToMat.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stats/subject_level/convertOnsetTsvToMat.m b/src/stats/subject_level/convertOnsetTsvToMat.m index 774a7c81..a6e184f5 100644 --- a/src/stats/subject_level/convertOnsetTsvToMat.m +++ b/src/stats/subject_level/convertOnsetTsvToMat.m @@ -58,7 +58,7 @@ tsv.content = bids.util.tsvread(tsv.file); for i = 1:numel(REQUIRED_COLUMNS) - if ~isfield(content, REQUIRED_COLUMNS{i}) + if ~isfield(tsv.content, REQUIRED_COLUMNS{i}) msg = sprintf(['''%s'' column is missing from file: %s', ... '\nAvailable columns are: %s'], ... REQUIRED_COLUMNS{i}, ... From e7f043111fca3c94ca7150742a17aaa03b1c2e5b Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 May 2023 12:44:06 -0400 Subject: [PATCH 21/22] fix typo --- src/stats/subject_level/convertOnsetTsvToMat.m | 2 +- src/stats/subject_level/getEventsData.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stats/subject_level/convertOnsetTsvToMat.m b/src/stats/subject_level/convertOnsetTsvToMat.m index a6e184f5..3d3cd555 100644 --- a/src/stats/subject_level/convertOnsetTsvToMat.m +++ b/src/stats/subject_level/convertOnsetTsvToMat.m @@ -65,7 +65,7 @@ tsv.file, ... bids.internal.create_unordered_list(fieldnames(tsv.content))); logger('ERROR', msg, ... - 'filename', filename, ... + 'filename', mfilename(), ... 'id', 'missingTrialypeColumn'); end end diff --git a/src/stats/subject_level/getEventsData.m b/src/stats/subject_level/getEventsData.m index cebd1693..fb542cc3 100644 --- a/src/stats/subject_level/getEventsData.m +++ b/src/stats/subject_level/getEventsData.m @@ -17,7 +17,7 @@ tsvFile, ... bids.internal.create_unordered_list(fieldnames(content))); logger('ERROR', msg, ... - 'filename', filename, ... + 'filename', mfilename(), ... 'id', 'missingTrialypeColumn'); end From 1d04e358dd9f575615989b5346b03f270cb2b19c Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 7 May 2023 08:58:28 -0400 Subject: [PATCH 22/22] fix tests --- src/utils/computeTsnr.m | 5 +++-- tests/tests_unit/IO/test_addGitIgnore.m | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/utils/computeTsnr.m b/src/utils/computeTsnr.m index 022a75b7..4b6a2db8 100755 --- a/src/utils/computeTsnr.m +++ b/src/utils/computeTsnr.m @@ -88,7 +88,7 @@ % Copyright (C) Stephan Heunis 2018 % Define variables - r = size(data); + [r, c] = size(data); % Create design matrix with model regressors if order == 1 @@ -97,7 +97,7 @@ X = [(1:r)' ((1:r)').^2]; end % Remove mean from design matrix - X = X - repmat(mean(X, 1), [r, 1]); + X = X - repmat(mean(X, 1), r, 1); % Add constant regressor X = [ones(r, 1) X]; @@ -121,4 +121,5 @@ narginchk(1, 2); y = mean(varargin{:}, 'omitnan'); + end diff --git a/tests/tests_unit/IO/test_addGitIgnore.m b/tests/tests_unit/IO/test_addGitIgnore.m index 6522f114..46af5b5b 100644 --- a/tests/tests_unit/IO/test_addGitIgnore.m +++ b/tests/tests_unit/IO/test_addGitIgnore.m @@ -49,6 +49,10 @@ function test_addGitIgnore_append() return end + if ispc || ismac + return + end + pth = tempName(); fid = fopen(fullfile(pth, '.gitignore'), 'w');