From bf859455b471199caebabce151610cbc4756e85f Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Thu, 12 Nov 2020 00:36:52 +0100 Subject: [PATCH 01/14] add main script for gzipping raw folder --- src/workflows/bidsGZipRawFolder.m | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/workflows/bidsGZipRawFolder.m diff --git a/src/workflows/bidsGZipRawFolder.m b/src/workflows/bidsGZipRawFolder.m new file mode 100644 index 00000000..b312fab4 --- /dev/null +++ b/src/workflows/bidsGZipRawFolder.m @@ -0,0 +1,49 @@ +% (C) Copyright 2020 CPP BIDS SPM-pipeline developers + +function bidsGZipRawFolder(opt, deleteUnzippedNii, modalitiesToZip) + + % + % GZip the nii files in a ``raw`` bids folders from the. It will do it independently of the task. + % + % USAGE:: + % + % bidsGZipRawFolder([opt,] ... + % [deleteUnzippedNii = true,] ... + % [modalitiesToZip = {'anat', 'func', 'fmap'}]) + % + % :param opt: structure or json filename containing the options. See + % ``checkOptions()`` and ``loadAndCheckOptions()``. + % :type opt: structure + % :param deleteUnzippedNii: will delete the original ``.nii`` if set to ``true`` + % :type deleteUnzippedNii: boolean + % :param modalitiesToZip: + % :type modalitiesToZip: cell + + %% input variables default values + + if nargin < 3 || isempty(modalitiesToZip) + % Will only copy those modalities if they exist + modalitiesToZip = {'anat', 'func', 'fmap'}; + end + + if nargin < 2 || isempty(deleteUnzippedNii) + % delete the original zipped nii.gz + deleteUnzippedNii = true; + end + + % if input has no opt, load the opt.mat file + if nargin < 1 || isempty(opt) + opt = []; + end + opt = loadAndCheckOptions(opt); + + cleanCrash(); + + printWorklowName('GZip data'); + + rawDir = opt.dataDir; + + unzippedNiifiles = spm_select('FPListRec', rawDir, '^.*.nii$'); + + + From ac887bd64637674856a308f28ef7a24b17adb57b Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Thu, 12 Nov 2020 15:14:37 +0100 Subject: [PATCH 02/14] update workflow script to gzip --- src/workflows/bidsGZipRawFolder.m | 45 +++++++++++++------------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/src/workflows/bidsGZipRawFolder.m b/src/workflows/bidsGZipRawFolder.m index b312fab4..c53e1dea 100644 --- a/src/workflows/bidsGZipRawFolder.m +++ b/src/workflows/bidsGZipRawFolder.m @@ -1,6 +1,6 @@ % (C) Copyright 2020 CPP BIDS SPM-pipeline developers -function bidsGZipRawFolder(opt, deleteUnzippedNii, modalitiesToZip) +function bidsGZipRawFolder(optSource, keepUnzippedNii) % % GZip the nii files in a ``raw`` bids folders from the. It will do it independently of the task. @@ -11,39 +11,32 @@ function bidsGZipRawFolder(opt, deleteUnzippedNii, modalitiesToZip) % [deleteUnzippedNii = true,] ... % [modalitiesToZip = {'anat', 'func', 'fmap'}]) % - % :param opt: structure or json filename containing the options. See - % ``checkOptions()`` and ``loadAndCheckOptions()``. + % :param optSource: The structure that contains the options set by the user to run the batch + % workflow for source processing % :type opt: structure - % :param deleteUnzippedNii: will delete the original ``.nii`` if set to ``true`` - % :type deleteUnzippedNii: boolean - % :param modalitiesToZip: - % :type modalitiesToZip: cell - - %% input variables default values + % :param keepUnzippedNii: will keep the original ``.nii`` if set to ``true``. Default is false + % :type keepUnzippedNii: boolean - if nargin < 3 || isempty(modalitiesToZip) - % Will only copy those modalities if they exist - modalitiesToZip = {'anat', 'func', 'fmap'}; - end + %% input variables default values + + optSource.dataDir = '/Users/barilari/Desktop/data_temp/raw_temp'; if nargin < 2 || isempty(deleteUnzippedNii) - % delete the original zipped nii.gz - deleteUnzippedNii = true; - end - - % if input has no opt, load the opt.mat file - if nargin < 1 || isempty(opt) - opt = []; + % delete the original unzipped .nii + keepUnzippedNii = false; end - opt = loadAndCheckOptions(opt); - - cleanCrash(); + tic; + printWorklowName('GZip data'); - rawDir = opt.dataDir; + rawDir = optSource.dataDir; - unzippedNiifiles = spm_select('FPListRec', rawDir, '^.*.nii$'); - + unzippedNiifiles = cellstr(spm_select('FPListRec', rawDir, '^.*.nii$')); + matlabbatch = setBatchGZip(unzippedNiifiles, keepUnzippedNii); + + spm_jobman('run', matlabbatch); + toc; + From 66b32b20a758cc39dad167611dd45855072b234c Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Thu, 12 Nov 2020 15:15:22 +0100 Subject: [PATCH 03/14] add spm batch for gzipping --- src/batches/setBatchGZip.m | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/batches/setBatchGZip.m diff --git a/src/batches/setBatchGZip.m b/src/batches/setBatchGZip.m new file mode 100644 index 00000000..2847db26 --- /dev/null +++ b/src/batches/setBatchGZip.m @@ -0,0 +1,22 @@ +% (C) Copyright 2020 CPP BIDS SPM-pipeline developers + +function matlabbatch = setBatchGZip(unzippedNiifiles, keepUnzippedNii) + % + % Set the batch for GZip the 4D volumes + % + % USAGE:: + % + % matlabbatch = setGZip(unzippedNiifiles, deleteUnzippedNii) + % + % :param unzippedNiifiles: List of volumes to be gzipped + % :type unzippedNiifiles: array + % :param keepUnzippedNii: Obligatory argument. Boolean to decide to delete the unzipped files + % :type keepUnzippedNii: boolean + % + % :returns: - :matlabbatch: (struct) The matlabbath ready to run the spm job + + matlabbatch{1}.cfg_basicio.file_dir.file_ops.cfg_gzip_files.files = unzippedNiifiles; + matlabbatch{1}.cfg_basicio.file_dir.file_ops.cfg_gzip_files.outdir = {''}; + matlabbatch{1}.cfg_basicio.file_dir.file_ops.cfg_gzip_files.keep = keepUnzippedNii; + +end From b1c430dae9ce6834cdbfdbfdd4f5086fff28c63b Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Thu, 12 Nov 2020 15:15:52 +0100 Subject: [PATCH 04/14] update the example workflow --- demos/sourceDataProcessing/batchSource.m | 6 ++++++ demos/sourceDataProcessing/getOptionSource.m | 2 ++ src/defaults/checkOptionsSource.m | 11 ++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/demos/sourceDataProcessing/batchSource.m b/demos/sourceDataProcessing/batchSource.m index 8b4f3b88..0be72c84 100644 --- a/demos/sourceDataProcessing/batchSource.m +++ b/demos/sourceDataProcessing/batchSource.m @@ -16,3 +16,9 @@ % Single volumes to 4D volumes conversion + remove n dummies convert3Dto4D(optSource); + +% Deface anatomical volumes in a raw folder +% defaceAnat(optSource); COMING SOON + +% GZip the volumes in a raw folder +bidsGZipRawFolder(optSource, 0) \ No newline at end of file diff --git a/demos/sourceDataProcessing/getOptionSource.m b/demos/sourceDataProcessing/getOptionSource.m index 84555cab..0b064a65 100644 --- a/demos/sourceDataProcessing/getOptionSource.m +++ b/demos/sourceDataProcessing/getOptionSource.m @@ -17,6 +17,8 @@ % Set the folder where sequences folders exist optSource.sourceDir = '/Users/barilari/Desktop/DICOM_UCL_leuven/renamed/sub-pilot001/ses-002/MRI'; + + optSource.dataDir = '/Users/barilari/Desktop/DICOM_UCL_leuven/raw'; % List of the sequences that you want to skip (folder name pattern) optSource.sequenceToIgnore = {'AAHead_Scout', ... diff --git a/src/defaults/checkOptionsSource.m b/src/defaults/checkOptionsSource.m index f6891759..23483280 100644 --- a/src/defaults/checkOptionsSource.m +++ b/src/defaults/checkOptionsSource.m @@ -22,10 +22,16 @@ error('The source folder does not exist, try again.'); end + + if isempty(optSource.dataDir) || ~isdir(optSource.dataDir) + + error('The raw folder does not exist, try again.'); + + end if isempty(optSource.sequenceToIgnore) - warning('No sequence to ignore provided, I will convert all the images that I can found'); + warning('No sequence-to-ignore provided, I will convert all the images that I can found'); end @@ -36,6 +42,9 @@ % The directory where the source data are located fieldsToSet.sourceDir = ''; + + % The directory where the raw data to apply changes are located + fieldsToSet.dataDir = ''; % The list of sequence(s) to ignore fieldsToSet.sequenceToIgnore = {}; From 04c95ac9af801f5d84c28705a0e0dfce607087de Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Thu, 12 Nov 2020 15:16:29 +0100 Subject: [PATCH 05/14] fix doc error in setBatch3Dto4D --- src/batches/setBatch3Dto4D.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/batches/setBatch3Dto4D.m b/src/batches/setBatch3Dto4D.m index 1756091d..69693595 100644 --- a/src/batches/setBatch3Dto4D.m +++ b/src/batches/setBatch3Dto4D.m @@ -4,7 +4,7 @@ % % Set the batch for 3D to 4D conversion % - % USAGE:: matlabbatch = setBatch3Dto4D(volumesList, outputName, dataType, RT) + % USAGE:: % % matlabbatch = setBatch3Dto4D(volumesList, outputName, dataType, RT) % From 19fa2d72c360d357081d277162963b09588ab746 Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Thu, 12 Nov 2020 15:26:00 +0100 Subject: [PATCH 06/14] change errors to warnings --- src/defaults/checkOptionsSource.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/defaults/checkOptionsSource.m b/src/defaults/checkOptionsSource.m index 23483280..cfc19a4d 100644 --- a/src/defaults/checkOptionsSource.m +++ b/src/defaults/checkOptionsSource.m @@ -19,13 +19,13 @@ if isempty(optSource.sourceDir) || ~isdir(optSource.sourceDir) - error('The source folder does not exist, try again.'); + warning('The source folder is not provided or does not exist.'); end if isempty(optSource.dataDir) || ~isdir(optSource.dataDir) - error('The raw folder does not exist, try again.'); + warning('The raw folder is not provided or does not exist.'); end From 1036a480cf1e315f25fc7b8d2cfdfb8abcb614fe Mon Sep 17 00:00:00 2001 From: marcobarilari <38101692+marcobarilari@users.noreply.github.com> Date: Thu, 12 Nov 2020 16:35:14 +0100 Subject: [PATCH 07/14] Update src/workflows/bidsGZipRawFolder.m --- src/workflows/bidsGZipRawFolder.m | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/workflows/bidsGZipRawFolder.m b/src/workflows/bidsGZipRawFolder.m index c53e1dea..15b37740 100644 --- a/src/workflows/bidsGZipRawFolder.m +++ b/src/workflows/bidsGZipRawFolder.m @@ -19,7 +19,6 @@ function bidsGZipRawFolder(optSource, keepUnzippedNii) %% input variables default values - optSource.dataDir = '/Users/barilari/Desktop/data_temp/raw_temp'; if nargin < 2 || isempty(deleteUnzippedNii) % delete the original unzipped .nii @@ -39,4 +38,3 @@ function bidsGZipRawFolder(optSource, keepUnzippedNii) spm_jobman('run', matlabbatch); toc; - From 225e14b4923ddc864eb840f0d20735a34aa9a2f6 Mon Sep 17 00:00:00 2001 From: marcobarilari <38101692+marcobarilari@users.noreply.github.com> Date: Fri, 13 Nov 2020 10:45:05 +0100 Subject: [PATCH 08/14] Update src/batches/setBatchGZip.m Co-authored-by: Remi Gau --- src/batches/setBatchGZip.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/batches/setBatchGZip.m b/src/batches/setBatchGZip.m index 2847db26..4860f968 100644 --- a/src/batches/setBatchGZip.m +++ b/src/batches/setBatchGZip.m @@ -6,7 +6,7 @@ % % USAGE:: % - % matlabbatch = setGZip(unzippedNiifiles, deleteUnzippedNii) + % matlabbatch = setBatchGZip(unzippedNiifiles, keepUnzippedNii) % % :param unzippedNiifiles: List of volumes to be gzipped % :type unzippedNiifiles: array From d180facc95816b5a005493c2386e7d4f6ab4fd3c Mon Sep 17 00:00:00 2001 From: marcobarilari <38101692+marcobarilari@users.noreply.github.com> Date: Fri, 13 Nov 2020 10:45:26 +0100 Subject: [PATCH 09/14] Update src/batches/setBatchGZip.m Co-authored-by: Remi Gau --- src/batches/setBatchGZip.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/batches/setBatchGZip.m b/src/batches/setBatchGZip.m index 4860f968..104cbac8 100644 --- a/src/batches/setBatchGZip.m +++ b/src/batches/setBatchGZip.m @@ -10,7 +10,7 @@ % % :param unzippedNiifiles: List of volumes to be gzipped % :type unzippedNiifiles: array - % :param keepUnzippedNii: Obligatory argument. Boolean to decide to delete the unzipped files + % :param keepUnzippedNii: Boolean to decide to delete the unzipped files % :type keepUnzippedNii: boolean % % :returns: - :matlabbatch: (struct) The matlabbath ready to run the spm job From 5d31a7474e3dd93a2e3c7a809c452dce6d476001 Mon Sep 17 00:00:00 2001 From: marcobarilari <38101692+marcobarilari@users.noreply.github.com> Date: Fri, 13 Nov 2020 10:45:53 +0100 Subject: [PATCH 10/14] Update src/workflows/bidsGZipRawFolder.m Co-authored-by: Remi Gau --- src/workflows/bidsGZipRawFolder.m | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/workflows/bidsGZipRawFolder.m b/src/workflows/bidsGZipRawFolder.m index 15b37740..c7fc9b7e 100644 --- a/src/workflows/bidsGZipRawFolder.m +++ b/src/workflows/bidsGZipRawFolder.m @@ -7,9 +7,8 @@ function bidsGZipRawFolder(optSource, keepUnzippedNii) % % USAGE:: % - % bidsGZipRawFolder([opt,] ... - % [deleteUnzippedNii = true,] ... - % [modalitiesToZip = {'anat', 'func', 'fmap'}]) + % bidsGZipRawFolder(optSource ... + % [, keepUnzippedNii = false]) % % :param optSource: The structure that contains the options set by the user to run the batch % workflow for source processing From e3f76616a961d19e8e54241266b5987de93a9683 Mon Sep 17 00:00:00 2001 From: marcobarilari <38101692+marcobarilari@users.noreply.github.com> Date: Fri, 13 Nov 2020 10:46:03 +0100 Subject: [PATCH 11/14] Update src/workflows/bidsGZipRawFolder.m Co-authored-by: Remi Gau --- src/workflows/bidsGZipRawFolder.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workflows/bidsGZipRawFolder.m b/src/workflows/bidsGZipRawFolder.m index c7fc9b7e..490b710c 100644 --- a/src/workflows/bidsGZipRawFolder.m +++ b/src/workflows/bidsGZipRawFolder.m @@ -19,7 +19,7 @@ function bidsGZipRawFolder(optSource, keepUnzippedNii) %% input variables default values - if nargin < 2 || isempty(deleteUnzippedNii) + if nargin < 2 || isempty(keepUnzippedNii) % delete the original unzipped .nii keepUnzippedNii = false; end From a80bd6bee089e77fac0298126efea0d57997e18f Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Fri, 13 Nov 2020 12:13:00 +0100 Subject: [PATCH 12/14] update documentation --- src/defaults/checkOptionsSource.m | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/defaults/checkOptionsSource.m b/src/defaults/checkOptionsSource.m index cfc19a4d..e1db9187 100644 --- a/src/defaults/checkOptionsSource.m +++ b/src/defaults/checkOptionsSource.m @@ -12,6 +12,16 @@ % to run the batch workflow for source processing % % :returns: - :optSource: (struc) The structure with any unset fields with the deaufalt values + % + % OPTIONS (with their defaults): + % - ``optSource.sourceDir = ''`` - The directory where the source data are located. + % - ``optSource.dataDir = ''`` - The directory where the raw data to apply changes are located. + % - ``optSource.sequenceToIgnore = {}`` - The list of sequence(s) to ignore. + % - ``optSource.dataType = 0`` - Data format conversion (0 is reccomended). + % - ``optSource.zip = 0`` - Boolean to enable gzip of the new 4D file in ``convert3Dto4D``. + % - ``optSource.nbDummies = 0`` - Number of volumes to discard ad dummies in ``convert3Dto4D``. + % - ``optSource.sequenceRmDummies = {}`` - The list of sequence(s) where to discarding the + % dummies. fieldsToSet = setDefaultOptionSource(); From a8ae5135a414de258a6d28adaee72fbb97eb1166 Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Fri, 13 Nov 2020 12:13:14 +0100 Subject: [PATCH 13/14] add test for checkOptionsSource --- tests/test_checkOptionsSource.m | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/test_checkOptionsSource.m diff --git a/tests/test_checkOptionsSource.m b/tests/test_checkOptionsSource.m new file mode 100644 index 00000000..dc2d17a2 --- /dev/null +++ b/tests/test_checkOptionsSource.m @@ -0,0 +1,7 @@ +function test_suite = test_checkOptionsSource %#ok<*STOUT> + try % assignment of 'localfunctions' is necessary in Matlab >= 2016 + test_functions = localfunctions(); %#ok<*NASGU> + catch % no problem; early Matlab versions can use initTestSuite fine + end + initTestSuite; +end \ No newline at end of file From eefa8ae8e3e64061231a4cafec6fac1ca94da50d Mon Sep 17 00:00:00 2001 From: marcobarilari Date: Fri, 13 Nov 2020 12:24:46 +0100 Subject: [PATCH 14/14] add code to test fun checkOptionsSource --- tests/test_checkOptionsSource.m | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/test_checkOptionsSource.m b/tests/test_checkOptionsSource.m index dc2d17a2..48fa1e74 100644 --- a/tests/test_checkOptionsSource.m +++ b/tests/test_checkOptionsSource.m @@ -4,4 +4,48 @@ catch % no problem; early Matlab versions can use initTestSuite fine end initTestSuite; +end + +function test_checkOptionsSourceBasic() + + optSource.nbDummies = 0; + optSource = checkOptionsSource(optSource); + + expectedOptionsSource = defaultOptionsSource(); + expectedOptionsSource.nbDummies = 0; + + assertEqual(optSource, expectedOptionsSource); + +end + +function test_checkOptionsSourceDoNotOverwrite() + + optSource.dataType = 666; + optSource.someExtraField = 'test'; + optSource.nbDummies = 42; + + optSource = checkOptionsSource(optSource); + + assertEqual(optSource.dataType, 666); + assertEqual(optSource.someExtraField, 'test'); + assertEqual(optSource.nbDummies, 42); + +end + +function expectedOptionsSource = defaultOptionsSource() + + expectedOptionsSource.sourceDir = ''; + + expectedOptionsSource.dataDir = ''; + + expectedOptionsSource.sequenceToIgnore = {}; + + expectedOptionsSource.dataType = 0; + + expectedOptionsSource.zip = 0; + + expectedOptionsSource.nbDummies = 0; + + expectedOptionsSource.sequenceRmDummies = {}; + end \ No newline at end of file