improve creation of stim file and their conversion to raw

.gitignore

@@ -6,16 +6,17 @@
 *octave-workspace
 
 # exclude content of logfiles folders
-tests/output/*
-manualTests/output/*
-output/*
+*output*
 *.tsv
 *.mat
 
 # exclude temp files from tests and coverage
-tests/coverage*
-tests/test_code_report.txt
-test_code_report.txt
+*test_code_report.txt
+*coverage*
+
+tests/*.nii*
+tests/*.json*
+tests/*.tsv*
 
 # exclude report from check_my_code
 check_my_code_report.txt

README.md

@@ -2,11 +2,13 @@
 
 [![Build Status](https://travis-ci.com/cpp-lln-lab/CPP_BIDS.svg?branch=master)](https://travis-ci.com/cpp-lln-lab/CPP_BIDS)
 
-**Unit tests**
+**Unit tests and coverage**
 
 [![](https://img.shields.io/badge/Octave-CI-blue?logo=Octave&logoColor=white)](https://github.com/cpp-lln-lab/CPP_BIDS/actions)
 ![](https://github.com/cpp-lln-lab/CPP_BIDS/workflows/CI/badge.svg) 
 
+[![codecov](https://codecov.io/gh/cpp-lln-lab/CPP_BIDS/branch/master/graph/badge.svg)](https://codecov.io/gh/cpp-lln-lab/CPP_BIDS)
+
 **Contributors**
 
 [![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-) 

manualTests/test_makeRawDataset.m

@@ -68,12 +68,25 @@ function test_makeRawDataset()
     % close the file
     saveEventsFile('close', cfg, logFile);
 
+    % add dummy stim data
+    stimLogFile = saveEventsFile('open_stim', cfg, logFile);
+    for i = 1:100
+        stimLogFile(i, 1).onset = cfg.mri.repetitionTime * i;
+        stimLogFile(i, 1).trial_type = 'test';
+        stimLogFile(i, 1).duration = 1;
+        stimLogFile(i, 1).Speed = rand(1);
+        stimLogFile(i, 1).is_Fixation = rand > 0.5;
+        stimLogFile(i, 1).LHL24 = randn(1, 3);
+    end
+    saveEventsFile('save', cfg, stimLogFile);
+    saveEventsFile('close', cfg, stimLogFile);
+
     % add dummy functional data
     funcDir = fullfile(cfg.dir.output, 'source', 'sub-001', 'ses-001', 'func');
     boldFilename = 'sub-001_ses-001_task-testtask_run-001_bold.nii.gz';
 
     copyfile( ...
-        fullfile('..', 'dummyData', 'dummyData.nii.gz'), ...
+        fullfile('dummyData', 'dummyData.nii.gz'), ...
         fullfile(funcDir, boldFilename));
 
     %% MRI bold rest data and fancy suffixes
@@ -87,6 +100,7 @@ function test_makeRawDataset()
     cfg.subject.sessionNb = 3;
     cfg.subject.runNb = 4;
 
+    % deal with MRI suffixes
     cfg.mri.reconstruction = 'fast recon';
     cfg.mri.contrastEnhancement = 'test';
     cfg.mri.phaseEncodingDirection = 'y pos';
@@ -100,19 +114,21 @@ function test_makeRawDataset()
 
     createBoldJson(cfg);
 
-    % add dummy functional data
+    %% add dummy functional data
     funcDir = fullfile(cfg.dir.output, 'source', 'sub-002', 'ses-003', 'func');
     boldFilename = ['sub-002_ses-003_task-rest',  ...
         '_acq-newTYpe_ce-test_dir-yPos_rec-fastRecon', ...
         '_run-004_echo-1_bold.nii.gz'];
 
     copyfile( ...
-        fullfile('..', 'dummyData', 'dummyData.nii.gz'), ...
+        fullfile('dummyData', 'dummyData.nii.gz'), ...
         fullfile(funcDir, boldFilename));
 
-    %%
+    %% actually do the conversion of the source data thus created
     clear;
+
     outputDir = fullfile(fileparts(mfilename('fullpath')), 'output');
     cfg.dir.output = outputDir;
     convertSourceToRaw(cfg);
+
 end

src/convertSourceToRaw.m

@@ -7,16 +7,17 @@ function convertSourceToRaw(cfg)
     % - copy source dir to raw dir
     % - remove the date suffix (_date-YYYYMMDDHHMM) from the files where it is present
     %
+    % Only covers func folder at the moment
 
     sourceDir = fullfile(cfg.dir.output, 'source');
     rawDir = fullfile(cfg.dir.output, 'rawdata');
 
     % add dummy README and CHANGE file
     copyfile(fullfile( ...
-        fileparts(mfilename('fullpath')), '..', 'dummyData', 'README'), ...
+        fileparts(mfilename('fullpath')), '..', 'manualTests', 'dummyData', 'README'), ...
         sourceDir);
     copyfile(fullfile( ...
-        fileparts(mfilename('fullpath')), '..', 'dummyData', 'CHANGES'), ...
+        fileparts(mfilename('fullpath')), '..', 'manualTests', 'dummyData', 'CHANGES'), ...
         sourceDir);
 
     copyfile(sourceDir, rawDir);
@@ -40,43 +41,3 @@ function convertSourceToRaw(cfg)
     end
 
 end
-
-function parseFunc(rawDir, subjName, sesName)
-
-    subjectPath = fullfile(rawDir, subjName, sesName, 'func');
-
-    if exist(subjectPath, 'dir')
-
-        % do events
-        filenames = file_utils('List', subjectPath, ...
-            sprintf('^%s.*_task-.*_events_date-.*$', subjName));
-
-        removeDateSuffix(filenames, subjectPath);
-
-        % do bold
-        filenames = file_utils('List', subjectPath, ...
-            sprintf('^%s.*_task-.*_bold_date-.*$', subjName));
-
-        removeDateSuffix(filenames, subjectPath);
-
-    end
-end
-
-function removeDateSuffix(filenames, subjectPath)
-    if isempty(filenames)
-        filenames = {};
-    else
-        filenames = cellstr(filenames);
-    end
-
-    for i = 1:numel(filenames)
-
-        [~, name, ext] = fileparts(filenames{i});
-
-        [parts, ~] = regexp(name, '(?:_date-)+', 'split', 'match');
-
-        movefile(fullfile(subjectPath, filenames{i}), ...
-            fullfile(subjectPath, [parts{1} ext]));
-
-    end
-end

src/saveEventsFile.m

@@ -64,13 +64,13 @@
 
         case 'open'
 
-            logFile.filename = cfg.fileName.events;
+            logFile(1).filename = cfg.fileName.events;
 
             logFile = initializeFile(cfg, logFile);
 
         case 'open_stim'
 
-            logFile.filename = cfg.fileName.stim;
+            logFile(1).filename = cfg.fileName.stim;
 
             logFile = initializeFile(cfg, logFile);
 
@@ -143,21 +143,22 @@
 
     % Initialize txt logfiles and empty fields for the standard BIDS
     %  event file
-    logFile.fileID = fopen( ...
+    logFile(1).fileID = fopen( ...
         fullfile( ...
         cfg.dir.outputSubject, ...
         cfg.fileName.modality, ...
         logFile.filename), ...
         'w');
 
     % print the basic BIDS columns
-    fprintf(logFile.fileID, '%s\t%s\t%s', 'onset', 'duration', 'trial_type');
+    fprintf(logFile(1).fileID, '%s\t%s\t%s', 'onset', 'duration', 'trial_type');
     fprintf(1, '%s\t%s\t%s', 'onset', 'duration', 'trial_type');
 
     printHeaderExtraColumns(logFile);
 
     % next line so we start printing at the right place
-    fprintf(logFile.fileID, '\n');
+    fprintf(logFile(1).fileID, '\n');
+    fprintf(1, '\n');
 
 end
 
@@ -174,7 +175,7 @@ function printHeaderExtraColumns(logFile)
 
             headerName = returnHeaderName(namesExtraColumns{iExtraColumn}, nbCol, iCol);
 
-            fprintf(logFile.fileID, '\t%s', headerName);
+            fprintf(logFile(1).fileID, '\t%s', headerName);
             fprintf(1, '\t%s', headerName);
 
         end

src/subfun/parseFunc.m

@@ -0,0 +1,45 @@
+function parseFunc(rawDir, subjName, sesName)
+    % parseFunc(rawDir, subjName, sesName)
+    %
+
+    subjectPath = fullfile(rawDir, subjName, sesName, 'func');
+
+    if exist(subjectPath, 'dir')
+
+        % do events
+        filenames = file_utils('List', subjectPath, ...
+            sprintf('^%s.*_task-.*_events_date-.*$', subjName));
+
+        removeDateSuffix(filenames, subjectPath);
+
+        % do bold
+        filenames = file_utils('List', subjectPath, ...
+            sprintf('^%s.*_task-.*_bold_date-.*$', subjName));
+
+        removeDateSuffix(filenames, subjectPath);
+
+        % do stim
+        filenames = file_utils('List', subjectPath, ...
+            sprintf('^%s.*_task-.*_stim_date-.*tsv$', subjName));
+        compressFiles(filenames, subjectPath);
+        filenames = file_utils('List', subjectPath, ...
+            sprintf('^%s.*_task-.*_stim_date-.*tsv.gz$', subjName));
+        removeDateSuffix(filenames, subjectPath);
+
+    end
+end
+
+function compressFiles(filenames, subjectPath)
+    if isempty(filenames)
+        filenames = {};
+    else
+        filenames = cellstr(filenames);
+    end
+
+    for i = 1:numel(filenames)
+
+        gzip(fullfile(subjectPath, filenames{i}));
+        delete(fullfile(subjectPath, filenames{i}));
+
+    end
+end

src/subfun/removeDateSuffix.m

@@ -0,0 +1,31 @@
+function removeDateSuffix(filenames, subjectPath)
+    % removeDateSuffix(filenames, subjectPath)
+    %
+    %
+
+    if isempty(filenames)
+        filenames = {};
+    else
+        filenames = cellstr(filenames);
+    end
+
+    for i = 1:numel(filenames)
+
+        [~, name, ext] = fileparts(filenames{i});
+
+        if strcmp(ext, '.gz')
+            [~, name, ext2] = fileparts(name);
+            ext = [ext2, ext]; %#ok<AGROW>
+        end
+
+        [parts, ~] = regexp(name, '(?:_date-)+', 'split', 'match');
+
+        % remove suffix file if there was one
+        if ~strcmp(filenames{i}, [parts{1} ext])
+            movefile(fullfile(subjectPath, filenames{i}), ...
+                fullfile(subjectPath, [parts{1} ext]));
+        end
+
+    end
+
+end

tests/test_removeDateSuffix.m

@@ -0,0 +1,70 @@
+function test_suite = test_removeDateSuffix %#ok<*STOUT>
+    try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+        test_functions = localfunctions(); %#ok<*NASGU>
+    catch % no problem; early Matlab versions can use initTestSuite fine
+    end
+    initTestSuite;
+end
+
+function test_removeDateSuffixBasic()
+
+    outputDir = pwd;
+
+    createFiles = 0;
+    testDo = 1;
+
+    %% set up
+    boldName = 'test_bold_date-202008050730.nii.gz';
+    boldName2 = 'test2_bold.nii.gz';
+    boldName3 = 'test3_bold_date-202008050730.nii';
+    jsonName = 'test_bold_date-202008050730.json';
+    eventsName = 'test_events_date-202008050730.tsv';
+    stimName = 'test_stim_date-202008050730.tsv';
+    stimNameZipped = 'test2_stim_date-202008050730.tsv.gz';
+
+    filesToProcess = { ...
+        boldName ;
+        boldName2 ;
+        boldName3 ;
+        jsonName ;
+        eventsName ;
+        stimName ;
+        stimNameZipped ;
+        };
+
+    % create new files for new tests
+    for iFile = 1:numel(filesToProcess)
+        system(sprintf('touch %s', filesToProcess{iFile}));
+    end
+
+    %% do stuff
+    filenames = file_utils('List', outputDir, '^test.*$');
+
+    %% expected data
+    expectedBoldName = 'test_bold.nii.gz';
+    expectedBoldName2 = 'test2_bold.nii.gz';
+    expectedBoldName3 = 'test3_bold.nii';
+    expectedJsonName = 'test_bold.json';
+    expectedEventsName = 'test_events.tsv';
+    expectedStimName = 'test_stim.tsv';
+    expectedStimNameZipped = 'test2_stim.tsv.gz';
+
+    removeDateSuffix(filenames, outputDir);
+
+    %% test
+    fprintf(1, fullfile(outputDir, expectedBoldName3));
+    assertEqual(exist(fullfile(outputDir, expectedBoldName3), 'file'), 2);
+    assertEqual(exist(fullfile(outputDir, expectedJsonName), 'file'), 2);
+    fprintf(1, fullfile(outputDir, expectedEventsName));
+    assertEqual(exist(fullfile(outputDir, expectedEventsName), 'file'), 2);
+    assertEqual(exist(fullfile(outputDir, expectedStimName), 'file'), 2);
+    assertEqual(exist(fullfile(outputDir, expectedStimNameZipped), 'file'), 2);
+    assertEqual(exist(fullfile(outputDir, expectedBoldName2), 'file'), 2);
+    assertEqual(exist(fullfile(outputDir, expectedBoldName), 'file'), 2);
+
+    % clean up
+    delete('*.nii*');
+    delete('*.tsv*');
+    delete('*.json');
+
+end