Merge pull request #197 from Remi-Gau/rm_data

[ENH] refactor removing date entity and allow to use bids fitler to only convert certain files from source to raw

.github/workflows/bids_validator.yml

@@ -2,10 +2,9 @@ name: BIDS validator
 
 on:
   push:
-    branches:
-      - master
+    branches: ['master', 'main', 'dev']
   pull_request:
-    branches: ['master']
+    branches: ['*']
 
 env:
   OCTFLAGS: --no-gui --no-window-system --silent
@@ -15,7 +14,11 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
 
-    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v3
+      with:
+        node-version: '14'
+
+    - uses: actions/checkout@v3
       with:
         submodules: true
         fetch-depth: 1
@@ -37,9 +40,10 @@ jobs:
 
     - name: Install BIDS validator
       run: |
-        sudo npm install -g bids-validator@1.9.0
+        sudo npm install -g bids-validator
 
     - name: Create dummy dataset and validate it
       run: |
         cd tests/manualTests
-        octave $OCTFLAGS --eval "test_makeRawDataset" && bids-validator `pwd`/output/raw/ --ignoreNiftiHeaders
+        octave $OCTFLAGS --eval "test_makeRawDataset"
+        bids-validator `pwd`/output/raw/ --ignoreNiftiHeaders

src/convertSourceToRaw.m

@@ -1,16 +1,19 @@
-function convertSourceToRaw(cfg)
+function convertSourceToRaw(varargin)
     %
     % Function attempts to convert a source dataset created with CPP_BIDS into a valid
     % BIDS data set.
     %
     %
     % USAGE::
     %
-    %   convertSourceToRaw(cfg)
+    %   convertSourceToRaw(cfg, 'filter', filter)
     %
     % :param cfg: cfg structure is needed only for providing the path in ``cfg.dir.output``.
     % :type cfg: structure
     %
+    % :param filter: bids.query filter to only convert a subset of files.
+    % :type filter: structure
+    %
     % :output:
     %          - :creates: a dummy README and CHANGE file
     %          - :copies: ``source`` directory to ``raw`` directory
@@ -19,39 +22,80 @@ function convertSourceToRaw(cfg)
     %
     % (C) Copyright 2020 CPP_BIDS developers
 
+    args = inputParser;
+
+    default_filter = struct([]);
+
+    args.addRequired('cfg', @isstruct);
+    args.addParameter('filter', default_filter, @isstruct);
+
+    args.parse(varargin{:});
+
+    cfg = args.Results.cfg;
+    filter = args.Results.filter;
+
     sourceDir = fullfile(cfg.dir.output, 'source');
     rawDir = fullfile(cfg.dir.output, 'raw');
 
-    % add dummy README and CHANGE file
-    templateFolder = fullfile(fileparts(mfilename('fullpath')), '..', 'templates');
+    % back up description to not overwrite
+    % TODO bids malab should be smart enought to not do that
+    isFile = @(x) exist(x, 'file');
+    if isFile(fullfile(rawDir, 'dataset_description.json'))
+        copyfile(fullfile(rawDir, 'dataset_description.json'), ...
+                 fullfile(rawDir, 'dataset_description_bu.json'));
+    end
 
-    copyfile(fullfile(templateFolder, 'README'), ...
-             sourceDir);
-    copyfile(fullfile(templateFolder, 'CHANGES'), ...
-             sourceDir);
-    copyfile(fullfile(templateFolder, '.bidsignore'), ...
-             sourceDir);
+    % trick use bids matlab copy dataset function
+    bids.copy_to_derivative(sourceDir, ...
+                            'pipeline_name', '.', ...
+                            'out_path', rawDir, ...
+                            'filter', filter, ...
+                            'unzip', false, ...
+                            'force', true, ...
+                            'skip_dep', false, ...
+                            'use_schema', false, ...
+                            'verbose', true);
 
-    copyfile(sourceDir, rawDir);
+    if isFile(fullfile(rawDir, 'dataset_description_bu.json'))
+        copyfile(fullfile(rawDir, 'dataset_description_bu.json'), ...
+                 fullfile(rawDir, 'dataset_description.json'));
+        delete(fullfile(rawDir, 'dataset_description_bu.json'));
+    else
+        % clean up description
+        description = bids.util.jsondecode(fullfile(rawDir, 'dataset_description.json'));
+        description.BIDSVersion = '1.7.0';
+        description.Name = 'FIXME';
+        description.DatasetType = 'raw';
+        description = rmfield(description, 'GeneratedBy');
+        description = rmfield(description, 'SourceDatasets');
+        bids.util.jsonencode(fullfile(rawDir, 'dataset_description.json'), description);
+    end
 
-    BIDS = bids.layout(rawDir,  'use_schema', false);
+    removeDateEntity(rawDir, 'filter', filter);
 
-    data = bids.query(BIDS, 'data');
-    metadata = bids.query(BIDS, 'metadata');
+    gunzipTimeSeries(rawDir);
 
-    for i = 1:size(data, 1)
-        bf = bids.File(data{i});
-        if isfield(bf.entities, 'date')
-            % TODO probably JSON renaming should be passed to bids-matlab
-            sourceJson = fullfile(fileparts(bf.path), bf.json_filename);
-            bf.entities.date = '';
-            bf.rename('dry_run', false, 'force', true);
-            bids.util.jsonencode(fullfile(fileparts(bf.path), bf.json_filename), metadata{i});
-            delete(sourceJson);
-        end
+    % add dummy README and CHANGE file
+    templateFolder = fullfile(fileparts(mfilename('fullpath')), '..', 'templates');
+
+    if ~isFile(fullfile(rawDir, 'README'))
+        copyfile(fullfile(templateFolder, 'README'), ...
+                 rawDir);
+    end
+    if ~isFile(fullfile(rawDir, 'CHANGES'))
+        copyfile(fullfile(templateFolder, 'CHANGES'), ...
+                 rawDir);
     end
+    if ~isFile(fullfile(rawDir, '.bidsignore'))
+        copyfile(fullfile(templateFolder, '.bidsignore'), ...
+                 rawDir);
+    end
+
+end
 
-    BIDS = bids.layout(rawDir,  'use_schema', false);
+function gunzipTimeSeries(pathToDataSet)
+
+    BIDS = bids.layout(pathToDataSet,  'use_schema', false);
     data = bids.query(BIDS, 'data', 'suffix', {'stim', 'physio' }, 'ext', '.tsv');
 
     for i = 1:size(data, 1)
@@ -60,5 +104,4 @@ function convertSourceToRaw(cfg)
             delete(data{i});
         end
     end
-
 end

src/utils/removeDateEntity.m

@@ -0,0 +1,41 @@
+function removeDateEntity(varargin)
+    %
+    % Removes the date entity from all the files in a BIDS data set
+    %
+    %
+    % USAGE::
+    %
+    %   removeDateEntity(pathToDataSet, 'filter', filter)
+    %
+    %
+    % (C) Copyright 2022 CPP_BIDS developers
+
+    args = inputParser;
+
+    default_filter = struct([]);
+
+    args.addRequired('pathToDataSet', @isdir);
+    args.addParameter('filter', default_filter, @isstruct);
+
+    args.parse(varargin{:});
+
+    pathToDataSet = args.Results.pathToDataSet;
+    filter = args.Results.filter;
+
+    BIDS = bids.layout(pathToDataSet,  'use_schema', false);
+
+    filter.date = '[0-9]+';
+    data = bids.query(BIDS, 'data', filter);
+    metadata = bids.query(BIDS, 'metadata', filter);
+
+    for i = 1:size(data, 1)
+        bf = bids.File(data{i});
+        % TODO probably JSON renaming should be passed to bids-matlab
+        sourceJson = fullfile(fileparts(bf.path), bf.json_filename);
+        bf.entities.date = '';
+        bf.rename('dry_run', false, 'force', true);
+        bids.util.jsonencode(fullfile(fileparts(bf.path), bf.json_filename), metadata{i});
+        delete(sourceJson);
+    end
+
+end