Merge pull request #75 from bpoldrack/rf-conversion-procedures

[WIP] RF:  conversion procedures

.travis.yml

@@ -27,7 +27,7 @@ env:
     - COVERAGE=coverage
     - DATALAD_DATASETS_TOPURL=http://datasets-tests.datalad.org
   matrix:
-    - DATALAD_REPO_DIRECT=yes
+    - DATALAD_REPO_VERSION=6
     - DATALAD_REPO_VERSION=5
 
 

datalad_hirni/commands/dicom2spec.py

@@ -42,15 +42,22 @@ def add_to_spec(ds_metadata, spec_list, basepath,
             # Note: The first 4 entries aren't a dict and have no
             # "approved flag", since they are automatically managed
             'type': 'dicomseries',
-            #'status': None,  # TODO: process state convention; flags
             'location': op.relpath(ds_metadata['path'], basepath),
             'uid': series['SeriesInstanceUID'],
-            'dataset_id': ds_metadata['dsid'],
-            'dataset_refcommit': ds_metadata['refcommit'],
-            'converter': {
-                'value': 'heudiconv' if series_is_valid(series) else 'ignore',
-                # TODO: not clear yet, what exactly to specify here
-                'approved': False},
+            'dataset-id': ds_metadata['dsid'],
+            'dataset-refcommit': ds_metadata['refcommit'],
+            'converter': [{
+                # special value 'hirni-dicom-converter' is interpreted by
+                # spec2bids and doesn't need a 'procedure-call' entry:
+                'procedure-name': {'value': 'hirni-dicom-converter'
+                                            if series_is_valid(series)
+                                            else 'ignore',
+                                   'approved': False},
+                'procedure-call': {'value': None,
+                                   'approved': False},
+                'once-per-acquisition': {'value': True if series_is_valid(series) else None,
+                                         'approved': False}
+            }]
         })
 
     # get rules to apply:
@@ -250,8 +257,8 @@ def __call__(path=None, spec=None, dataset=None, subject=None,
         for i in range(len(spec_series_list)):
             if spec_series_list[i]["type"] == "dicomseries" and \
                     heuristic.has_specval(spec_series_list[i], "converter") and \
-                            heuristic.get_specval(spec_series_list[i], "bids_run") in \
-                            [heuristic.get_specval(s, "bids_run")
+                            heuristic.get_specval(spec_series_list[i], "bids-run") in \
+                            [heuristic.get_specval(s, "bids-run")
                              for s in spec_series_list[i + 1:]
                              if heuristic.get_specval(s,
                                                       "description") == heuristic.get_specval(

datalad_hirni/commands/spec4anything.py

@@ -24,7 +24,7 @@
 
 # TODO: Prob. should be (partially?) editable, but for now we need consistency
 # here:
-non_editables = ['location', 'type', 'dataset_id', 'dataset_refcommit']
+non_editables = ['location', 'type', 'dataset-id', 'dataset-refcommit']
 
 
 def _get_edit_dict(value=None, approved=False):
@@ -51,8 +51,8 @@ def _add_to_spec(spec, spec_dir, path, meta, overrides=None, replace=False):
     snippet = {
         'type': 'generic_' + path['type'],
         'location': posixpath.relpath(path['path'], spec_dir),
-        'dataset_id': meta['dsid'],
-        'dataset_refcommit': meta['refcommit'],
+        'dataset-id': meta['dsid'],
+        'dataset-refcommit': meta['refcommit'],
         'id': _get_edit_dict(),
         'converter': _get_edit_dict(),
         'comment': _get_edit_dict(value=""),

datalad_hirni/resources/webapp/specedit.html

@@ -66,7 +66,7 @@
         Common properties (record: <code>{{ studyspecfile }}</code>)
       </div>
       <div class="card-body">
-          <div v-for="(sval, skey) in {'Subject ID': 'subject', 'Anonymized Subject ID': 'anon_subject'}" class="form-group row">
+          <div v-for="(sval, skey) in {'Subject ID': 'subject', 'Anonymized Subject ID': 'anon-subject'}" class="form-group row">
             <label form="specform" class="col-sm-2 col-form-label">{{ skey }}</label>
             <div class="col-sm-10">
               <div v-if="specs[0]" class="input-group">
@@ -85,12 +85,12 @@
         {{ studyspecfile }}/{{ spec.location }} [ <code>{{ spec.type }}</code><span v-if="spec.uid">; {{ spec.uid }}</span> ]
       </div>
       <ul class="card-body">
-          <li v-for="(sval, skey) in spec" v-if="sval === Object(sval) && !(skey in {subject: null, anon_subject: null})">
+          <li v-for="(sval, skey) in spec" v-if="sval === Object(sval) && !(skey in {subject: null, 'anon-subject': null, converter: null})">
             <div class="form-group row" v-bind:title="field_purpose[skey] || ''">
               <!-- this is some editable property -->
               <label form="specform" class="col-sm-2 col-form-label">
-                  <span v-if="skey !== skey.replace('bids_', '')">
-                      <span class="badge badge-primary">BIDS</span> {{ skey.replace('bids_', '') }}
+                  <span v-if="skey !== skey.replace('bids-', '')">
+                      <span class="badge badge-primary">BIDS</span> {{ skey.replace('bids-', '') }}
                   </span>
                   <span v-else>
                     {{ skey }}
@@ -113,7 +113,7 @@
             </div>
           </li>
 
-        <li data-toggle="tooltip" data-placement="right" title="Add a new field to this record, prefix field name with 'bids_' to add a BIDS property">
+        <li data-toggle="tooltip" data-placement="right" title="Add a new field to this record, prefix field name with 'bids-' to add a BIDS property">
           <div class="input-group add-field">
             <input v-bind:id="'tba_fieldname_' + specs_key" type="text" placeholder="new field name">
             <div class="input-group-append">
@@ -148,10 +148,9 @@
       specfiles: [],
       studyspecfile: null,
       field_purpose: {
-          comment: "Comments or notes for this record",
-          converter: "Command to call for converting data associated with this record to BIDS format, may use template syntax and placeholders",
-          converter_path: "Location of a program/script to use as converter command for this record, value can be used in converter command via {_hs[converter_path]} placeholder",
-          description: "Record description, originally extracted from the source file(s)"
+        comment: "Comments or notes for this record",
+        converter: "Procedures to call for converting data associated with this record to BIDS format, may use template syntax and placeholders",
+        description: "Record description, originally extracted from the source file(s)"
       }
     },
     methods: {

datalad_hirni/support/dicom2bids_rules.py

@@ -7,10 +7,10 @@
 # subjects to the rule set
 # XXX this is not used at all
 spec_keys = [
-    'bids_session',
-    'bids_task',
-    'bids_run',
-    'bids_modality',
+    'bids-session',
+    'bids-task',
+    'bids-run',
+    'bids-modality',
     'comment',
     'converter',
     'description',
@@ -131,19 +131,19 @@ def _rules(self, record, subject=None, anon_subject=None, session=None):
                 'description': record['SeriesDescription'] if "SeriesDescription" in record else '',
                 'comment': '',
                 'subject': apply_bids_label_restrictions(_guess_subject(record) if not subject else subject),
-                'anon_subject': apply_bids_label_restrictions(anon_subject) if anon_subject else None,
-                'bids_session': apply_bids_label_restrictions(_guess_session(record) if not session else session),
-                'bids_task': apply_bids_label_restrictions(_guess_task(record)),
-                'bids_run': apply_bids_label_restrictions(run) if run else str(self.runs[protocol_name]),
-                'bids_modality': apply_bids_label_restrictions(_guess_modality(record)),
+                'anon-subject': apply_bids_label_restrictions(anon_subject) if anon_subject else None,
+                'bids-session': apply_bids_label_restrictions(_guess_session(record) if not session else session),
+                'bids-task': apply_bids_label_restrictions(_guess_task(record)),
+                'bids-run': apply_bids_label_restrictions(run) if run else str(self.runs[protocol_name]),
+                'bids-modality': apply_bids_label_restrictions(_guess_modality(record)),
 
                 # TODO: No defaults yet (May be there shouldn't be defaults, but
                 # right now, that's not a conscious decision ...):
-                'bids_acquisition': apply_bids_label_restrictions(None), #acq
-                'bids_contrast_enhancement': apply_bids_label_restrictions(None), # ce
-                'bids_reconstruction_algorithm': apply_bids_label_restrictions(None), #rec
-                'bids_echo': apply_bids_label_restrictions(None), #echo
-                'bids_direction': apply_bids_label_restrictions(None), #dir
+                'bids-acquisition': apply_bids_label_restrictions(None), #acq
+                'bids-contrast-enhancement': apply_bids_label_restrictions(None), # ce
+                'bids-reconstruction-algorithm': apply_bids_label_restrictions(None), #rec
+                'bids-echo': apply_bids_label_restrictions(None), #echo
+                'bids-direction': apply_bids_label_restrictions(None), #dir
 
                 'id': record.get('SeriesNumber', None),
                 }
@@ -243,6 +243,10 @@ def _guess_modality(record):
             return "t2w"
         # END
 
+        # TEMP: Reproin workaround
+        if "func" in prot_parts:
+            return "bold"
+
     # found nothing, but modality isn't necessarily required
     return None
 

datalad_hirni/support/hirni_heuristic.py

@@ -59,17 +59,17 @@
 # map specification keys to BIDS abbreviation used in paths
 spec2bids_map = {
     'subject': "sub",
-    'anon_subject': "sub",
-    'bids_session': "ses",
-    'bids_task': "task",
-    'bids_run': "run",
-    'bids_modality': "mod",
-    'bids_acquisition': "acq",
-    'bids_scan': "scan",
-    'bids_contrast_enhancement': "ce",
-    'bids_reconstruction_algorithm': "rec",
-    'bids_echo': "echo",
-    'bids_direction': "dir",
+    'anon-subject': "sub",
+    'bids-session': "ses",
+    'bids-task': "task",
+    'bids-run': "run",
+    'bids-modality': "mod",
+    'bids-acquisition': "acq",
+    'bids-scan': "scan",
+    'bids-contrast-enhancement': "ce",
+    'bids-reconstruction-algorithm': "rec",
+    'bids-echo': "echo",
+    'bids-direction': "dir",
 }
 
 
@@ -160,22 +160,22 @@ def validate_spec(spec):
         raise ValueError("Image series specification is empty.")
 
     # check converter
-    converter = get_specval(spec, 'converter')
-    if converter == 'ignore':
-        lgr.debug("Skip series %s (marked 'ignore' in spec)", spec['uid'])
-        return False
-    if converter != 'heudiconv':
-        lgr.debug("Skip series %s since it's not supposed to be converted by "
-                  "heudiconv.", spec['uid'])
-        return False
+    # converter = get_specval(spec, 'converter')
+    # if converter == 'ignore':
+    #     lgr.debug("Skip series %s (marked 'ignore' in spec)", spec['uid'])
+    #     return False
+    # if converter != 'heudiconv':
+    #     lgr.debug("Skip series %s since it's not supposed to be converted by "
+    #               "heudiconv.", spec['uid'])
+    #     return False
 
     # mandatory keys for any spec dict (not only dicomseries)
     for k in spec.keys():
         # automatically managed keys with no subdict:
         # TODO: Where to define this list?
         # TODO: Test whether those are actually present!
-        if k in ['type', 'status', 'location', 'uid', 'dataset_id',
-                 'dataset_refcommit']:
+        if k in ['type', 'location', 'uid', 'dataset-id',
+                 'dataset-refcommit', 'converter']:
             continue
         if 'value' not in spec[k]:
             lgr.warning("DICOM series specification (UID: {uid}) has no value "
@@ -190,7 +190,7 @@ def validate_spec(spec):
         lgr.warning("Missing image series UID.")
         return False
 
-    for var in ('bids_modality',):
+    for var in ('bids-modality',):
         if not has_specval(spec, var):
             lgr.warning("Missing specification value for key '%s'", var)
             return False
@@ -235,13 +235,13 @@ def infotodict(seqinfo):  # pragma: no cover
 
         dirname = filename = "sub-{}".format(_spec.subject)
         # session
-        if has_specval(series_spec, 'bids_session'):
-            ses = get_specval(series_spec, 'bids_session')
+        if has_specval(series_spec, 'bids-session'):
+            ses = get_specval(series_spec, 'bids-session')
             dirname += "/ses-{}".format(ses)
             filename += "_ses-{}".format(ses)
 
         # data type
-        modality = get_specval(series_spec, 'bids_modality')
+        modality = get_specval(series_spec, 'bids-modality')
         # make cannonical if possible
         modality = modality_label_map.get(modality, modality)
         # apply fixed mapping from modality -> data_type
@@ -257,9 +257,9 @@ def infotodict(seqinfo):  # pragma: no cover
             # func/sub-<participant_label>[_ses-<session_label>]
             # _task-<task_label>[_acq-<label>][_rec-<label>][_run-<index>][_echo-<index>]_<modality_label>.nii[.gz]
 
-            for spec_key in ['bids_task', 'bids_acquisition',
-                             'bids_reconstruction_algorithm', 'bids_run',
-                             'bids_echo']:
+            for spec_key in ['bids-task', 'bids-acquisition',
+                             'bids-reconstruction_algorithm', 'bids-run',
+                             'bids-echo']:
                 if has_specval(series_spec, spec_key):
                     filename += "_{}-{}".format(
                             spec2bids_map[spec_key],
@@ -271,10 +271,10 @@ def infotodict(seqinfo):  # pragma: no cover
             # anat/sub-<participant_label>[_ses-<session_label>]
             # [_acq-<label>][_ce-<label>][_rec-<label>][_run-<index>][_mod-<label>]_<modality_label>.nii[.gz]
 
-            for spec_key in ['bids_acquisition',
-                             'bids_contrast_enhancement',
-                             'bids_reconstruction_algorithm',
-                             'bids_run']:
+            for spec_key in ['bids-acquisition',
+                             'bids-contrast_enhancement',
+                             'bids-reconstruction_algorithm',
+                             'bids-run']:
                 if has_specval(series_spec, spec_key):
                     filename += "_{}-{}".format(
                             spec2bids_map[spec_key],
@@ -289,8 +289,8 @@ def infotodict(seqinfo):  # pragma: no cover
             # dwi/sub-<participant_label>[_ses-<session_label>]
             # [_acq-<label>][_run-<index>]_dwi.nii[.gz]
 
-            for spec_key in ['bids_acquisition',
-                             'bids_run']:
+            for spec_key in ['bids-acquisition',
+                             'bids-run']:
                 if has_specval(series_spec, spec_key):
                     filename += "_{}-{}".format(
                             spec2bids_map[spec_key],
@@ -314,9 +314,9 @@ def infotodict(seqinfo):  # pragma: no cover
             # _fieldmap
             # _epi
 
-            for spec_key in ['bids_acquisition',
-                             'bids_direction',
-                             'bids_run']:
+            for spec_key in ['bids-acquisition',
+                             'bids-direction',
+                             'bids-run']:
                 if has_specval(series_spec, spec_key):
                     filename += "_{}-{}".format(
                             spec2bids_map[spec_key],