Merge pull request #146 from oesteban/fix/ReviveDS005

[FIX] Revive DS005

circle.yml

@@ -11,14 +11,17 @@ dependencies:
     # Download test data
     - mkdir -p ~/data ~/docker ~/scratch
     - if [[ ! -d ~/data/ds054 ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q https://3552243d5be815c1b09152da6525cb8fe7b900a6.googledrive.com/host/0BxI12kyv2olZVUswazA3NkFvOXM/ds054_downsampled.tar.gz && tar xzf ds054_downsampled.tar.gz -C ~/data/; fi
+    - if [[ ! -d ~/data/ds005 ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q https://3552243d5be815c1b09152da6525cb8fe7b900a6.googledrive.com/host/0BxI12kyv2olZVUswazA3NkFvOXM/ds005_downsampled.tar.gz && tar xzf ds005_downsampled.tar.gz -C ~/data/; fi
   override:
     - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
     - docker build -t poldracklab/fmriprep:latest .
     - mkdir -p ~/docker; docker save poldracklab/fmriprep:latest > ~/docker/image.tar
 test:
   override:
     - docker run -ti --rm --entrypoint="/usr/bin/run_unittests" poldracklab/fmriprep:latest
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/data:/data -v ~/scratch:/scratch -w /scratch poldracklab/fmriprep:latest /data/ds054 /scratch/out participant -w /scratch/work --debug --ants-nthreads 32 :
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/data:/data -v ~/scratch:/scratch -w /scratch poldracklab/fmriprep:latest /data/ds054 /scratch/out participant -w /scratch/work -t ds054 --debug --ants-nthreads 30 :
+        timeout: 4800
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/data:/data -v ~/scratch:/scratch -w /scratch poldracklab/fmriprep:latest /data/ds005 /scratch/out participant -w /scratch/work -t ds005 --debug --ants-nthreads 30 :
         timeout: 4800
 
 general:

fmriprep/run_workflow.py

@@ -3,7 +3,7 @@
 # @Author: oesteban
 # @Date:   2015-11-19 16:44:27
 # @Last Modified by:   oesteban
-# @Last Modified time: 2016-08-17 15:01:57
+# @Last Modified time: 2016-08-30 16:52:12
 """
 fMRI preprocessing workflow
 =====
@@ -28,7 +28,7 @@ def main():
     from nipype import config as ncfg
     from nipype.pipeline import engine as pe
     from fmriprep import __version__
-    from fmriprep.workflows import fmriprep_single
+    from fmriprep.workflows import base as fwb
 
     parser = ArgumentParser(description='fMRI Preprocessing workflow',
                             formatter_class=RawTextHelpFormatter)
@@ -62,6 +62,9 @@ def main():
                          help='nipype plugin configuration file')
     g_input.add_argument('-w', '--work-dir', action='store',
                            default=op.join(os.getcwd(), 'work'))
+    g_input.add_argument('-t', '--workflow-type', default='ds005', required=True,
+                         action='store', choices=['ds005', 'ds054', 'HPC', 'spiral'],
+                         help='workflow type, a monkeypatch while it is not automatically identified')
 
     # ANTs options
     g_ants = parser.add_argument_group('specific settings for ANTs registrations')
@@ -110,7 +113,7 @@ def main():
         if not op.exists(log_dir):
             os.makedirs(log_dir)
 
-    logger.addHandler(logging.FileHandler(op.join(log_dir,'run_workflow')))
+    logger.addHandler(logging.FileHandler(op.join(log_dir, 'run_workflow')))
 
     # Warn for default work/output directories
     if settings['work_dir'] == parser.get_default('work_dir'):
@@ -147,7 +150,8 @@ def main():
     logger.info('Subject list: %s', ', '.join(subject_list))
 
     # Build main workflow and run
-    preproc_wf = fmriprep_single(subject_list, settings=settings)
+    workflow_generator = getattr(fwb, 'wf_{}_type'.format(opts.workflow_type))
+    preproc_wf = workflow_generator(subject_list, settings=settings)
     preproc_wf.base_dir = settings['work_dir']
     preproc_wf.run(**plugin_settings)
 

fmriprep/workflows/__init__.py

@@ -2,7 +2,6 @@
 # -*- coding: utf-8 -*-
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
-from fmriprep.workflows.base import fmriprep_single
 from fmriprep.workflows.anatomical import t1w_preprocessing
 from fmriprep.workflows.sbref import sbref_preprocess, sbref_t1_registration
 from fmriprep.workflows.epi import epi_sbref_registration

fmriprep/workflows/anatomical.py

@@ -23,7 +23,7 @@
 from niworkflows.common import reorient as mri_reorient_wf
 
 from fmriprep.interfaces import DerivativesDataSink, IntraModalMerge
-from fmriprep.data import get_ants_oasis_template_ras, get_mni_template_ras
+from fmriprep.data import get_ants_oasis_template_ras, get_mni_template
 from fmriprep.viz import stripped_brain_overlay, anatomical_overlay
 
 
@@ -46,7 +46,7 @@ def t1w_preprocessing(name='t1w_preprocessing', settings=None):
     t1wmrg = pe.Node(IntraModalMerge(), name='MergeT1s')
 
     # 1. Reorient T1
-    arw = pe.Node(fs.MRIConvert(out_type='niigz', out_orientation='RAS'), name='Reorient')
+    arw = pe.Node(fs.MRIConvert(out_type='niigz', out_orientation='LAS'), name='Reorient')
 
     # 2. T1 Bias Field Correction
     inu_n4 = pe.Node(ants.N4BiasFieldCorrection(dimension=3), name='CorrectINU')
@@ -62,9 +62,9 @@ def t1w_preprocessing(name='t1w_preprocessing', settings=None):
     # 5. T1w to MNI registration
     t1_2_mni = pe.Node(ants.Registration(), name='T1_2_MNI_Registration')
     t1_2_mni.inputs.num_threads = settings.get('ants_threads', 8)
-    t1_2_mni.inputs.fixed_image = op.join(get_mni_template_ras(), 'MNI152_T1_1mm.nii.gz')
+    t1_2_mni.inputs.fixed_image = op.join(get_mni_template(), 'MNI152_T1_1mm.nii.gz')
     t1_2_mni.inputs.fixed_image_mask = op.join(
-        get_mni_template_ras(), 'MNI152_T1_1mm_brain_mask.nii.gz')
+        get_mni_template(), 'MNI152_T1_1mm_brain_mask.nii.gz')
 
     # Hack to avoid re-running ANTs all the times
     grabber_interface = nio.JSONFileGrabber()
@@ -78,10 +78,10 @@ def t1w_preprocessing(name='t1w_preprocessing', settings=None):
     # Resampe the brain mask and the tissue probability maps into mni space
     bmask_mni = pe.Node(ants.ApplyTransforms(
         dimension=3, default_value=0, interpolation='NearestNeighbor'), name='brain_mni_warp')
-    bmask_mni.inputs.reference_image = op.join(get_mni_template_ras(), 'MNI152_T1_1mm.nii.gz')
+    bmask_mni.inputs.reference_image = op.join(get_mni_template(), 'MNI152_T1_1mm.nii.gz')
     tpms_mni = pe.MapNode(ants.ApplyTransforms(dimension=3, default_value=0, interpolation='Linear'),
                           iterfield=['input_image'], name='tpms_mni_warp')
-    tpms_mni.inputs.reference_image = op.join(get_mni_template_ras(), 'MNI152_T1_1mm.nii.gz')
+    tpms_mni.inputs.reference_image = op.join(get_mni_template(), 'MNI152_T1_1mm.nii.gz')
 
 
     workflow.connect([
@@ -119,13 +119,13 @@ def t1w_preprocessing(name='t1w_preprocessing', settings=None):
     # The T1-to-MNI will be plotted using the segmentation. That's why we transform it first
     seg_2_mni = pe.Node(ants.ApplyTransforms(
         dimension=3, default_value=0, interpolation='NearestNeighbor'), name='T1_2_MNI_warp')
-    seg_2_mni.inputs.reference_image = op.join(get_mni_template_ras(), 'MNI152_T1_1mm.nii.gz')
+    seg_2_mni.inputs.reference_image = op.join(get_mni_template(), 'MNI152_T1_1mm.nii.gz')
 
     t1_2_mni_overlay = pe.Node(niu.Function(
         input_names=['in_file', 'overlay_file', 'out_file'], output_names=['out_file'],
         function=stripped_brain_overlay), name='PNG_T1_to_MNI')
     t1_2_mni_overlay.inputs.out_file = 't1_to_mni_overlay.png'
-    t1_2_mni_overlay.inputs.overlay_file = op.join(get_mni_template_ras(), 'MNI152_T1_1mm.nii.gz')
+    t1_2_mni_overlay.inputs.overlay_file = op.join(get_mni_template(), 'MNI152_T1_1mm.nii.gz')
 
     datasink = pe.Node(
         interface=nio.DataSink(

fmriprep/workflows/base.py

@@ -22,9 +22,16 @@
     epi_unwarp, epi_hmc, epi_sbref_registration,
     epi_mean_t1_registration, epi_mni_transformation)
 
-def fmriprep_single(subject_list, name='fMRI_prep', settings=None):
+def wf_ds054_type(subject_list, name='fMRI_prep', settings=None):
     """
-    The main fmri preprocessing workflow.
+    The main fmri preprocessing workflow, for the ds054-type of data:
+
+      * [x] Has at least one T1w and at least one bold file (minimal reqs.)
+      * [x] Has one or more SBRefs
+      * [x] Has one or more GRE-phasediff images, including the corresponding magnitude images.
+      * [ ] No SE-fieldmap images
+      * [ ] No Spiral Echo fieldmap
+
     """
 
     if settings is None:
@@ -81,105 +88,63 @@ def fmriprep_single(subject_list, name='fMRI_prep', settings=None):
                                   ('outputnode.fmap_mask', 'inputnode.fmap_mask'),
                                   ('outputnode.fmap_ref', 'inputnode.fmap_ref')])
     ])
+    return workflow
 
 
-    # inputnode = pe.Node(niu.IdentityInterface(
-    #     fields=['fieldmaps', 't1', 'func', 'sbref']), name='inputnode')
-
-    # for key in subject_list.keys():
-    #     if key != 'func':
-    #         setattr(inputnode.inputs, key, subject_list[key])
-
-    # inputfmri = pe.Node(niu.IdentityInterface(
-    #                     fields=['func']), name='inputfmri')
-    # inputfmri.iterables = [('func', subject_list['func'])]
-
-    # outputnode = pe.Node(niu.IdentityInterface(
-    #     fields=['fieldmap', 'corrected_sbref', 'fmap_mag', 'fmap_mag_brain',
-    #             't1', 'stripped_epi', 'corrected_epi_mean', 'sbref_brain',
-    #             'stripped_epi_mask', 'stripped_t1', 't1_segmentation',
-    #             't1_2_mni', 't1_wm_seg']),
-    #     name='outputnode'
-    # )
-
-    # try:
-    #     fmap_wf = fieldmap_decider(getattr(inputnode.inputs, 'fieldmaps'), settings)
-    # except NotImplementedError:
-    #     fmap_wf = None
-
-    # # Reorient EPI to RAS
-    # split = pe.Node(fsl.Split(dimension='t'), name='SplitEPI')
-    # orient = pe.MapNode(fs.MRIConvert(out_type='niigz', out_orientation='RAS'),
-    #                     iterfield=['in_file'], name='ReorientEPI')
-    # merge = pe.Node(fsl.Merge(dimension='t'), name='MergeEPI')
-
-    #
-    # epi_hmc_wf = epi_hmc(settings=settings, sbref_present=sbref_present)
-    # epi_mni_trans_wf = epi_mni_transformation(settings=settings)
-
-    # #  Connecting Workflow pe.Nodes
-    # workflow.connect([
-    #     (inputfmri, split, [('func', 'in_file')]),
-    #     (split, orient, [('out_files', 'in_file')]),
-    #     (orient, merge, [('out_file', 'in_files')]),
-
-    #     (inputnode, t1w_preproc, [('t1', 'inputnode.t1')]),
-    #     (merge, epi_hmc_wf, [('merged_file', 'inputnode.epi_ras')]),
-    #     (merge, epi_mni_trans_wf, [('merged_file', 'inputnode.epi_ras')]),
-
-    #     # These are necessary sources for the DerivativesDataSink
-    #     (inputfmri, epi_hmc_wf, [('func', 'inputnode.epi')]),
-    #     (inputfmri, epi_mni_trans_wf, [('func', 'inputnode.epi')])
-    # ])
-
-    # if not sbref_present:
-    #     epi_2_t1 = epi_mean_t1_registration(settings=settings)
-    #     workflow.connect([
-    #         (inputfmri, epi_2_t1, [('func', 'inputnode.epi')]),
-    #         (epi_hmc_wf, epi_2_t1, [('outputnode.epi_mean', 'inputnode.epi_mean')]),
-    #         (t1w_preproc, epi_2_t1, [('outputnode.t1_brain', 'inputnode.t1_brain'),
-    #                                  ('outputnode.t1_seg', 'inputnode.t1_seg')]),
-    #         (epi_2_t1, epi_mni_trans_wf, [('outputnode.mat_epi_to_t1', 'inputnode.mat_epi_to_t1')]),
-
-    #         (epi_hmc_wf, epi_mni_trans_wf, [('outputnode.xforms', 'inputnode.hmc_xforms'),
-    #                                         ('outputnode.epi_mask', 'inputnode.epi_mask')]),
-    #         (t1w_preproc, epi_mni_trans_wf, [('outputnode.t1_brain', 'inputnode.t1'),
-    #                                          ('outputnode.t1_2_mni_forward_transform',
-    #                                           'inputnode.t1_2_mni_forward_transform')])
-    #     ])
-
-    # if fmap_wf:
-    #     sbref_wf = sbref_workflow(settings=settings)
-    #     sbref_wf.inputs.inputnode.hmc_mats = []  # FIXME: plug MCFLIRT output here
-    #     sbref_t1 = sbref.sbref_t1_registration(settings=settings)
-    #     unwarp_wf = epi_unwarp(settings=settings)
-    #     sepair_wf = se_pair_workflow(settings=settings)
-    #     workflow.connect([
-    #         (inputnode, sepair_wf, [('fieldmaps', 'inputnode.input_images')]),
-    #         (inputnode, sbref_wf, [('sbref', 'inputnode.sbref')]),
-    #         (inputfmri, unwarp_wf, [('func', 'inputnode.epi')]),
-    #         (sepair_wf, sbref_wf, [
-    #             ('outputnode.mag_brain', 'inputnode.fmap_ref_brain'),
-    #             ('outputnode.fmap_mask', 'inputnode.fmap_mask'),
-    #             ('outputnode.fieldmap', 'inputnode.fieldmap')
-    #         ]),
-    #         (sbref_wf, sbref_t1, [
-    #             ('outputnode.sbref_unwarped', 'inputnode.sbref_brain')]),
-    #         (t1w_preproc, sbref_t1, [
-    #             ('outputnode.t1_brain', 'inputnode.t1_brain'),
-    #             ('outputnode.t1_seg', 'inputnode.t1_seg')]),
-    #         (sbref_wf, unwarp_wf, [
-    #             ('outputnode.sbref_unwarped', 'inputnode.sbref_brain')]),
-    #         (sbref_wf, epi_hmc_wf, [
-    #             ('outputnode.sbref_unwarped', 'inputnode.sbref_brain')]),
-    #         (epi_hmc_wf, unwarp_wf, [
-    #             ('outputnode.epi_brain', 'inputnode.epi_brain')]),
-    #         (sepair_wf, unwarp_wf, [
-    #             ('outputnode.fmap_mask', 'inputnode.fmap_mask'),
-    #             ('outputnode.fieldmap', 'inputnode.fieldmap')
-    #         ]),
-    #     ])
+def wf_ds005_type(subject_list, name='fMRI_prep', settings=None):
+    """
+    The main fmri preprocessing workflow, for the ds005-type of data:
+
+      * [x] Has at least one T1w and at least one bold file (minimal reqs.)
+      * [ ] No SBRefs
+      * [ ] No GRE-phasediff images, including the corresponding magnitude images.
+      * [ ] No SE-fieldmap images
+      * [ ] No Spiral Echo fieldmap
 
+    """
+
+    if settings is None:
+        settings = {}
+
+    workflow = pe.Workflow(name=name)
+
+    inputnode = pe.Node(niu.IdentityInterface(fields=['subject_id']),
+                        name='inputnode')
+    inputnode.iterables = [('subject_id', subject_list)]
+
+    bidssrc = pe.Node(BIDSDataGrabber(bids_root=settings['bids_root']),
+                      name='BIDSDatasource')
+
+    # Preprocessing of T1w (includes registration to MNI)
+    t1w_pre = t1w_preprocessing(settings=settings)
+
+    # HMC on the EPI
+    hmcwf = epi_hmc(settings=settings)
+
+    # mean EPI registration to T1w
+    epi_2_t1 = epi_mean_t1_registration(settings=settings)
+
+    # Apply transforms in 1 shot
+    epi_mni_trans_wf = epi_mni_transformation(settings=settings)
+
+    workflow.connect([
+        (inputnode, bidssrc, [('subject_id', 'subject_id')]),
+        (bidssrc, t1w_pre, [('t1w', 'inputnode.t1w')]),
+        (bidssrc, hmcwf, [(('func', _first), 'inputnode.epi')]),
+        (bidssrc, epi_2_t1, [(('func', _first), 'inputnode.epi')]),
+
+        (hmcwf, epi_2_t1, [('outputnode.epi_mean', 'inputnode.epi_mean')]),
+        (t1w_pre, epi_2_t1, [('outputnode.t1_brain', 'inputnode.t1_brain'),
+                             ('outputnode.t1_seg', 'inputnode.t1_seg')]),
+        (bidssrc, epi_mni_trans_wf, [(('func', _first), 'inputnode.epi')]),
+        (epi_2_t1, epi_mni_trans_wf, [('outputnode.mat_epi_to_t1', 'inputnode.mat_epi_to_t1')]),
+        (hmcwf, epi_mni_trans_wf, [('outputnode.xforms', 'inputnode.hmc_xforms'),
+                                   ('outputnode.epi_mask', 'inputnode.epi_mask')]),
+        (t1w_pre, epi_mni_trans_wf, [('outputnode.t1_brain', 'inputnode.t1'),
+                                     ('outputnode.t1_2_mni_forward_transform',
+                                      'inputnode.t1_2_mni_forward_transform')])
+
+    ])
     return workflow
 
 def _first(inlist):

fmriprep/workflows/epi.py

@@ -234,7 +234,6 @@ def epi_mni_transformation(name='EPIMNITransformation', settings=None):
             'mat_epi_to_t1',
             't1_2_mni_forward_transform',
             'epi',
-            'epi_ras',
             'epi_mask',
             't1',
             'hmc_xforms'
@@ -277,7 +276,7 @@ def _aslist(in_value):
             suffix='hmc_mni_bmask'), name='DerivativesHMCMNImask')
 
     workflow.connect([
-        (inputnode, pick_1st, [('epi_ras', 'in_file')]),
+        (inputnode, pick_1st, [('epi', 'in_file')]),
         (inputnode, ds_mni, [('epi', 'source_file')]),
         (inputnode, ds_mni_mask, [('epi', 'source_file')]),
         (pick_1st, gen_ref, [('roi_file', 'moving_image')]),
@@ -286,7 +285,7 @@ def _aslist(in_value):
                                        ('hmc_xforms', 'in3')]),
         (inputnode, mask_merge_tfms, [('t1_2_mni_forward_transform', 'in1'),
                                       (('mat_epi_to_t1', _aslist), 'in2')]),
-        (inputnode, split, [('epi_ras', 'in_file')]),
+        (inputnode, split, [('epi', 'in_file')]),
         (split, epi_to_mni_transform, [('out_files', 'input_image')]),
         (merge_transforms, epi_to_mni_transform, [('out', 'transforms')]),
         (gen_ref, epi_to_mni_transform, [('out_file', 'reference_image')]),