Merge pull request #212 from poldracklab/revert-211-revert-208-enh/de…

…rivatives

Revert "Revert "[RTM] Enh/derivatives""

Dockerfile

@@ -43,7 +43,7 @@ RUN rm -rf /usr/local/miniconda/lib/python*/site-packages/nipype* && \
     conda install -y pandas && \
     python -c "from matplotlib import font_manager"
 
-RUN pip install -e git+https://github.com/nipy/nipype.git@8ddca5a03fcad26887c862dc23c82ef23f2ee506#egg=nipype
+RUN pip install -e git+https://github.com/nipy/nipype.git@17e31abfd0a6a6b64c8c84586916bd463608e4b9#egg=nipype
 RUN pip install -e git+https://github.com/poldracklab/niworkflows.git@74760e479e79eec454e32df1e06e9f99b1908ae0#egg=niworkflows
 
 RUN mkdir /niworkflows_data

fmriprep/interfaces/bids.py

@@ -84,6 +84,7 @@ class DerivativesDataSinkInputSpec(BaseInterfaceInputSpec):
                              desc='the object to be saved')
     source_file = File(exists=True, mandatory=True, desc='the input func file')
     suffix = traits.Str('', mandatory=True, desc='suffix appended to source_file')
+    extra_values = traits.List(traits.Str)
 
 class DerivativesDataSinkOutputSpec(TraitedSpec):
     out_file = OutputMultiPath(File(exists=True, desc='written file path'))
@@ -132,7 +133,7 @@ def _run_interface(self, runtime):
         base_fname = op.join(out_path, fname)
 
         formatstr = '{bname}_{suffix}{ext}'
-        if len(self.inputs.in_file) > 1:
+        if len(self.inputs.in_file) > 1 and not isdefined(self.inputs.extra_values):
             formatstr = '{bname}_{suffix}{i:04d}{ext}'
 
 
@@ -142,6 +143,8 @@ def _run_interface(self, runtime):
                 suffix=self.inputs.suffix,
                 i=i,
                 ext=ext)
+            if isdefined(self.inputs.extra_values):
+                out_file = out_file.format(extra_value=self.inputs.extra_values[i])
             self._results['out_file'].append(out_file)
             copy(self.inputs.in_file[i], out_file)
 

fmriprep/interfaces/utils.py

@@ -97,7 +97,7 @@ def _tsv_format(translations, rot_angles, fmt='confounds'):
     elif fmt == 'confounds':
         out_file = op.abspath('movpar.tsv')
         np.savetxt(out_file, parameters,
-                   header='Motion parameters: X, Y, Z, Rx, Ry, Rz',
+                   header='X\tY\tZ\tRotX\tRotY\tRotZ',
                    delimiter='\t')
     else:
         raise NotImplementedError

fmriprep/workflows/anatomical.py

@@ -216,39 +216,40 @@ def t1w_preprocessing(name='t1w_preprocessing', settings=None):
     # Write corrected file in the designated output dir
     ds_t1_bias = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='inu'),
+                            suffix='preproc'),
         name='DerivT1_inu'
     )
     ds_t1_seg = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='inu_seg'),
+                            suffix='dtissue'),
         name='DerivT1_seg'
     )
     ds_mask = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='bmask'),
+                            suffix='brainmask'),
         name='DerivT1_mask'
     )
     ds_t1_mni = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='mni'),
+                            suffix='space-MNI152NLin2009cAsym_preproc'),
         name='DerivT1w_MNI'
     )
     ds_t1_mni_aff = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='mni_affine'),
+                            suffix='target-MNI152NLin2009cAsym_affine'),
         name='DerivT1w_MNI_affine'
     )
     ds_bmask_mni = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='bmask_mni'),
+                            suffix='space-MNI152NLin2009cAsym_brainmask'),
         name='DerivT1_Mask_MNI'
     )
     ds_tpms_mni = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='tpm_mni'),
+                            suffix='space-MNI152NLin2009cAsym_class-{extra_value}_probtissue'),
         name='DerivT1_TPMs_MNI'
     )
+    ds_tpms_mni.inputs.extra_values = ['CSF', 'GM', 'WM']
 
     if settings.get('debug', False):
         workflow.connect([
@@ -257,7 +258,7 @@ def t1w_preprocessing(name='t1w_preprocessing', settings=None):
     else:
         ds_t1_mni_warp = pe.Node(
             DerivativesDataSink(base_directory=settings['output_dir'],
-                                suffix='mni_warp'), name='DerivT1w_MNI_warp')
+                                suffix='target-MNI152NLin2009cAsym_warp'), name='mni_warp')
 
         def _get_aff(inlist):
             return inlist[:-1]

fmriprep/workflows/base.py

@@ -123,7 +123,10 @@ def wf_ds054_type(subject_data, settings, name='fMRI_prep'):
         (t1_to_epi_transforms, confounds_wf, [('out_file', 'inputnode.t1_transform')]),
 
         (hmcwf, confounds_wf, [('outputnode.movpar_file', 'inputnode.movpar_file'),
-                               ('outputnode.epi_mean', 'inputnode.reference_image')]),
+                               ('outputnode.epi_mean', 'inputnode.reference_image'),
+                               ('outputnode.motion_confounds_file',
+                                'inputnode.motion_confounds_file'),
+                               ('inputnode.epi', 'inputnode.source_file')]),
         (epiunwarp_wf, confounds_wf, [('outputnode.epi_mask', 'inputnode.epi_mask'),
                                       ('outputnode.epi_unwarp', 'inputnode.fmri_file')]),
         (t1w_pre, confounds_wf, [('outputnode.t1_seg', 'inputnode.t1_seg')]),
@@ -174,6 +177,7 @@ def wf_ds005_type(subject_data, settings, name='fMRI_prep'):
 
     workflow.connect([
         (bidssrc, t1w_pre, [('t1w', 'inputnode.t1w')]),
+        (bidssrc, epi_2_t1, [('t1w', 'inputnode.t1w')]),
         (hmcwf, epi_2_t1, [('inputnode.epi', 'inputnode.epi')]),
         (hmcwf, epi_2_t1, [('outputnode.epi_mean', 'inputnode.epi_mean')]),
         (t1w_pre, epi_2_t1, [('outputnode.t1_brain', 'inputnode.t1_brain'),
@@ -183,8 +187,10 @@ def wf_ds005_type(subject_data, settings, name='fMRI_prep'):
         (hmcwf, confounds_wf, [('outputnode.movpar_file', 'inputnode.movpar_file'),
                                ('outputnode.epi_brain', 'inputnode.fmri_file'),
                                ('outputnode.epi_mean', 'inputnode.reference_image'),
-                               ('outputnode.epi_mask', 'inputnode.epi_mask')]),
+                               ('outputnode.epi_mask', 'inputnode.epi_mask'),
+                               ('outputnode.motion_confounds_file', 'inputnode.motion_confounds_file')]),
         (epi_2_t1, confounds_wf, [('outputnode.mat_t1_to_epi', 'inputnode.t1_transform')]),
+        (hmcwf, confounds_wf, [('inputnode.epi', 'inputnode.source_file')]),
 
         (hmcwf, epi_mni_trans_wf, [('inputnode.epi', 'inputnode.epi')]),
         (epi_2_t1, epi_mni_trans_wf, [('outputnode.itk_epi_to_t1', 'inputnode.itk_epi_to_t1')]),

fmriprep/workflows/confounds.py

@@ -9,7 +9,7 @@
 from fmriprep.interfaces import mask
 from fmriprep import interfaces
 
-FAST_DEFAULT_SEGS = ['CSF', 'gray matter', 'white matter']
+FAST_DEFAULT_SEGS = ['CSF', 'GrayMatter', 'WhiteMatter']
 
 
 def discover_wf(settings, name="ConfoundDiscoverer"):
@@ -27,7 +27,8 @@ def discover_wf(settings, name="ConfoundDiscoverer"):
 
     inputnode = pe.Node(utility.IdentityInterface(fields=['fmri_file', 'movpar_file', 't1_seg',
                                                           'epi_mask', 't1_transform',
-                                                          'reference_image']),
+                                                          'reference_image', 'motion_confounds_file',
+                                                          'source_file']),
                         name='inputnode')
     outputnode = pe.Node(utility.IdentityInterface(fields=['confounds_file']),
                          name='outputnode')
@@ -47,15 +48,16 @@ def discover_wf(settings, name="ConfoundDiscoverer"):
     # CompCor
     tcompcor = pe.Node(confounds.TCompCor(components_file='tcompcor.tsv'), name="tCompCor")
     acompcor_roi = pe.Node(mask.BinarizeSegmentation(
-        false_values=[FAST_DEFAULT_SEGS.index('gray matter') + 1, 0]),  # 0 denotes background
+        false_values=[FAST_DEFAULT_SEGS.index('GrayMatter') + 1, 0]),  # 0 denotes background
                            name="CalcaCompCorROI")
     acompcor = pe.Node(confounds.ACompCor(components_file='acompcor.tsv'), name="aCompCor")
 
     # misc utilities
     concat = pe.Node(utility.Function(function=_gather_confounds, input_names=['signals', 'dvars',
                                                                                'frame_displace',
                                                                                'tcompcor',
-                                                                               'acompcor'],
+                                                                               'acompcor',
+                                                                               'motion'],
                                       output_names=['combined_out']),
                      name="ConcatConfounds")
     ds_confounds = pe.Node(interfaces.DerivativesDataSink(base_directory=settings['output_dir'],
@@ -89,38 +91,43 @@ def discover_wf(settings, name="ConfoundDiscoverer"):
         (frame_displace, concat, [('out_file', 'frame_displace')]),
         (tcompcor, concat, [('components_file', 'tcompcor')]),
         (acompcor, concat, [('components_file', 'acompcor')]),
+        (inputnode, concat, [('motion_confounds_file', 'motion')]),
 
         (concat, outputnode, [('combined_out', 'confounds_file')]),
 
         # print stuff in derivatives
         (concat, ds_confounds, [('combined_out', 'in_file')]),
-        (inputnode, ds_confounds, [('fmri_file', 'source_file')])
+        (inputnode, ds_confounds, [('source_file', 'source_file')])
     ])
 
     return workflow
 
 
-def _gather_confounds(signals=None, dvars=None, frame_displace=None, tcompcor=None, acompcor=None):
+def _gather_confounds(signals=None, dvars=None, frame_displace=None,
+                      tcompcor=None, acompcor=None, motion=None):
     ''' load confounds from the filenames, concatenate together horizontally, and re-save '''
     import pandas as pd
     import os.path as op
 
     def less_breakable(a_string):
         ''' hardens the string to different envs (i.e. case insensitive, no whitespace, '#' '''
-        return ''.join(a_string.split()).lower().strip('#')
+        return ''.join(a_string.split()).strip('#')
 
-    all_files = [confound for confound in [signals, dvars, frame_displace, tcompcor, acompcor]
+    all_files = [confound for confound in [signals, dvars, frame_displace,
+                                           tcompcor, acompcor, motion]
                  if confound is not None]
 
     confounds_data = pd.DataFrame()
     for file_name in all_files:  # assumes they all have headings already
         new = pd.read_csv(file_name, sep="\t")
         for column_name in new.columns:
-            new.rename(columns={column_name: less_breakable(column_name)}, inplace=True)
+            new.rename(columns={column_name: less_breakable(column_name)},
+                       inplace=True)
         confounds_data = pd.concat((confounds_data, new), axis=1)
 
     combined_out = op.abspath('confounds.tsv')
-    confounds_data.to_csv(combined_out, sep=str("\t"), index=False)
+    confounds_data.to_csv(combined_out, sep=str("\t"), index=False,
+                          na_rep="n/a")
 
     return combined_out
 

fmriprep/workflows/epi.py

@@ -37,7 +37,8 @@ def epi_hmc(name='EPI_HMC', settings=None):
     workflow = pe.Workflow(name=name)
     inputnode = pe.Node(niu.IdentityInterface(fields=['epi']), name='inputnode')
     outputnode = pe.Node(niu.IdentityInterface(
-        fields=['epi_brain', 'xforms', 'epi_mask', 'epi_mean', 'movpar_file']), name='outputnode')
+        fields=['epi_brain', 'xforms', 'epi_mask', 'epi_mean', 'movpar_file',
+                'motion_confounds_file']), name='outputnode')
 
     # Head motion correction (hmc)
     hmc = pe.Node(fsl.MCFLIRT(
@@ -69,35 +70,24 @@ def epi_hmc(name='EPI_HMC', settings=None):
                                ('rot_angles', 'rot_angles')]),
         (hmc, bet_hmc, [('mean_img', 'in_file')]),
         (hmc, avscale, [('mean_img', 'ref_file')]),
+        (avs_format, outputnode, [('out_file', 'motion_confounds_file')]),
         (bet_hmc, outputnode, [('mask_file', 'epi_mask'),
                                ('out_file', 'epi_mean')]),
     ])
 
     # Write corrected file in the designated output dir
     ds_hmc = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc'),
+                            suffix='preproc'),
         name='DerivativesHMC'
     )
 
-    ds_mats = pe.Node(
-        DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc'),
-        name='DerivativesHMCmats'
-    )
-
     ds_mask = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc_bmask'),
+                            suffix='brainmask'),
         name='DerivativesEPImask'
     )
 
-    ds_motion = pe.Node(
-        DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc'),
-        name='DerivativesParamsHMC'
-    )
-
     ds_betrpt = pe.Node(nio.DataSink(), name="BETRPTDS")
     ds_betrpt.inputs.base_directory = op.join(settings['output_dir'],
                                               'reports')
@@ -120,14 +110,10 @@ def epi_hmc(name='EPI_HMC', settings=None):
 
     workflow.connect([
         (inputnode, ds_hmc, [('epi', 'source_file')]),
-        (inputnode, ds_mats, [('epi', 'source_file')]),
         (inputnode, ds_mask, [('epi', 'source_file')]),
-        (inputnode, ds_motion, [('epi', 'source_file')]),
         (inputnode, mean_epi_overlay_ds, [('epi', 'origin_file')]),
         (hmc, ds_hmc, [('out_file', 'in_file')]),
-        (hcm2itk, ds_mats, [('itk_transform', 'in_file')]),
         (bet_hmc, ds_mask, [('mask_file', 'in_file')]),
-        (avs_format, ds_motion, [('out_file', 'in_file')]),
         (hmc, mean_epi_stripped_overlay, [('mean_img', 'overlay_file')]),
         (bet_hmc, mean_epi_stripped_overlay, [('mask_file', 'in_file')]),
         (hmc, mean_epi_overlay_ds, [('mean_img', 'overlay_file')]),
@@ -148,7 +134,7 @@ def epi_mean_t1_registration(name='EPIMeanNormalization', settings=None):
     workflow = pe.Workflow(name=name)
     inputnode = pe.Node(
         niu.IdentityInterface(fields=['epi', 'epi_mean', 't1_brain',
-                                      't1_seg']),
+                                      't1_seg', 't1w']),
         name='inputnode'
     )
     outputnode = pe.Node(
@@ -187,21 +173,15 @@ def epi_mean_t1_registration(name='EPIMeanNormalization', settings=None):
     fsl2itk_inv = pe.Node(c3.C3dAffineTool(fsl2ras=True, itk_transform=True),
                           name='fsl2itk_inv')
 
-    # Write EPI mean in T1w space
-    ds_t1w = pe.Node(
-        DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc_t1'),
-        name='DerivHMC_T1w'
-    )
     # Write registrated file in the designated output dir
     ds_tfm_fwd = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='epi2t1w_affine'),
+                            suffix='target-T1w_affine'),
         name='DerivEPI_to_T1w_fwd'
     )
     ds_tfm_inv = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='t1w2epi_affine'),
+                            suffix='target-meanBOLD_affine'),
         name='DerivEPI_to_T1w_inv'
     )
 
@@ -229,11 +209,9 @@ def epi_mean_t1_registration(name='EPIMeanNormalization', settings=None):
         (fsl2itk_fwd, outputnode, [('itk_transform', 'itk_epi_to_t1')]),
         (fsl2itk_inv, outputnode, [('itk_transform', 'itk_t1_to_epi')]),
         (inputnode, ds_tfm_fwd, [('epi', 'source_file')]),
-        (inputnode, ds_tfm_inv, [('epi', 'source_file')]),
-        (inputnode, ds_t1w, [('epi', 'source_file')]),
+        (inputnode, ds_tfm_inv, [('t1w', 'source_file')]),
         (fsl2itk_fwd, ds_tfm_fwd, [('itk_transform', 'in_file')]),
         (fsl2itk_inv, ds_tfm_inv, [('itk_transform', 'in_file')]),
-        (flt_bbr, ds_t1w, [('out_file', 'in_file')]),
         (flt_bbr, ds_flt_bbr, [('out_report', 'flt_bbr_rpt')])
     ])
 
@@ -291,7 +269,7 @@ def epi_sbref_registration(settings, name='EPI_SBrefRegistration'):
 
     ds_sbref = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc_sbref'), name='DerivHMC_SBRef')
+                            suffix='preproc'), name='DerivHMC_SBRef')
 
     ds_flirtrpt = pe.Node(nio.DataSink(), name="FLIRTRPTDS")
     ds_flirtrpt.inputs.base_directory = op.join(settings['output_dir'],
@@ -391,12 +369,12 @@ def _aslist(in_value):
     # Write corrected file in the designated output dir
     ds_mni = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc_mni'),
+                            suffix='space-MNI152NLin2009cAsym_preproc'),
         name='DerivativesHMCMNI'
     )
     ds_mni_mask = pe.Node(
         DerivativesDataSink(base_directory=settings['output_dir'],
-                            suffix='hmc_mni_bmask'),
+                            suffix='space-MNI152NLin2009cAsym_brainmask'),
         name='DerivativesHMCMNImask'
     )
 

test/workflows/test_confounds.py

@@ -59,4 +59,4 @@ def test_gather_confounds(self, df_equality, mock_df, mock_csv_reader):
         confounds = pd.DataFrame({'a': [0.1], 'b': [0.2]})
 
         mock_df.assert_called_once_with(confounds, os.path.abspath("confounds.tsv"),
-                                        index=False, sep="\t")
+                                        na_rep='n/a', index=False, sep="\t")