Merge pull request #1343 from effigies/bids/derivatives_rc1

ENH: Conform confound regressor names to Derivatives RC2

.circleci/config.yml

@@ -354,10 +354,10 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - ds005-anat-v9-{{ .Branch }}-{{ epoch }}
-            - ds005-anat-v9-{{ .Branch }}
-            - ds005-anat-v9-master
-            - ds005-anat-v9-
+            - ds005-anat-v10-{{ .Branch }}-{{ epoch }}
+            - ds005-anat-v10-{{ .Branch }}
+            - ds005-anat-v10-master
+            - ds005-anat-v10-
       - run:
           name: Setting up test
           command: |
@@ -391,7 +391,7 @@ jobs:
                 --sloppy --write-graph --mem_mb 4096 \
                 --nthreads 2 --anat-only -vv
       - save_cache:
-         key: ds005-anat-v9-{{ .Branch }}-{{ epoch }}
+         key: ds005-anat-v10-{{ .Branch }}-{{ epoch }}
          paths:
             - /tmp/ds005/work
             - /tmp/ds005/derivatives/fmriprep

.circleci/ds005_outputs.txt

@@ -6,6 +6,8 @@ fmriprep/logs/CITATION.md
 fmriprep/logs/CITATION.tex
 fmriprep/sub-01
 fmriprep/sub-01/anat
+fmriprep/sub-01/anat/sub-01_desc-aparcaseg_dseg.nii.gz
+fmriprep/sub-01/anat/sub-01_desc-aseg_dseg.nii.gz
 fmriprep/sub-01/anat/sub-01_desc-brain_mask.nii.gz
 fmriprep/sub-01/anat/sub-01_desc-preproc_T1w.nii.gz
 fmriprep/sub-01/anat/sub-01_dseg.nii.gz
@@ -21,8 +23,6 @@ fmriprep/sub-01/anat/sub-01_hemi-R_inflated.surf.gii
 fmriprep/sub-01/anat/sub-01_hemi-R_midthickness.surf.gii
 fmriprep/sub-01/anat/sub-01_hemi-R_pial.surf.gii
 fmriprep/sub-01/anat/sub-01_hemi-R_smoothwm.surf.gii
-fmriprep/sub-01/anat/sub-01_label-aparcaseg_dseg.nii.gz
-fmriprep/sub-01/anat/sub-01_label-aseg_dseg.nii.gz
 fmriprep/sub-01/anat/sub-01_label-CSF_probseg.nii.gz
 fmriprep/sub-01/anat/sub-01_label-GM_probseg.nii.gz
 fmriprep/sub-01/anat/sub-01_label-WM_probseg.nii.gz
@@ -45,10 +45,10 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAs
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-smoothAROMAnonaggr_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_boldref.nii.gz
+fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-aparcaseg_dseg.nii.gz
+fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-aseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-brain_mask.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-preproc_bold.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_label-aparcaseg_dseg.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_label-aseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_AROMAnoiseICs.csv
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_bold.dtseries.json
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_bold.dtseries.nii
@@ -61,9 +61,9 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAs
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-smoothAROMAnonaggr_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_boldref.nii.gz
+fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_desc-aparcaseg_dseg.nii.gz
+fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_desc-aseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_desc-brain_mask.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_desc-preproc_bold.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_label-aparcaseg_dseg.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-T1w_label-aseg_dseg.nii.gz
 fmriprep/sub-01.html
 /tmp/ds005/derivatives

.circleci/ds005_partial_outputs.txt

@@ -6,6 +6,8 @@ fmriprep/logs/CITATION.md
 fmriprep/logs/CITATION.tex
 fmriprep/sub-01
 fmriprep/sub-01/anat
+fmriprep/sub-01/anat/sub-01_desc-aparcaseg_dseg.nii.gz
+fmriprep/sub-01/anat/sub-01_desc-aseg_dseg.nii.gz
 fmriprep/sub-01/anat/sub-01_desc-brain_mask.nii.gz
 fmriprep/sub-01/anat/sub-01_desc-preproc_T1w.nii.gz
 fmriprep/sub-01/anat/sub-01_dseg.nii.gz
@@ -21,8 +23,6 @@ fmriprep/sub-01/anat/sub-01_hemi-R_inflated.surf.gii
 fmriprep/sub-01/anat/sub-01_hemi-R_midthickness.surf.gii
 fmriprep/sub-01/anat/sub-01_hemi-R_pial.surf.gii
 fmriprep/sub-01/anat/sub-01_hemi-R_smoothwm.surf.gii
-fmriprep/sub-01/anat/sub-01_label-aparcaseg_dseg.nii.gz
-fmriprep/sub-01/anat/sub-01_label-aseg_dseg.nii.gz
 fmriprep/sub-01/anat/sub-01_label-CSF_probseg.nii.gz
 fmriprep/sub-01/anat/sub-01_label-GM_probseg.nii.gz
 fmriprep/sub-01/anat/sub-01_label-WM_probseg.nii.gz
@@ -45,9 +45,9 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAs
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-smoothAROMAnonaggr_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_boldref.nii.gz
+fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-aparcaseg_dseg.nii.gz
+fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-aseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-brain_mask.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_desc-preproc_bold.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_label-aparcaseg_dseg.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-T1w_label-aseg_dseg.nii.gz
 fmriprep/sub-01.html
 /tmp/ds005/derivatives_partial

fmriprep/interfaces/confounds.py

@@ -10,6 +10,7 @@
 
 """
 import os
+import re
 import shutil
 import numpy as np
 import pandas as pd
@@ -27,6 +28,7 @@
 class GatherConfoundsInputSpec(BaseInterfaceInputSpec):
     signals = File(exists=True, desc='input signals')
     dvars = File(exists=True, desc='file containing DVARS')
+    std_dvars = File(exists=True, desc='file containing standardized DVARS')
     fd = File(exists=True, desc='input framewise displacement')
     tcompcor = File(exists=True, desc='input tCompCorr')
     acompcor = File(exists=True, desc='input aCompCorr')
@@ -79,6 +81,7 @@ def _run_interface(self, runtime):
         combined_out, confounds_list = _gather_confounds(
             signals=self.inputs.signals,
             dvars=self.inputs.dvars,
+            std_dvars=self.inputs.std_dvars,
             fdisp=self.inputs.fd,
             tcompcor=self.inputs.tcompcor,
             acompcor=self.inputs.acompcor,
@@ -124,7 +127,7 @@ def _run_interface(self, runtime):
         return runtime
 
 
-def _gather_confounds(signals=None, dvars=None, fdisp=None,
+def _gather_confounds(signals=None, dvars=None, std_dvars=None, fdisp=None,
                       tcompcor=None, acompcor=None, cos_basis=None,
                       motion=None, aroma=None, newpath=None):
     """
@@ -134,16 +137,16 @@ def _gather_confounds(signals=None, dvars=None, fdisp=None,
     >>> from tempfile import TemporaryDirectory
     >>> tmpdir = TemporaryDirectory()
     >>> os.chdir(tmpdir.name)
-    >>> pd.DataFrame({'a': [0.1]}).to_csv('signals.tsv', index=False, na_rep='n/a')
-    >>> pd.DataFrame({'b': [0.2]}).to_csv('dvars.tsv', index=False, na_rep='n/a')
+    >>> pd.DataFrame({'Global Signal': [0.1]}).to_csv('signals.tsv', index=False, na_rep='n/a')
+    >>> pd.DataFrame({'stdDVARS': [0.2]}).to_csv('dvars.tsv', index=False, na_rep='n/a')
     >>> out_file, confound_list = _gather_confounds('signals.tsv', 'dvars.tsv')
     >>> confound_list
     ['Global signals', 'DVARS']
 
     >>> pd.read_csv(out_file, sep='\s+', index_col=None,
     ...             engine='python')  # doctest: +NORMALIZE_WHITESPACE
-         a    b
-    0  0.1  0.2
+       global_signal  std_dvars
+    0            0.1        0.2
     >>> tmpdir.cleanup()
 
 
@@ -153,6 +156,12 @@ def less_breakable(a_string):
         ''' hardens the string to different envs (i.e. case insensitive, no whitespace, '#' '''
         return ''.join(a_string.split()).strip('#')
 
+    # Taken from https://stackoverflow.com/questions/1175208/
+    # If we end up using it more than just here, probably worth pulling in a well-tested package
+    def camel_to_snake(name):
+        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+        return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
+
     def _adjust_indices(left_df, right_df):
         # This forces missing values to appear at the beggining of the DataFrame
         # instead of the end
@@ -167,6 +176,7 @@ def _adjust_indices(left_df, right_df):
     all_files = []
     confounds_list = []
     for confound, name in ((signals, 'Global signals'),
+                           (std_dvars, 'Standardized DVARS'),
                            (dvars, 'DVARS'),
                            (fdisp, 'Framewise displacement'),
                            (tcompcor, 'tCompCor'),
@@ -183,7 +193,7 @@ def _adjust_indices(left_df, right_df):
     for file_name in all_files:  # assumes they all have headings already
         new = pd.read_csv(file_name, sep="\t")
         for column_name in new.columns:
-            new.rename(columns={column_name: less_breakable(column_name)},
+            new.rename(columns={column_name: camel_to_snake(less_breakable(column_name))},
                        inplace=True)
 
         _adjust_indices(confounds_data, new)
@@ -245,7 +255,7 @@ def _get_ica_confounds(ica_out_dir, skip_vols, newpath=None):
     # add one to motion_ic_indices to match melodic report.
     aroma_confounds = os.path.join(newpath, "AROMAAggrCompAROMAConfounds.tsv")
     pd.DataFrame(aggr_confounds.T,
-                 columns=['AROMAAggrComp%02d' % (x + 1) for x in motion_ic_indices]).to_csv(
+                 columns=['aroma_motion_%02d' % (x + 1) for x in motion_ic_indices]).to_csv(
         aroma_confounds, sep="\t", index=None)
 
     return aroma_confounds, motion_ics_out, melodic_mix_out

fmriprep/workflows/anatomical.py

@@ -1313,10 +1313,10 @@ def init_anat_derivatives_wf(output_dir, output_spaces, template, freesurfer,
 
     if freesurfer:
         ds_t1_fsaseg = pe.Node(
-            DerivativesDataSink(base_directory=output_dir, suffix='label-aseg_dseg'),
+            DerivativesDataSink(base_directory=output_dir, desc='aseg', suffix='dseg'),
             name='ds_t1_fsaseg', run_without_submitting=True)
         ds_t1_fsparc = pe.Node(
-            DerivativesDataSink(base_directory=output_dir, suffix='label-aparcaseg_dseg'),
+            DerivativesDataSink(base_directory=output_dir, desc='aparcaseg', suffix='dseg'),
             name='ds_t1_fsparc', run_without_submitting=True)
         workflow.connect([
             (inputnode, lta_2_itk, [('t1_2_fsnative_forward_transform', 'in_lta')]),

fmriprep/workflows/bold/base.py

@@ -889,11 +889,11 @@ def init_func_derivatives_wf(output_dir, output_spaces, template, freesurfer,
 
     if freesurfer:
         ds_bold_aseg_t1 = pe.Node(DerivativesDataSink(
-            base_directory=output_dir, space='T1w', suffix='label-aseg_dseg'),
+            base_directory=output_dir, space='T1w', desc='aseg', suffix='dseg'),
             name='ds_bold_aseg_t1', run_without_submitting=True,
             mem_gb=DEFAULT_MEMORY_MIN_GB)
         ds_bold_aparc_t1 = pe.Node(DerivativesDataSink(
-            base_directory=output_dir,  space='T1w', suffix='label-aparcaseg_dseg'),
+            base_directory=output_dir,  space='T1w', desc='aparcaseg', suffix='dseg'),
             name='ds_bold_aparc_t1', run_without_submitting=True,
             mem_gb=DEFAULT_MEMORY_MIN_GB)
         workflow.connect([

fmriprep/workflows/bold/confounds.py

@@ -43,18 +43,17 @@ def init_bold_confs_wf(mem_gb, metadata, name="bold_confs_wf"):
 
     The following confounds are calculated, with column headings in parentheses:
 
-    #. Region-wise average signal (``CSF``, ``WhiteMatter``, ``GlobalSignal``)
-    #. DVARS - standard, nonstandard, and voxel-wise standard variants
-       (``stdDVARS``, ``non-stdDVARS``, ``vx-wisestdDVARS``)
-    #. Framewise displacement, based on MCFLIRT motion parameters
-       (``FramewiseDisplacement``)
-    #. Temporal CompCor (``tCompCorXX``)
-    #. Anatomical CompCor (``aCompCorXX``)
+    #. Region-wise average signal (``csf``, ``white_matter``, ``global_signal``)
+    #. DVARS - original and standardized variants (``dvars``, ``std_dvars``)
+    #. Framewise displacement, based on head-motion parameters
+       (``framewise_displacement``)
+    #. Temporal CompCor (``t_comp_cor_XX``)
+    #. Anatomical CompCor (``a_comp_cor_XX``)
     #. Cosine basis set for high-pass filtering w/ 0.008 Hz cut-off
-       (``CosineXX``)
-    #. Non-steady-state volumes (``NonSteadyStateXX``)
+       (``cosine_XX``)
+    #. Non-steady-state volumes (``non_steady_state_XX``)
     #. Estimated head-motion parameters, in mm and rad
-       (``X``, ``Y``, ``Z``, ``RotX``, ``RotY``, ``RotZ``)
+       (``trans_x``, ``trans_y``, ``trans_z``, ``rot_x``, ``rot_y``, ``rot_z``)
 
 
     Prior to estimating aCompCor and tCompCor, non-steady-state volumes are
@@ -172,20 +171,22 @@ def init_bold_confs_wf(mem_gb, metadata, name="bold_confs_wf"):
     tcc_msk = pe.Node(niu.Function(function=_maskroi), name='tcc_msk')
 
     # DVARS
-    dvars = pe.Node(nac.ComputeDVARS(save_all=True, remove_zerovariance=True),
+    dvars = pe.Node(nac.ComputeDVARS(save_nstd=True, save_std=True, remove_zerovariance=True),
                     name="dvars", mem_gb=mem_gb)
 
     # Frame displacement
     fdisp = pe.Node(nac.FramewiseDisplacement(parameter_source="SPM"),
                     name="fdisp", mem_gb=mem_gb)
 
     # a/t-CompCor
-    tcompcor = pe.Node(TCompCor(
-        components_file='tcompcor.tsv', pre_filter='cosine', save_pre_filter=True,
-        percentile_threshold=.05), name="tcompcor", mem_gb=mem_gb)
-
-    acompcor = pe.Node(ACompCor(
-        components_file='acompcor.tsv', pre_filter='cosine', save_pre_filter=True),
+    tcompcor = pe.Node(
+        TCompCor(components_file='tcompcor.tsv', header_prefix='t_comp_cor_', pre_filter='cosine',
+                 save_pre_filter=True, percentile_threshold=.05),
+        name="tcompcor", mem_gb=mem_gb)
+
+    acompcor = pe.Node(
+        ACompCor(components_file='acompcor.tsv', header_prefix='a_comp_cor_', pre_filter='cosine',
+                 save_pre_filter=True),
         name="acompcor", mem_gb=mem_gb)
 
     # Set TR if present
@@ -195,12 +196,19 @@ def init_bold_confs_wf(mem_gb, metadata, name="bold_confs_wf"):
 
     # Global and segment regressors
     mrg_lbl = pe.Node(niu.Merge(3), name='merge_rois', run_without_submitting=True)
-    signals = pe.Node(SignalExtraction(class_labels=["CSF", "WhiteMatter", "GlobalSignal"]),
+    signals = pe.Node(SignalExtraction(class_labels=["csf", "white_matter", "global_signal"]),
                       name="signals", mem_gb=mem_gb)
 
     # Arrange confounds
-    add_header = pe.Node(AddTSVHeader(columns=["X", "Y", "Z", "RotX", "RotY", "RotZ"]),
-                         name="add_header", mem_gb=0.01, run_without_submitting=True)
+    add_dvars_header = pe.Node(
+        AddTSVHeader(columns=["dvars"]),
+        name="add_dvars_header", mem_gb=0.01, run_without_submitting=True)
+    add_std_dvars_header = pe.Node(
+        AddTSVHeader(columns=["std_dvars"]),
+        name="add_std_dvars_header", mem_gb=0.01, run_without_submitting=True)
+    add_motion_headers = pe.Node(
+        AddTSVHeader(columns=["trans_x", "trans_y", "trans_z", "rot_x", "rot_y", "rot_z"]),
+        name="add_motion_headers", mem_gb=0.01, run_without_submitting=True)
     concat = pe.Node(GatherConfounds(), name="concat", mem_gb=0.01, run_without_submitting=True)
 
     # Generate reportlet
@@ -273,14 +281,17 @@ def _pick_wm(files):
         (mrg_lbl, signals, [('out', 'label_files')]),
 
         # Collate computed confounds together
-        (inputnode, add_header, [('movpar_file', 'in_file')]),
+        (inputnode, add_motion_headers, [('movpar_file', 'in_file')]),
+        (dvars, add_dvars_header, [('out_nstd', 'in_file')]),
+        (dvars, add_std_dvars_header, [('out_std', 'in_file')]),
         (signals, concat, [('out_file', 'signals')]),
-        (dvars, concat, [('out_all', 'dvars')]),
         (fdisp, concat, [('out_file', 'fd')]),
         (tcompcor, concat, [('components_file', 'tcompcor'),
                             ('pre_filter_file', 'cos_basis')]),
         (acompcor, concat, [('components_file', 'acompcor')]),
-        (add_header, concat, [('out_file', 'motion')]),
+        (add_motion_headers, concat, [('out_file', 'motion')]),
+        (add_dvars_header, concat, [('out_file', 'dvars')]),
+        (add_std_dvars_header, concat, [('out_file', 'std_dvars')]),
 
         # Set outputs
         (concat, outputnode, [('confounds_file', 'confounds_file')]),
@@ -356,11 +367,11 @@ def init_carpetplot_wf(mem_gb, metadata, name="bold_carpet_wf"):
     conf_plot = pe.Node(FMRISummary(
         tr=metadata['RepetitionTime'],
         confounds_list=[
-            ('GlobalSignal', None, 'GS'),
-            ('CSF', None, 'GSCSF'),
-            ('WhiteMatter', None, 'GSWM'),
-            ('stdDVARS', None, 'DVARS'),
-            ('FramewiseDisplacement', 'mm', 'FD')]),
+            ('global_signal', None, 'GS'),
+            ('csf', None, 'GSCSF'),
+            ('white_matter', None, 'GSWM'),
+            ('std_dvars', None, 'DVARS'),
+            ('framewise_displacement', 'mm', 'FD')]),
         name='conf_plot', mem_gb=mem_gb)
     ds_report_bold_conf = pe.Node(
         DerivativesDataSink(suffix='carpetplot'),