[ENH] Inform about failed individual reports

mriqc/bin/mriqc_run.py

@@ -28,6 +28,7 @@ def main():
     from nipype.pipeline.engine import Workflow
     from mriqc.utils.bids import collect_bids_data
     from mriqc.workflows.core import build_workflow
+    from mriqc.reports.utils import check_reports
 
     parser = ArgumentParser(description='MRI Quality Control',
                             formatter_class=RawTextHelpFormatter)
@@ -238,6 +239,8 @@ def main():
 
             if not opts.dry_run:
                 workflow.run(**plugin_settings)
+                if check_reports(dataset, settings):
+                    MRIQC_LOG.warn('Some reports were not generated')
 
     # Set up group level
     if opts.analysis_level == 'group' or opts.participant_label is None:
@@ -263,7 +266,9 @@ def main():
 
             out_html = op.join(reports_dir, qctype[:4] + '_group.html')
             MRIQC_LOG.info('Summary CSV table for the %s data generated (%s)', qctype, out_csv)
-            group_html(out_csv, qctype, out_file=out_html)
+            group_html(out_csv, qctype,
+                       csv_failed=op.join(settings['output_dir'], 'failed_' + qctype + '.csv'),
+                       out_file=out_html)
             MRIQC_LOG.info('Group-%s report generated (%s)', qctype, out_html)
 
 if __name__ == '__main__':

mriqc/data/reports/group.html

@@ -16,6 +16,13 @@ <h2>Summary</h2>
 <ul class="simple">
 <li>Date and time: {{ timestamp }}.</li>
 <li>MRIQC version: {{ version }}.</li>
+{% if failed %}
+<li><span class="text-warning">Some individual reports failed</span>:
+{% for f in failed %}
+<span class="text-warning">{{ f }}</span>{% if loop.last %}.{% else %}, {% endif %}
+{% endfor %}
+</li>
+{% endif %}
 </ul>
 
 <script src="resources/boxplots.js" charset="utf-8"></script>

mriqc/data/reports/resources/boxplots.css

@@ -6,6 +6,11 @@
 body {
     font-family: helvetica;
 }
+
+.text-warning {
+    font-weight: bold;
+    color: red;
+}
 /*Primary Chart*/
 
 /*Nested divs for responsiveness*/
@@ -137,7 +142,7 @@ body {
 .chart-options  {
     min-width: 200px;
     font-size: 13px;
-    font-family: sans-serif;
+    font-family: helvetica;
 }
 .chart-options button {
     margin: 3px;

mriqc/interfaces/bids.py

@@ -18,11 +18,11 @@ class ReadSidecarJSONInputSpec(BaseInterfaceInputSpec):
 
 class ReadSidecarJSONOutputSpec(TraitedSpec):
     subject_id = traits.Str()
-    session_id = traits.Str()
-    task_id = traits.Str()
-    acq_id = traits.Str()
-    rec_id = traits.Str()
-    run_id = traits.Str()
+    session_id = traits.Either(None, traits.Str())
+    task_id = traits.Either(None, traits.Str())
+    acq_id = traits.Either(None, traits.Str())
+    rec_id = traits.Either(None, traits.Str())
+    run_id = traits.Either(None, traits.Str())
     out_dict = traits.Dict()
 
 class ReadSidecarJSON(MRIQCBaseInterface):
@@ -41,10 +41,7 @@ def _run_interface(self, runtime):
         outputs = self.expr.search(op.basename(self.inputs.in_file)).groupdict()
 
         for key in output_keys:
-            if outputs.get(key) is not None:
-                self._results[key] = outputs.get(key)
-            else:
-                self._results[key] = 'default_' + key.split('_')[0]
+            self._results[key] = outputs.get(key)
 
         if isdefined(self.inputs.fields) and self.inputs.fields:
             for fname in self.inputs.fields:

mriqc/interfaces/viz.py

@@ -7,7 +7,6 @@
 # @Date:   2016-01-05 11:29:40
 # @Email:  code@oscaresteban.es
 # @Last modified by:   oesteban
-# @Last Modified time: 2016-11-14 10:15:37
 """ Visualization interfaces """
 from __future__ import print_function, division, absolute_import, unicode_literals
 
@@ -17,12 +16,13 @@
 from nipype.interfaces.base import (BaseInterface, traits, TraitedSpec, File,
                                     OutputMultiPath, BaseInterfaceInputSpec,
                                     isdefined)
-from io import open
+from io import open # pylint: disable=W0622
 from mriqc.utils.misc import split_ext
-from mriqc.interfaces.viz_utils import (plot_mosaic_helper, plot_fd, plot_segmentation)
+from mriqc.interfaces.viz_utils import (plot_mosaic_helper, plot_segmentation)
 from mriqc.interfaces.base import MRIQCBaseInterface
 from matplotlib.cm import get_cmap
 
+
 class PlotContoursInputSpec(BaseInterfaceInputSpec):
     in_file = File(exists=True, mandatory=True,
                    desc='File to be plotted')
@@ -74,18 +74,14 @@ def _run_interface(self, runtime):
         return runtime
 
 
-class PlotMosaicInputSpec(BaseInterfaceInputSpec):
+class PlotBaseInputSpec(BaseInterfaceInputSpec):
     in_file = File(exists=True, mandatory=True,
                    desc='File to be plotted')
     subject_id = traits.Str(mandatory=True, desc='subject id')
-    session_id = traits.Str(mandatory=True, desc='session id')
-    task_id = traits.Str(desc='task id')
-    run_id = traits.Str(mandatory=True, desc='run id')
-    task_id = traits.Str(desc='task id')
-    title = traits.Str('Volume', usedefault=True,
-                       desc='modality name to be prepended')
-    bbox_mask_file = File(exists=True, desc='brain mask')
-    only_noise = traits.Bool(False, desc='plot only noise')
+    session_id = traits.Either(None, traits.Str(desc='session id'))
+    task_id = traits.Either(None, traits.Str(desc='task id'))
+    run_id = traits.Either(None, traits.Str(desc='run id'))
+    title = traits.Str(desc='a title string for the plot')
 
     figsize = traits.Tuple(
         (11.69, 8.27), traits.Float, traits.Float, usedefault=True,
@@ -94,12 +90,34 @@ class PlotMosaicInputSpec(BaseInterfaceInputSpec):
     out_file = File('mosaic.svg', usedefault=True, desc='output file name')
     cmap = traits.Str('Greys_r', usedefault=True)
 
+class PlotBase(MRIQCBaseInterface):
+    def _get_title(self):
+        title = None
+        if isdefined(self.inputs.title):
+            title = self.inputs.title
+
+            elements = []
+            for k in ['session_id', 'task_id', 'run_id']:
+                value = getattr(self.inputs, k, None)
+                if isdefined(value) and value is not None and value.lower() != 'none':
+                    elements.append(value)
+
+            if elements:
+                title += ' (%s).' % ', '.join(elements)
+
+        return title
+
+
+class PlotMosaicInputSpec(PlotBaseInputSpec):
+    bbox_mask_file = File(exists=True, desc='brain mask')
+    only_noise = traits.Bool(False, desc='plot only noise')
+
 
 class PlotMosaicOutputSpec(TraitedSpec):
     out_file = File(exists=True, desc='output pdf file')
 
 
-class PlotMosaic(MRIQCBaseInterface):
+class PlotMosaic(PlotBase):
 
     """
     Plots slices of a 3D volume into a pdf file
@@ -119,23 +137,23 @@ def _run_interface(self, runtime):
             task_id=self.inputs.task_id,
             run_id=self.inputs.run_id,
             out_file=self.inputs.out_file,
-            title=self.inputs.title,
+            title=self._get_title(),
             only_plot_noise=self.inputs.only_noise,
             bbox_mask_file=mask,
             cmap=get_cmap(self.inputs.cmap))
         self._results['out_file'] = op.abspath(self.inputs.out_file)
         return runtime
 
 
-class PlotSpikesInputSpec(PlotMosaicInputSpec):
+class PlotSpikesInputSpec(PlotBaseInputSpec):
     in_spikes = File(exists=True, mandatory=True, desc='tsv file of spikes')
 
 
 class PlotSpikesOutputSpec(TraitedSpec):
     out_file = File(exists=True, desc='output svg file')
 
 
-class PlotSpikes(MRIQCBaseInterface):
+class PlotSpikes(PlotBase):
     """
     Plot slices of a dataset with spikes
     """
@@ -184,50 +202,9 @@ def _run_interface(self, runtime):
             self.inputs.session_id,
             self.inputs.run_id,
             out_file,
-            title=self.inputs.title,
+            title=self._get_title(),
             cmap=get_cmap(self.inputs.cmap),
             plot_sagittal=False,
             only_plot_noise=False,
             labels=labels)
         return runtime
-
-
-class PlotFDInputSpec(BaseInterfaceInputSpec):
-    in_file = File(exists=True, mandatory=True,
-                   desc='File to be plotted')
-    fd_radius = traits.Float(80., mandatory=True, usedefault=True,
-                             desc='Radius to compute power of FD')
-    figsize = traits.Tuple(
-        (8.27, 3.0), traits.Float, traits.Float, usedefault=True,
-        desc='Figure size')
-    dpi = traits.Int(300, usedefault=True, desc='Desired DPI of figure')
-    out_file = File('fd_power_2012.pdf', usedefault=True, desc='output file name')
-
-
-class PlotFDOutputSpec(TraitedSpec):
-    out_file = File(exists=True, desc='output pdf file')
-
-
-class PlotFD(MRIQCBaseInterface):
-    """
-    Plots the frame displacement of a dataset
-    """
-
-    input_spec = PlotFDInputSpec
-    output_spec = PlotFDOutputSpec
-
-    def _run_interface(self, runtime):
-
-        if isdefined(self.inputs.figsize):
-            fig = plot_fd(
-                self.inputs.in_file,
-                self.inputs.fd_radius,
-                figsize=self.inputs.figsize)
-        else:
-            fig = plot_fd(self.inputs.in_file,
-                          self.inputs.fd_radius)
-
-        fig.savefig(self.inputs.out_file, dpi=float(self.inputs.dpi))
-
-        self._results['out_file'] = op.abspath(self.inputs.out_file)
-        return runtime

mriqc/reports/group.py

@@ -18,7 +18,7 @@
 MRIQC_REPORT_LOG = logging.getLogger('mriqc.report')
 MRIQC_REPORT_LOG.setLevel(logging.INFO)
 
-def gen_html(csv_file, qctype, out_file=None):
+def gen_html(csv_file, qctype, csv_failed=None, out_file=None):
     import os.path as op
     from os import remove
     from shutil import copy, rmtree
@@ -84,16 +84,31 @@ def gen_html(csv_file, qctype, out_file=None):
     dataframe = pd.read_csv(csv_file, index_col=False)
 
     # format participant labels
-    formatter = lambda row: '{subject_id}_ses-{session_id}_run-{run_id}'.format(**row)
     id_labels = ['subject_id', 'session_id', 'run_id']
     if qctype.startswith('func'):
-        formatter = lambda row: ('{subject_id}_ses-{session_id}_task-{task_id}'
-                                 '_run-{run_id}').format(**row)
         id_labels.insert(2, 'task_id')
 
-    dataframe['label'] = dataframe[id_labels].apply(formatter, axis=1)
+    def myfmt(row, cols):
+        crow = [row[k] for k in cols if pd.notnull(row[k])]
+        return '_'.join(crow)
+
+    dataframe['label'] = dataframe[id_labels].apply(myfmt, args=(id_labels,), axis=1)
     nPart = len(dataframe)
 
+    failed = None
+    if csv_failed is not None and op.isfile(csv_failed):
+        MRIQC_REPORT_LOG.warn('Found failed-workflows table "%s"', csv_failed)
+        failed_df = pd.read_csv(csv_failed, index_col=False)
+        cols = list(set(id_labels) & set(failed_df.columns.ravel().tolist()))
+
+        try:
+            failed_df = failed_df.sort_values(by=cols)
+        except AttributeError:
+            #pylint: disable=E1101
+            failed_df = failed_df.sort(columns=cols)
+
+        failed = failed_df[cols].apply(myfmt, args=(cols,), axis=1).ravel().tolist()
+
     csv_groups = []
     for group, units in QCGROUPS[qctype[:4]]:
         dfdict = {'iqm': [], 'value': [], 'label': [], 'units': []}
@@ -119,7 +134,7 @@ def gen_html(csv_file, qctype, out_file=None):
             'timestamp': datetime.datetime.now().strftime("%Y-%m-%d, %H:%M"),
             'version': ver,
             'csv_groups': csv_groups,
-
+            'failed': failed
         }, out_file)
 
     res_folder = op.join(op.dirname(out_file), 'resources')

mriqc/reports/individual.py

@@ -12,7 +12,7 @@
 from __future__ import print_function, division, absolute_import, unicode_literals
 
 def individual_html(in_iqms, in_metadata=None, in_plots=None, exclude_index=0,
-                    wf_details=None, metadata=None):
+                    wf_details=None):
     import os.path as op  #pylint: disable=W0404
     import datetime
     import re
@@ -47,13 +47,15 @@ def individual_html(in_iqms, in_metadata=None, in_plots=None, exclude_index=0,
 
             svg_files.append('\n'.join(svg_lines_corrected))
 
-    sub_id = iqms_dict.pop('subject_id')
-    ses_id = iqms_dict.pop('session_id')
-    task_id = iqms_dict.pop('task_id', None)
-    run_id = iqms_dict.pop('run_id')
-    qctype = iqms_dict.pop('qc_type')
-    if qctype == 'anat':
-        qctype = 'anatomical'
+    name_els = [iqms_dict.pop(k, None) for k in [
+        'qc_type', 'subject_id', 'session_id', 'task_id', 'run_id']]
+    if not name_els[1].startswith('sub-'):
+        name_els[1] = 'sub-' + name_els[1]
+
+    MRIQC_REPORT_LOG.info('Elements %s', [el for el in name_els if el is not None])
+
+    if name_els[0].startswith('anat'):
+        name_els[0] = 'anatomical'
         msk_vals = []
         for k in ['snr_d_csf', 'snr_d_gm', 'snr_d_wm', 'fber']:
             elements = k.split('_')
@@ -70,28 +72,25 @@ def individual_html(in_iqms, in_metadata=None, in_plots=None, exclude_index=0,
                                    'the original file could be masked</span>.')
             else:
                 wf_details[-1] += '.'
-        out_file = op.abspath('{}_sub-{}_ses-{}_run-{}_report.html'.format(
-            qctype, sub_id[4:] if sub_id.startswith('sub-') else sub_id,
-            ses_id, run_id))
+    elif name_els[0].startswith('func'):
+        name_els[0] = 'functional'
+    else:
+        RuntimeError('Unknown QC type "%s"' % name_els[0])
 
-    if qctype == 'func':
-        qctype = 'functional'
 
-        out_file = op.abspath('{}_sub-{}_ses-{}_task-{}_run-{}_report.html'.format(
-            qctype, sub_id[4:] if sub_id.startswith('sub-') else sub_id,
-            ses_id, task_id, run_id))
+    out_file = op.abspath('_'.join([el for el in name_els if el is not None]) + '_report.html')
 
     tpl = IndividualTemplate()
     tpl.generate_conf({
-            'qctype': qctype,
-            'sub_id': sub_id,
+            'qctype': name_els[0],
+            'sub_id': name_els[1][4:],
             'timestamp': datetime.datetime.now().strftime("%Y-%m-%d, %H:%M"),
             'version': ver,
-            'imparams': iqms2html(iqms_dict),
+            'imparams': iqms2html(iqms_dict, 'iqms-table'),
             'svg_files': svg_files,
             'exclude_index': exclude_index,
             'workflow_details': wf_details,
-            'metadata': iqms2html(in_metadata),
+            'metadata': iqms2html(in_metadata, 'metadata-table'),
         }, out_file)
 
     MRIQC_REPORT_LOG.info('Generated individual log (%s)', out_file)