Merge pull request #1154 from nipreps/fix/drop-legacy-bids-querying

ENH: Implement BIDS filters file and drop legacy BIDS querying

mriqc/cli/parser.py

@@ -172,6 +172,12 @@ def _bids_filter(value):
         help="A space delimited list of participant identifiers or a single "
         "identifier (the sub- prefix can be removed).",
     )
+    g_bids.add_argument(
+        '--bids-filter-file', action='store', type=Path, metavar='PATH',
+        help='a JSON file describing custom BIDS input filter using pybids '
+             '{<suffix>:{<entity>:<filter>,...},...} '
+             '(https://github.com/bids-standard/pybids/blob/master/bids/layout/config/bids.json)'
+    )
     g_bids.add_argument(
         "--session-id",
         action="store",
@@ -184,7 +190,7 @@ def _bids_filter(value):
         action="store",
         type=int,
         nargs="*",
-        help="Filter input dataset by run ID (only integer run IDs are valid).",
+        help="DEPRECATED - This argument will be disabled. Use ``--bids-filter-file`` instead.",
     )
     g_bids.add_argument(
         "--task-id",
@@ -447,8 +453,10 @@ def parse_args(args=None, namespace=None):
     """Parse args and run further checks on the command line."""
     from logging import DEBUG
     from contextlib import suppress
+    from json import loads
+    from pprint import pformat
 
-    from mriqc.utils.bids import collect_bids_data
+    from niworkflows.utils.bids import collect_data, DEFAULT_BIDS_QUERIES
 
     parser = _build_parser()
     opts = parser.parse_args(args, namespace)
@@ -469,6 +477,10 @@ def parse_args(args=None, namespace=None):
                 "nprocs", config.nipype.nprocs
             )
 
+    # Load BIDS filters
+    if opts.bids_filter_file:
+        config.execution.bids_filters = loads(opts.bids_filter_file.read_text())
+
     bids_dir = config.execution.bids_dir
     output_dir = config.execution.output_dir
     work_dir = config.execution.work_dir
@@ -520,33 +532,40 @@ def parse_args(args=None, namespace=None):
     config.workflow.analysis_level = list(analysis_level)
 
     # List of files to be run
-    bids_filters = {
-        "participant_label": config.execution.participant_label,
-        "session": config.execution.session_id,
-        "run": config.execution.run_id,
-        "task": config.execution.task_id,
-        "bids_type": config.execution.modalities,
-    }
-    config.workflow.inputs = {
-        mod: files
-        for mod, files in collect_bids_data(
-            config.execution.layout, **bids_filters
-        ).items()
-        if files
+    lc_modalities = [mod.lower() for mod in config.execution.modalities]
+    bids_dataset, _ = collect_data(
+        config.execution.layout,
+        config.execution.participant_label,
+        session_id=config.execution.session_id,
+        task=config.execution.task_id,
+        group_echos=False,
+        bids_filters=config.execution.bids_filters,
+        queries={mod: DEFAULT_BIDS_QUERIES[mod] for mod in lc_modalities}
+    )
+
+    # Drop empty queries
+    bids_dataset = {
+        mod: files for mod, files in bids_dataset.items() if files
     }
+    config.workflow.inputs = bids_dataset
 
     # Check the query is not empty
     if not list(config.workflow.inputs.values()):
-        _j = "\n *"
+        ffile = (
+            "(--bids-filter-file was not set)" if not opts.bids_filter_file
+            else f"(with '--bids-filter-file {opts.bids_filter_file}')"
+        )
         parser.error(
             f"""\
 Querying BIDS dataset at <{config.execution.bids_dir}> got an empty result.
-Please, check out your currently set filters:
-{_j.join([''] + [': '.join((k, str(v))) for k, v in bids_filters.items()])}"""
+Please, check out your currently set filters {ffile}:
+{pformat(config.execution.bids_filters, indent=2, width=99)}"""
         )
 
     # Check no DWI or others are sneaked into MRIQC
-    unknown_mods = set(config.workflow.inputs.keys()) - set(config.SUPPORTED_SUFFIXES)
+    unknown_mods = set(config.workflow.inputs.keys()) - set(
+        suffix.lower() for suffix in config.SUPPORTED_SUFFIXES
+    )
     if unknown_mods:
         parser.error(
             "MRIQC is unable to process the following modalities: "

mriqc/config.py

@@ -361,6 +361,8 @@ class execution(_Config):
     """Wipe out previously existing BIDS indexing caches, forcing re-indexing."""
     bids_description_hash = None
     """Checksum (SHA256) of the ``dataset_description.json`` of the BIDS dataset."""
+    bids_filters = None
+    """A dictionary describing custom BIDS input filter using PyBIDS."""
     cwd = os.getcwd()
     """Current working directory."""
     debug = False
@@ -436,6 +438,15 @@ class execution(_Config):
     @classmethod
     def init(cls):
         """Create a new BIDS Layout accessible with :attr:`~execution.layout`."""
+
+        if cls.bids_filters is None:
+            cls.bids_filters = {}
+
+        # Process --run-id if the argument was provided
+        if cls.run_id:
+            for mod in cls.modalities:
+                cls.bids_filters.setdefault(mod.lower(), {})["run"] = cls.run_id
+
         if cls._layout is None:
             import re
             from bids.layout.index import BIDSLayoutIndexer

mriqc/utils/__init__.py

@@ -1,28 +0,0 @@
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
-# vi: set ft=python sts=4 ts=4 sw=4 et:
-#
-# Copyright 2021 The NiPreps Developers <nipreps@gmail.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# We support and encourage derived works from this project, please read
-# about our expectations at
-#
-#     https://www.nipreps.org/community/licensing/
-#
-"""Module utils.misc contains utilities."""
-from mriqc.utils.bids import collect_bids_data
-
-__all__ = [
-    "collect_bids_data",
-]

mriqc/utils/bids.py

@@ -23,49 +23,11 @@
 """PyBIDS tooling."""
 import json
 import os
-from collections import defaultdict
 from pathlib import Path
 
-from bids.utils import listify
-
 DOI = "https://doi.org/10.1371/journal.pone.0184661"
 
 
-def collect_bids_data(
-    layout,
-    bids_type,
-    participant_label=None,
-    session=None,
-    run=None,
-    task=None,
-):
-    """Get files in dataset"""
-
-    basequery = {
-        "subject": participant_label,
-        "session": session,
-        "task": task,
-        "run": run,
-        "datatype": "func",
-        "return_type": "file",
-        "extension": ["nii", "nii.gz"],
-    }
-    # Filter empty lists, strings, zero runs, and Nones
-    basequery = {k: v for k, v in basequery.items() if v}
-
-    # Start querying
-    imaging_data = defaultdict(list, {})
-    for btype in listify(bids_type):
-        _entities = basequery.copy()
-        _entities["suffix"] = btype
-        if btype in ("T1w", "T2w", "dwi"):
-            _entities["datatype"] = "dwi" if btype == "dwi" else "anat"
-            _entities.pop("task", None)
-        imaging_data[btype] = layout.get(**_entities)
-
-    return imaging_data
-
-
 def write_bidsignore(deriv_dir):
     from mriqc.config import SUPPORTED_SUFFIXES
     bids_ignore = [

mriqc/workflows/anatomical/base.py

@@ -87,7 +87,7 @@ def anat_qc_workflow(name="anatMRIQC"):
 
     """
 
-    dataset = config.workflow.inputs.get("T1w", []) + config.workflow.inputs.get("T2w", [])
+    dataset = config.workflow.inputs.get("t1w", []) + config.workflow.inputs.get("t2w", [])
 
     message = BUILDING_WORKFLOW.format(
         modality="anatomical",

mriqc/workflows/core.py

@@ -28,7 +28,7 @@
 from mriqc.workflows.functional.base import fmri_qc_workflow
 from mriqc.workflows.diffusion.base import dmri_qc_workflow
 
-ANATOMICAL_KEYS = "T1w", "T2w"
+ANATOMICAL_KEYS = "t1w", "t2w"
 FMRI_KEY = "bold"
 DMRI_KEY = "dwi"
 

pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "nipype ~= 1.4",
     "nireports ~= 23.1", 
     "nitransforms ~= 23.0", 
-    "niworkflows >= 1.7.7", 
+    "niworkflows @ git+https://github.com/nipreps/niworkflows.git@master", 
     "numpy ~=1.20",
     "pandas ~=1.0",
     "pybids >= 0.15.6",