[FIX] Load compcor related confounds generated from fmriprep 21.x ser…

…ies (#3285) * [TEST] add test data for fmriprep 21.x series - refactor the test file loader - add test data and make sure the corret files are loaded - apply compcor loading suggestions from @askieslinger Still need to investigate if the actual behaviour is inline with the literature * ENH refactor compcor for 21.x.x the code should now work for both before and after 21.x update. * simply _check_compcor_method by using all five options as prefix checking keys * add tests to test_load_confounds_strategy.py::test_strategy_compcor and test_load_confounds.py::test_n_compcor to make sure the correct componenets are loaded * FIX pep8 * RF factorise `test_strategy_compcor` * shorten the `test_strategy_compcor` by factorising the check on different fmriprep versions * add an entry in `latest.rst` * PEP8 * FIX typo Co-authored-by: Gensollen <nicolas.gensollen@gmail.com> * FIX typo in function name... * fix tests * lint * test on both data inputs for most tests * refactor tests --------- Co-authored-by: Gensollen <nicolas.gensollen@gmail.com> Co-authored-by: Remi Gau <remi_gau@hotmail.com>
nilearn · Dec 14, 2023 · 2d6ff1c · 2d6ff1c
1 parent be0ad56
commit 2d6ff1c
Show file tree

Hide file tree

Showing 10 changed files with 379 additions and 107 deletions.
diff --git a/doc/changes/latest.rst b/doc/changes/latest.rst
@@ -14,6 +14,7 @@ NEW
 Fixes
 -----
 
+- :bdg-dark:`Code` Update the ``CompCor`` strategy in :func:`~interfaces.fmriprep.load_confounds` and :func:`~interfaces.fmriprep.load_confounds_strategy` to support ``fmriprep`` 21.x series and above. (:gh:`3285` by `Hao-Ting Wang`_).
 - :bdg-success:`API` :class:`~maskers.MultiNiftiMasker` can now call :meth:`~maskers.NiftiMasker.generate_report` which will generate a report for the first subject in the list of subjects (:gh:`4001` by `Yasmin Mzayek`_).
 - :bdg-dark:`Code` :func:`~image.clean_img` can now use kwargs ``clean__sample_mask`` argument to correctly reshape the nifti image to the dimensions of the mask in the output (:gh:`4051` by 'Mia Zwally`_).
 

diff --git a/nilearn/interfaces/fmriprep/data/test-v21_desc-confounds_timeseries.json b/nilearn/interfaces/fmriprep/data/test-v21_desc-confounds_timeseries.json
@@ -0,0 +1,134 @@
+{
+  "a_comp_cor_00": {
+    "CumulativeVarianceExplained": 0.2026280323,
+    "Mask": "combined",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 123.2000731257,
+    "VarianceExplained": 0.2026280323
+  },
+  "a_comp_cor_01": {
+    "CumulativeVarianceExplained": 0.3329213889,
+    "Mask": "combined",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 98.7921275175,
+    "VarianceExplained": 0.1302933566
+  },
+  "a_comp_cor_02": {
+    "CumulativeVarianceExplained": 0.4240879383,
+    "Mask": "combined",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 82.6378407134,
+    "VarianceExplained": 0.0911665494
+  },
+  "a_comp_cor_03": {
+    "CumulativeVarianceExplained": 0.4888242986,
+    "Mask": "combined",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 69.6362444247,
+    "VarianceExplained": 0.0647363603
+  },
+  "a_comp_cor_04": {
+    "CumulativeVarianceExplained": 0.5366018375,
+    "Mask": "combined",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 59.8236751211,
+    "VarianceExplained": 0.0477775389
+  },
+  "c_comp_cor_00": {
+    "CumulativeVarianceExplained": 0.2376671896,
+    "Mask": "CSF",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 100.4509741103,
+    "VarianceExplained": 0.2376671896
+  },
+  "c_comp_cor_01": {
+    "CumulativeVarianceExplained": 0.416993444,
+    "Mask": "CSF",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 87.2552317073,
+    "VarianceExplained": 0.1793262545
+  },
+  "c_comp_cor_02": {
+    "CumulativeVarianceExplained": 0.5132008054,
+    "Mask": "CSF",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 63.9107168937,
+    "VarianceExplained": 0.0962073614
+  },
+  "csf": {
+    "Method": "Mean"
+  },
+  "csf_wm": {
+    "Method": "Mean"
+  },
+  "global_signal": {
+    "Method": "Mean"
+  },
+  "t_comp_cor_00": {
+    "CumulativeVarianceExplained": 0.3071809951,
+    "Method": "tCompCor",
+    "Retained": true,
+    "SingularValue": 209.0973108093,
+    "VarianceExplained": 0.3071809951
+  },
+  "t_comp_cor_01": {
+    "CumulativeVarianceExplained": 0.422434229,
+    "Method": "tCompCor",
+    "Retained": true,
+    "SingularValue": 128.0789729064,
+    "VarianceExplained": 0.115253234
+  },
+  "t_comp_cor_02": {
+    "CumulativeVarianceExplained": 0.5047450057,
+    "Method": "tCompCor",
+    "Retained": true,
+    "SingularValue": 108.2379668127,
+    "VarianceExplained": 0.0823107766
+  },
+  "tcompcor": {
+    "Method": "Mean"
+  },
+  "w_comp_cor_00": {
+    "CumulativeVarianceExplained": 0.2600923672,
+    "Mask": "WM",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 94.8997276396,
+    "VarianceExplained": 0.2600923672
+  },
+  "w_comp_cor_01": {
+    "CumulativeVarianceExplained": 0.37516225,
+    "Mask": "WM",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 63.1221812188,
+    "VarianceExplained": 0.1150698828
+  },
+  "w_comp_cor_02": {
+    "CumulativeVarianceExplained": 0.4623071378,
+    "Mask": "WM",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 54.9315836787,
+    "VarianceExplained": 0.0871448878
+  },
+  "w_comp_cor_03": {
+    "CumulativeVarianceExplained": 0.5166338394,
+    "Mask": "WM",
+    "Method": "aCompCor",
+    "Retained": true,
+    "SingularValue": 43.3718384282,
+    "VarianceExplained": 0.0543267016
+  },
+  "white_matter": {
+    "Method": "Mean"
+  }
+}
diff --git a/nilearn/interfaces/fmriprep/data/test-v21_desc-confounds_timeseries.tsv b/nilearn/interfaces/fmriprep/data/test-v21_desc-confounds_timeseries.tsv
diff --git a/nilearn/interfaces/fmriprep/load_confounds_compcor.py b/nilearn/interfaces/fmriprep/load_confounds_compcor.py
@@ -1,9 +1,12 @@
 """Helper function for _load_compcor."""
 
 
-prefix_compcor = {"temporal_anat": ["t", "a"],
+prefix_compcor = {"temporal_anat_combined": ["t", "a"],
+                  "temporal_anat_separated": ["t", "a", "w", "c"],
                   "temporal": ["t"],
-                  "anat": ["a"]}
+                  "anat_combined": ["a"],
+                  "anat_separated": ["a", "w", "c"],
+                  }
 anat_masker = {"combined": ["combined"],
                "separated": ["WM", "CSF"],
                None: None}
@@ -32,8 +35,7 @@ def _find_compcor(confounds_json, compcor, n_compcor):
 
     collector = []
     for prefix in prefix_set:
-        # all possible compcor confounds in order, mixing different
-        # types of mask
+        # all possible compcor confounds in order
         all_compcor_name = [
             comp
             for comp in confounds_json.keys()
@@ -94,16 +96,11 @@ def _check_compcor_method(compcor):
     anat_mask : :obj:`list`
         List of anatomical masks to use for acompcor.
     """
-    compcor_type = compcor.split("_")
-    if len(compcor_type) > 1:
-        compcor_type = "_".join(compcor_type[:-1])
-        anat_mask_type = compcor.split("_")[-1:][0]
-    else:
-        compcor_type = compcor_type[0]
-        anat_mask_type = None
     # get relevant prefix from compcor strategy
-    prefix_set = prefix_compcor[compcor_type]
+    prefix_set = prefix_compcor[compcor]
     # get relevant compcor mask
+    check_masktype = compcor.split("_")
+    anat_mask_type = None if len(check_masktype) == 1 else check_masktype[-1]
     anat_mask = anat_masker[anat_mask_type]
     return prefix_set, anat_mask
 

diff --git a/nilearn/interfaces/fmriprep/load_confounds_utils.py b/nilearn/interfaces/fmriprep/load_confounds_utils.py
@@ -334,9 +334,9 @@ def _load_confounds_file_as_dataframe(confounds_raw_path):
     # header format. 1.0.x and 1.1.x series uses camel case
     if any(is_camel_case(col_name) for col_name in confounds_raw.columns):
         raise ValueError(
-            "The confound file contains header in camel case."
-            "This is likely the output from 1.0.x and 1.1.x "
-            "series. We only support fmriprep outputs >= 1.2.0."
+            "The confound file contains header in camel case. "
+            "This is likely the output from 1.0.x and 1.1.x series. "
+            "We only support fmriprep outputs >= 1.2.0."
             f"{confounds_raw.columns}"
         )
 

diff --git a/nilearn/interfaces/fmriprep/tests/_testing.py b/nilearn/interfaces/fmriprep/tests/_testing.py
@@ -45,17 +45,18 @@
 }
 
 
-def get_testdata_path(non_steady_state=True):
+def get_testdata_path(non_steady_state=True, fmriprep_version="1.4.x"):
     """Get file path for the confound regressors."""
-    derivative = "regressors"
+    derivative = "regressors" if fmriprep_version != "21.x.x" else "timeseries"
     path_data = os.path.join(os.path.dirname(
         load_confounds_utils.__file__), "data")
+    suffix = "test-v21" if fmriprep_version == "21.x.x" else "test"
     if non_steady_state:
         return [
             os.path.join(path_data, filename)
             for filename in [
-                f"test_desc-confounds_{derivative}.tsv",
-                f"test_desc-confounds_{derivative}.json",
+                f"{suffix}_desc-confounds_{derivative}.tsv",
+                f"{suffix}_desc-confounds_{derivative}.json",
             ]
         ]
     else:
@@ -74,18 +75,18 @@ def create_tmp_filepath(
     bids_fields=None,
     copy_confounds=False,
     copy_json=False,
-    old_derivative_suffix=False
+    fmriprep_version="1.4.x"
 ):
     entities_to_include = [
         *_bids_entities()["raw"],
         *_bids_entities()["derivatives"]
     ]
     if bids_fields is None:
-        bids_fields = {"entities": {"sub": "test01",
+        bids_fields = {"entities": {"sub": fmriprep_version.replace(".", ""),
                                     "task": "test"}}
 
     """Create test files in temporary directory."""
-    derivative = "regressors" if old_derivative_suffix else "timeseries"
+    derivative = "regressors" if fmriprep_version == "1.2.x" else "timeseries"
 
     # confound files
     bids_fields["entities"]["desc"] = "confounds"
@@ -98,7 +99,7 @@ def create_tmp_filepath(
     tmp_conf = base_path / confounds_filename
 
     if copy_confounds:
-        conf, meta = get_legal_confound()
+        conf, meta = get_legal_confound(fmriprep_version=fmriprep_version)
         conf.to_csv(tmp_conf, sep="\t", index=False)
     else:
         tmp_conf.touch()
@@ -110,7 +111,7 @@ def create_tmp_filepath(
             entities_to_include=entities_to_include
         )
         tmp_meta = base_path / confounds_sidecar
-        conf, meta = get_legal_confound()
+        conf, meta = get_legal_confound(fmriprep_version=fmriprep_version)
         with open(tmp_meta, "w") as file:
             json.dump(meta, file, indent=2)
 
@@ -140,11 +141,12 @@ def create_tmp_filepath(
     return tmp_img, tmp_conf
 
 
-def get_legal_confound(non_steady_state=True):
+def get_legal_confound(non_steady_state=True, fmriprep_version="1.4.x"):
     """Load the valid confound files for manipulation."""
-    conf, meta = get_testdata_path(non_steady_state=non_steady_state)
+    conf, meta = get_testdata_path(non_steady_state=non_steady_state,
+                                   fmriprep_version=fmriprep_version)
     conf = pd.read_csv(conf, delimiter="\t", encoding="utf-8")
-    with open(meta) as file:
+    with open(meta, "r") as file:
         meta = json.load(file)
     return conf, meta