Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ENH] integrate load_confounds into first_level_from_bids #4103

Merged
merged 21 commits into from
Dec 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .flake8
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ docstring-convention = numpy
max-line-length = 79
max_complexity = 43
max_function_length = 407
max_parameters_amount = 26
max_parameters_amount = 27
max_returns_amount = 10
# For PEP8 error codes see
# http://pep8.readthedocs.org/en/latest/intro.html#error-codes
Expand Down
1 change: 1 addition & 0 deletions doc/changes/latest.rst
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ Enhancements

- :bdg-primary:`Doc` Add backslash to homogenize :class:`~nilearn.regions.Parcellations` documentation (:gh:`4042` by `Nikhil Krish`_).
- :bdg-success:`API` Allow passing Pandas Series of image filenames to :class:`~nilearn.glm.second_level.SecondLevelModel` (:gh:`4070` by `Rémi Gau`_).
- :bdg-success:`API` Allow passing arguments to :func:`~nilearn.glm.first_level.first_level_from_bids` to build first level models that include specific set of confounds by relying on the strategies from :func:`~nilearn.interfaces.fmriprep.load_confounds` (:gh:`4103` by `Rémi Gau`_).
- :bdg-info:`Plotting` Allow setting ``vmin`` in :func:`~nilearn.plotting.plot_glass_brain` and :func:`~nilearn.plotting.plot_stat_map` (:gh:`3993` by `Michelle Wang`_).
- :bdg-success:`API` Support passing t and F contrasts to :func:`~nilearn.glm.compute_contrast` that that have fewer columns than the number of estimated parameters. Remaining columns are padded with zero (:gh:`4067` by `Rémi Gau`_).
- :bdg-dark:`Code` Multi-subject maskers' ``generate_report`` method no longer fails with 5D data but instead shows report of first subject. User can index input list to show report for different subjects (:gh:`3935` by `Yasmin Mzayek`_).
Expand Down
39 changes: 39 additions & 0 deletions nilearn/_utils/bids.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from nilearn.interfaces.bids._utils import _bids_entities


def create_bids_filename(fields, entities_to_include=None):
"""Create BIDS filename from dictionary of entity-label pairs.

Parameters
----------
fields : :obj:`dict` of :obj:`str`
Dictionary of entity-label pairs, for example:

{
"suffix": "T1w",
"extension": "nii.gz",
"entities": {"acq": "ap",
"desc": "preproc"}
}.

Returns
-------
BIDS filename : :obj:`str`

"""
if entities_to_include is None:
entities_to_include = _bids_entities()["raw"]

filename = ""

for key in entities_to_include:
if key in fields["entities"]:
value = fields["entities"][key]
if value not in (None, ""):
filename += f"{key}-{value}_"
if "suffix" in fields:
filename += f"{fields['suffix']}"
if "extension" in fields:
filename += f".{fields['extension']}"

return filename
82 changes: 27 additions & 55 deletions nilearn/_utils/data_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@

from nilearn import datasets, image, maskers, masking
from nilearn._utils import as_ndarray, logger
from nilearn._utils.bids import create_bids_filename
from nilearn.interfaces.bids._utils import _bids_entities, _check_bids_label
from nilearn.interfaces.fmriprep.tests._testing import get_legal_confound


def generate_mni_space_img(n_scans=1, res=30, random_state=0, mask_dilation=2):
Expand Down Expand Up @@ -736,9 +738,15 @@ def _basic_confounds(length, random_state=0):

"""
rand_gen = np.random.default_rng(random_state)
columns = ['csf', 'white_matter', 'global_signal',
'rot_x', 'rot_y', 'rot_z',
'trans_x', 'trans_y', 'trans_z']
columns = ['csf',
'white_matter',
'global_signal',
'rot_x',
'rot_y',
'rot_z',
'trans_x',
'trans_y',
'trans_z']
data = rand_gen.random((length, len(columns)))
confounds = pd.DataFrame(data, columns=columns)
return confounds
Expand Down Expand Up @@ -1200,46 +1208,6 @@ def _listify(n):
return [""] if n <= 0 else [f"{label:02}" for label in range(1, n + 1)]


def _create_bids_filename(
fields, entities_to_include=None
):
"""Create BIDS filename from dictionary of entity-label pairs.

Parameters
----------
fields : :obj:`dict` of :obj:`str`
Dictionary of entity-label pairs, for example:

{
"suffix": "T1w",
"extension": "nii.gz",
"entities": {"acq": "ap",
"desc": "preproc"}
}.

Returns
-------
BIDS filename : :obj:`str`

"""
if entities_to_include is None:
entities_to_include = _bids_entities()["raw"]

filename = ""

for key in entities_to_include:
if key in fields["entities"]:
value = fields["entities"][key]
if value not in (None, ""):
filename += f"{key}-{value}_"
if "suffix" in fields:
filename += f"{fields['suffix']}"
if "extension" in fields:
filename += f".{fields['extension']}"

return filename


def _init_fields(subject,
session,
task,
Expand Down Expand Up @@ -1267,7 +1235,7 @@ def _init_fields(subject,

See Also
--------
_create_bids_filename
create_bids_filename

"""
fields = {
Expand Down Expand Up @@ -1304,7 +1272,7 @@ def _write_bids_raw_anat(subses_dir, subject, session) -> None:
"extension": "nii.gz",
"entities": {"sub": subject, "ses": session},
}
(anat_path / _create_bids_filename(fields)).write_text("")
(anat_path / create_bids_filename(fields)).write_text("")


def _write_bids_raw_func(
Expand All @@ -1331,8 +1299,9 @@ def _write_bids_raw_func(
Random number generator.

"""
n_time_points = 100
bold_path = func_path / _create_bids_filename(fields)
n_time_points = 30
bold_path = func_path / create_bids_filename(fields)

write_fake_bold_img(
bold_path,
[n_voxels, n_voxels, n_voxels, n_time_points],
Expand All @@ -1341,12 +1310,12 @@ def _write_bids_raw_func(

repetition_time = 1.5
fields["extension"] = "json"
param_path = func_path / _create_bids_filename(fields)
param_path = func_path / create_bids_filename(fields)
param_path.write_text(json.dumps({"RepetitionTime": repetition_time}))

fields["suffix"] = "events"
fields["extension"] = "tsv"
events_path = func_path / _create_bids_filename(fields)
events_path = func_path / create_bids_filename(fields)
basic_paradigm().to_csv(events_path, sep="\t", index=None)


Expand Down Expand Up @@ -1387,17 +1356,20 @@ def _write_bids_derivative_func(
or "desc-confounds_regressors".

"""
n_time_points = 100
n_time_points = 30

if confounds_tag is not None:
fields["suffix"] = confounds_tag
fields["extension"] = "tsv"
confounds_path = func_path / _create_bids_filename(
confounds_path = func_path / create_bids_filename(
fields=fields, entities_to_include=_bids_entities()["raw"]
)
_basic_confounds(length=n_time_points, random_state=rand_gen).to_csv(
confounds_path, sep="\t", index=None
confounds, metadata = get_legal_confound()
confounds.to_csv(
confounds_path, sep="\t", index=None, encoding="utf-8"
)
with open(confounds_path.with_suffix(".json"), "w") as f:
json.dump(metadata, f)
Comment on lines +1367 to +1372
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One issue with this approach is that the "legal_confounds" have a set number of time points so when creating a fake bids dataset, we end up with images that have a number of time points that does not match the number of time points in the confounds.

This does not affect any tests AFIACT but this may lead to confusing errors when testing down the line.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean, because of scrubbing ? Sorry if I miss something obvious.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nope

let me try to rephrase

the way to generate "fake confounds" for the fake fmriprep datasets we use for testing would only create 6 confounds for the realignment parameters filled with random data and for a specified number of time points

to allow testing the load_confounds we need more realistic confounds with more columns with names that match what's in an actual fmriprep dataset

to do this I reuse the strategy used to test the load_confounds functions: use an actual confound file from an fmriprep dataset and copy its content every time it is needed in the fake fmriprep dataset

but this "template" confound file has only a limited number of time points

so we end up with fake fmriprep datasets that have nifti images with 100 volumes but with confounds with only 30 time points

possible solutions:

  • easy: set the number of volumes to match the number of time points in the confounds
  • hard(er): adapt the content of the confounds to the number of time points

hope this is clearer

for now I will go for the easy solution but we may have to implement the harder option in the future if we want to test more "exotic" stuff

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's go for the easy one. The number of volumes should be a parameters of the data simulation function anyhow ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for now it is not: it is hard coded. I would keep it that way for now until we need more flexibility during testing.

but I will change the place where it is hard coded so it is easier to adapt in the future. will also add a comment to explain why this value was chosen.

Comment on lines +1371 to +1372
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor change: using legal_confounds allows to add metada files for the confounds in the fake bids derivatives. some tests had to be changed to account for this.


fields["suffix"] = "bold"
fields["extension"] = "nii.gz"
Expand All @@ -1418,7 +1390,7 @@ def _write_bids_derivative_func(
fields["entities"]["space"] = space
fields["entities"]["desc"] = desc

bold_path = func_path / _create_bids_filename(
bold_path = func_path / create_bids_filename(
fields=fields, entities_to_include=entities_to_include
)
write_fake_bold_img(bold_path, shape=shape, random_state=rand_gen)
Expand All @@ -1428,7 +1400,7 @@ def _write_bids_derivative_func(
fields["entities"].pop("desc")
for hemi in ["L", "R"]:
fields["entities"]["hemi"] = hemi
gifti_path = func_path / _create_bids_filename(
gifti_path = func_path / create_bids_filename(
fields=fields,
entities_to_include=entities_to_include
)
Expand Down
10 changes: 5 additions & 5 deletions nilearn/_utils/tests/test_data_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,8 @@ def test_fake_bids_derivatives_with_session_and_runs(
)

all_files = list(bids_path.glob("derivatives/sub-*/ses-*/*/*"))
# per subject: (1 confound + 3 bold + 2 gifti) per run per session
n_derivatives_files_expected = n_sub * (6 * sum(n_runs) * n_ses)
# per subject: (2 confound + 3 bold + 2 gifti) per run per session
n_derivatives_files_expected = n_sub * (7 * sum(n_runs) * n_ses)
assert len(all_files) == n_derivatives_files_expected


Expand Down Expand Up @@ -375,10 +375,10 @@ def test_fake_bids_extra_raw_entity(tmp_path):
)

all_files = list(bids_path.glob("derivatives/sub-*/ses-*/*/*"))
# per subject: (1 confound + 3 bold + 2 gifti)
# per subject: (2 confound + 3 bold + 2 gifti)
# per run per session per entity
n_derivatives_files_expected = (
n_sub * (6 * sum(n_runs) * n_ses) * len(entities["acq"])
n_sub * (7 * sum(n_runs) * n_ses) * len(entities["acq"])
)
assert len(all_files) == n_derivatives_files_expected

Expand Down Expand Up @@ -420,7 +420,7 @@ def test_fake_bids_extra_derivative_entity(tmp_path):
# 1 confound per run per session
# + (3 bold + 2 gifti) per run per session per entity
n_derivatives_files_expected = n_sub * (
1 * sum(n_runs) * n_ses
2 * sum(n_runs) * n_ses
+ 5 * sum(n_runs) * n_ses * len(entities["res"])
)
assert len(all_files) == n_derivatives_files_expected
Expand Down