[FIX] Replace numpy with pandas in data loaders

nilearn/datasets/atlas.py

@@ -10,6 +10,7 @@
 
 import nibabel as nb
 import numpy as np
+import pandas as pd
 from numpy.lib import recfunctions
 import re
 from sklearn.utils import Bunch
@@ -112,7 +113,7 @@ def fetch_atlas_difumo(dimension=64, resolution_mm=2, data_dir=None, resume=True
 
     # Download the zip file, first
     files_ = _fetch_files(data_dir, files, verbose=verbose)
-    labels = np.recfromcsv(files_[0])
+    labels = pd.read_csv(files_[0]).to_records()
 
     # README
     readme_files = [('README.md', 'https://osf.io/4k9bf/download',
@@ -259,7 +260,8 @@ def fetch_atlas_destrieux_2009(lateralized=True, data_dir=None, url=None,
     files_ = _fetch_files(data_dir, files, resume=resume,
                           verbose=verbose)
 
-    params = dict(maps=files_[1], labels=np.recfromcsv(files_[0]))
+    params = dict(maps=files_[1],
+                  labels=pd.read_csv(files_[0], index_col=0).to_records())
 
     with open(files_[2], 'r') as rst_file:
         params['description'] = rst_file.read()
@@ -712,13 +714,13 @@ def fetch_atlas_msdl(data_dir=None, url=None, resume=True, verbose=1):
     data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
                                 verbose=verbose)
     files = _fetch_files(data_dir, files, resume=resume, verbose=verbose)
-    csv_data = np.recfromcsv(files[0])
+    csv_data = pd.read_csv(files[0])
     labels = [name.strip() for name in csv_data['name'].tolist()]
-    labels = [label.decode("utf-8") for label in labels]
+
     with warnings.catch_warnings():
         warnings.filterwarnings('ignore', module='numpy',
                                 category=FutureWarning)
-        region_coords = csv_data[['x', 'y', 'z']].tolist()
+        region_coords = csv_data[['x', 'y', 'z']].values.tolist()
     net_names = [net_name.strip() for net_name in csv_data['net_name'].tolist()]
     fdescr = _get_dataset_descr(dataset_name)
 
@@ -750,7 +752,7 @@ def fetch_coords_power_2011():
     fdescr = _get_dataset_descr(dataset_name)
     package_directory = os.path.dirname(os.path.abspath(__file__))
     csv = os.path.join(package_directory, "data", "power_2011.csv")
-    params = dict(rois=np.recfromcsv(csv), description=fdescr)
+    params = dict(rois=pd.read_csv(csv).to_records(), description=fdescr)
 
     return Bunch(**params)
 
@@ -1202,10 +1204,10 @@ def fetch_coords_dosenbach_2010(ordered_regions=True):
     fdescr = _get_dataset_descr(dataset_name)
     package_directory = os.path.dirname(os.path.abspath(__file__))
     csv = os.path.join(package_directory, "data", "dosenbach_2010.csv")
-    out_csv = np.recfromcsv(csv)
+    out_csv = pd.read_csv(csv)
 
     if ordered_regions:
-        out_csv = np.sort(out_csv, order=['network', 'name', 'y'])
+        out_csv = out_csv.sort_values(by=['network', 'name', 'y'])
 
     # We add the ROI number to its name, since names are not unique
     names = out_csv['name']
@@ -1267,10 +1269,8 @@ def fetch_coords_seitzman_2018(ordered_regions=True):
     anatomical_file = os.path.join(package_directory, "data",
                                    "seitzman_2018_ROIs_anatomicalLabels.txt")
 
-    rois = np.recfromcsv(roi_file, delimiter=" ")
-    rois = recfunctions.rename_fields(rois, {"netname": "network",
-                                             "radiusmm": "radius"})
-    rois.network = rois.network.astype(str)
+    rois = pd.read_csv(roi_file, delimiter=" ")
+    rois = rois.rename(columns={"netName": "network", "radius(mm)": "radius"})
 
     # get integer regional labels and convert to text labels with mapping
     # from header line
@@ -1284,16 +1284,15 @@ def fetch_coords_seitzman_2018(ordered_regions=True):
     anatomical = np.genfromtxt(anatomical_file, skip_header=1)
     anatomical_names = np.array([region_mapping[a] for a in anatomical])
 
-    rois = recfunctions.merge_arrays((rois, anatomical_names),
-                                     asrecarray=True, flatten=True)
-    rois.dtype.names = rois.dtype.names[:-1] + ("region",)
+    rois = pd.concat([rois, pd.DataFrame(anatomical_names)], axis=1)
+    rois.columns = list(rois.columns[:-1]) + ["region"]
 
     if ordered_regions:
-        rois = np.sort(rois, order=['network', 'y'])
+        rois = rois.sort_values(by=['network', 'y'])
 
     params = dict(rois=rois[['x', 'y', 'z']],
                   radius=rois['radius'],
-                  networks=rois['network'].astype(str),
+                  networks=rois['network'],
                   regions=rois['region'], description=fdescr)
 
     return Bunch(**params)

nilearn/datasets/func.py

@@ -740,20 +740,17 @@ def _is_valid_path(path, index, verbose):
     # Load covariates file
     from numpy.lib.recfunctions import join_by
     participants_file = os.path.join(data_dir, participants_file)
-    csv_data = np.recfromcsv(participants_file, delimiter='\t')
+    csv_data = pd.read_csv(participants_file, delimiter='\t')
     behavioural_file = os.path.join(data_dir, behavioural_file)
-    csv_data2 = np.recfromcsv(behavioural_file, delimiter='\t')
-    csv_data = join_by(
-        "participant_id", csv_data, csv_data2, usemask=False, asrecarray=True)
+    csv_data2 = pd.read_csv(behavioural_file, delimiter='\t')
+    csv_data = csv_data.merge(csv_data2)
     subject_names = csv_data["participant_id"].tolist()
     subjects_indices = []
     for name in subject_ids:
-        name = name.encode("utf8")
         if name not in subject_names:
             continue
         subjects_indices.append(subject_names.index(name))
-    csv_data = csv_data[subjects_indices]
-
+    csv_data = csv_data.iloc[subjects_indices]
     return Bunch(ext_vars=csv_data, description=fdescr, **files)
 
 
@@ -982,10 +979,10 @@ def fetch_abide_pcp(data_dir=None, n_subjects=None, pipeline='cpac',
     # bytes (encode()) needed for python 2/3 compat with numpy
     pheno = '\n'.join(pheno).encode()
     pheno = BytesIO(pheno)
-    pheno = np.recfromcsv(pheno, comments='$', case_sensitive=True)
+    pheno = pd.read_csv(pheno, comment='$')
 
     # First, filter subjects with no filename
-    pheno = pheno[pheno['FILE_ID'] != b'no_filename']
+    pheno = pheno[pheno['FILE_ID'] != 'no_filename']
     # Apply user defined filters
     user_filter = _filter_columns(pheno, kwargs)
     pheno = pheno[user_filter]
@@ -996,7 +993,7 @@ def fetch_abide_pcp(data_dir=None, n_subjects=None, pipeline='cpac',
 
     # Get the files
     results = {}
-    file_ids = [file_id.decode() for file_id in pheno['FILE_ID']]
+    file_ids = pheno['FILE_ID'].tolist()
     if n_subjects is not None:
         file_ids = file_ids[:n_subjects]
         pheno = pheno[:n_subjects]
@@ -1667,7 +1664,7 @@ def _reduce_confounds(regressors, keep_confounds):
         out_file = in_file.replace('desc-confounds',
                                    'desc-reducedConfounds')
         if not os.path.isfile(out_file):
-            confounds = np.recfromcsv(in_file, delimiter='\t')
+            confounds = pd.read_csv(in_file, delimiter='\t').to_records()
             selected_confounds = confounds[keep_confounds]
             header = '\t'.join(selected_confounds.dtype.names)
             np.savetxt(out_file, np.array(selected_confounds.tolist()),

nilearn/datasets/struct.py

@@ -10,6 +10,7 @@
 from pathlib import Path
 
 import numpy as np
+import pandas as pd
 from scipy import ndimage
 from sklearn.utils import Bunch
 
@@ -735,14 +736,14 @@ def fetch_oasis_vbm(n_subjects=None, dartel_version=True, data_dir=None,
     data_usage_agreement = files[-1]
 
     # Keep CSV information only for selected subjects
-    csv_data = np.recfromcsv(ext_vars_file)
+    csv_data = pd.read_csv(ext_vars_file)
     # Comparisons to recfromcsv data must be bytes.
     actual_subjects_ids = [("OAS1" +
                             str.split(os.path.basename(x),
                                       "OAS1")[1][:9]).encode()
                            for x in gm_maps]
     subject_mask = np.asarray([subject_id in actual_subjects_ids
-                               for subject_id in csv_data['id']])
+                               for subject_id in csv_data['ID']])
     csv_data = csv_data[subject_mask]
 
     fdescr = _get_dataset_descr(dataset_name)

nilearn/datasets/tests/test_func.py

@@ -170,20 +170,20 @@ def test_fetch_localizer_contrasts(tmp_path, request_mocker, localizer_mocker):
     assert not hasattr(dataset, 'tmaps')
     assert not hasattr(dataset, 'masks')
     assert isinstance(dataset.cmaps[0], str)
-    assert isinstance(dataset.ext_vars, np.recarray)
+    assert isinstance(dataset.ext_vars, pd.DataFrame)
     assert len(dataset.cmaps) == 2
-    assert dataset.ext_vars.size == 2
+    assert len(dataset['ext_vars']) == 2
 
     # Multiple contrasts
     dataset = func.fetch_localizer_contrasts(
         ['checkerboard', 'horizontal checkerboard'],
         n_subjects=2,
         data_dir=tmp_path,
         verbose=1)
-    assert isinstance(dataset.ext_vars, np.recarray)
+    assert isinstance(dataset.ext_vars, pd.DataFrame)
     assert isinstance(dataset.cmaps[0], str)
     assert len(dataset.cmaps) == 2 * 2  # two contrasts are fetched
-    assert dataset.ext_vars.size == 2
+    assert len(dataset['ext_vars']) == 2
 
     # all get_*=True
     dataset = func.fetch_localizer_contrasts(
@@ -194,12 +194,12 @@ def test_fetch_localizer_contrasts(tmp_path, request_mocker, localizer_mocker):
         get_masks=True,
         get_tmaps=True,
         verbose=1)
-    assert isinstance(dataset.ext_vars, np.recarray)
+    assert isinstance(dataset.ext_vars, pd.DataFrame)
     assert isinstance(dataset.anats[0], str)
     assert isinstance(dataset.cmaps[0], str)
     assert isinstance(dataset.masks[0], str)
     assert isinstance(dataset.tmaps[0], str)
-    assert dataset.ext_vars.size == 1
+    assert len(dataset['ext_vars']) == 1
     assert len(dataset.anats) == 1
     assert len(dataset.cmaps) == 1
     assert len(dataset.masks) == 1
@@ -212,10 +212,11 @@ def test_fetch_localizer_contrasts(tmp_path, request_mocker, localizer_mocker):
         n_subjects=[2, 3, 5],
         data_dir=tmp_path,
         verbose=1)
-    assert dataset2.ext_vars.size == 3
+    assert len(dataset2['ext_vars']) == 3
     assert len(dataset2.cmaps) == 3
-    assert ([row[0] for row in dataset2.ext_vars] ==
-                 [b'S02', b'S03', b'S05'])
+    assert (list(dataset2['ext_vars']['participant_id'].values) == ['S02',
+                                                                    'S03',
+                                                                    'S05'])
 
 
 def test_fetch_localizer_calculation_task(tmp_path, request_mocker,
@@ -225,9 +226,9 @@ def test_fetch_localizer_calculation_task(tmp_path, request_mocker,
         n_subjects=2,
         data_dir=tmp_path,
         verbose=1)
-    assert isinstance(dataset.ext_vars, np.recarray)
+    assert isinstance(dataset.ext_vars, pd.DataFrame)
     assert isinstance(dataset.cmaps[0], str)
-    assert dataset.ext_vars.size == 2
+    assert len(dataset['ext_vars']) == 2
     assert len(dataset.cmaps) == 2
     assert dataset.description != ''
 

nilearn/datasets/tests/test_struct.py

@@ -120,7 +120,7 @@ def test_fetch_oasis_vbm(tmp_path, request_mocker):
     assert len(dataset.white_matter_maps) == 403
     assert isinstance(dataset.gray_matter_maps[0], str)
     assert isinstance(dataset.white_matter_maps[0], str)
-    assert isinstance(dataset.ext_vars, np.recarray)
+    assert isinstance(dataset.ext_vars, pd.DataFrame)
     assert isinstance(dataset.data_usage_agreement, str)
     assert request_mocker.url_count == 1
 
@@ -130,7 +130,7 @@ def test_fetch_oasis_vbm(tmp_path, request_mocker):
     assert len(dataset.white_matter_maps) == 415
     assert isinstance(dataset.gray_matter_maps[0], str)
     assert isinstance(dataset.white_matter_maps[0], str)
-    assert isinstance(dataset.ext_vars, np.recarray)
+    assert isinstance(dataset.ext_vars, pd.DataFrame)
     assert isinstance(dataset.data_usage_agreement, str)
     assert request_mocker.url_count == 2
     assert dataset.description != ''