notebooks/validator_err_counts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "24eca685-9780-4046-95b2-b1e2f638e73e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate summary info for CuBIDs validator\n",
    "\n",
    "#import packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "id": "4362e865-af7e-4a49-b4b8-2421befc3080",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set path:\n",
    "path='/cbica/projects/wolf_satterthwaite_reward/Margaret/Day2/curation/code/sandbox/validator_outputs'\n",
    "bids_path='/cbica/projects/wolf_satterthwaite_reward/Margaret/Day2/curation/BIDS'\n",
    "iteration='1.2'\n",
    "prefix='d2_r2'\n",
    "\n",
    "#define df\n",
    "df=pd.read_csv(path+'/iteration'+iteration+'/'+prefix+'_validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "bd9949e7-32de-4308-b7be-c494cb217136",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "num_er=np.sum(df['severity']=='error')\n",
    "print(num_er)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "id": "21527232-915a-4bae-9759-7d8b0cf3da26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "497\n"
     ]
    }
   ],
   "source": [
    "num_war=np.sum(df['severity']=='warning')\n",
    "print(num_war)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "83fe4ae9-d7b2-49b2-821b-9afadda4cfb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_war=df[df['severity']=='warning']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "1f5056d8-3ae5-48fa-8ca1-58f7f0aee21c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EVENTS_TSV_MISSING' 'README_FILE_MISSING' 'NO_AUTHORS']\n"
     ]
    }
   ],
   "source": [
    "war_type=df_war['type'].unique()\n",
    "print(war_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "5db1f5dd-2e52-4ef7-ac22-ca367e66d3ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVENTS_TSV_MISSING ( Task scans should have a corresponding events.tsv file. If this is a resting state scan you can ignore this warning or rename the task to include the word \"rest\". ) : 495 subjects\n",
      "README_FILE_MISSING ( The recommended file /README is missing. See Section 03 (Modality agnostic files) of the BIDS specification. ) : 1 subjects\n",
      "NO_AUTHORS ( The Authors field of dataset_description.json should contain an array of fields - with one author per field. This was triggered because there are no authors, which will make DOI registration from dataset metadata impossible. ) : 1 subjects\n"
     ]
    }
   ],
   "source": [
    "for warning in war_type:\n",
    "    num_war_type=np.sum(df_war['type']==warning)\n",
    "    descrip=df_war.loc[df['type'] == warning, 'description'].unique()\n",
    "    dprint=' '.join(map(str, descrip))\n",
    "    print(warning,'(',dprint,')', ':', num_war_type, 'subjects')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0c4500f-c57c-41ec-ad4b-7ac006829c41",
   "metadata": {},
   "source": [
    "##searching subject data for participants with more than one T1w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "186f60d2-d9cc-4a6b-b45a-c6b1a53ec50d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_files=pd.read_csv(path+'/iteration'+iteration+'/'+prefix+'_files.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "6476f524-4ff4-420b-935c-8c6282cb81c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_anat=df_files[df_files['Modality']=='anat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "33298a42-5a61-4f0e-96dd-820f83b109f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KeyParamGroup                     128\n",
       "KeyGroup                          128\n",
       "ParamGroup                        128\n",
       "Counts                            128\n",
       "Dim1Size                          128\n",
       "Dim2Size                          128\n",
       "Dim3Size                          128\n",
       "EchoTime                          128\n",
       "EffectiveEchoSpacing                0\n",
       "FlipAngle                         128\n",
       "HasFieldmap                       128\n",
       "KeyGroupCount                     128\n",
       "Modality                          128\n",
       "NSliceTimes                       128\n",
       "NumVolumes                        128\n",
       "Obliquity                         128\n",
       "ParallelReductionFactorInPlane     69\n",
       "PartialFourier                    128\n",
       "PhaseEncodingDirection              0\n",
       "RepetitionTime                    128\n",
       "TotalReadoutTime                    0\n",
       "UsedAsFieldmap                    128\n",
       "VoxelSizeDim1                     128\n",
       "VoxelSizeDim2                     128\n",
       "VoxelSizeDim3                     128\n",
       "FilePath                          128\n",
       "dtype: int64"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_anat.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "54952acb-0718-44a3-a411-b6bd4e933a90",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_anat=df_anat[['KeyParamGroup','ParamGroup','Counts','FilePath']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "2c9a8e07-aeef-4044-a65d-42ff1f4ec856",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_anat['Subject']=df_anat['FilePath'].str.split('\\/').str[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "520c5b2e-9606-43d6-aa75-d9b0713d6f4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_anat['Multiple_T1']=df_anat.duplicated(subset=['Subject'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "9ab0f734-cec2-4d94-8ceb-2e17a673c4de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KeyParamGroup    3\n",
       "ParamGroup       3\n",
       "Counts           3\n",
       "FilePath         3\n",
       "Subject          3\n",
       "Multiple_T1      3\n",
       "dtype: int64"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_repeat=df_anat[df_anat['Multiple_T1']==True]\n",
    "df_repeat.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "id": "beb4e20d-9ae1-4bad-9cef-ebc9082b74e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_repeat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "48490e29-cd66-43f7-8620-6d20e9457de2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AcqGroup</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   AcqGroup\n",
       "0         3\n",
       "1         3\n",
       "2         3"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#finding acquisition group(s) for subj. with T2 scans\n",
    "df_acq=pd.read_csv(path+'/iteration'+iteration+'/'+prefix+'_AcqGrouping.csv')\n",
    "df_repeat_acq=pd.merge(df_repeat, df_acq, left_on = 'Subject', right_on = 'subject', how = 'inner')\n",
    "df_repeat_acq=df_repeat_acq[['AcqGroup']]\n",
    "df_repeat_acq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "5766df57-faec-4990-8e07-e169baf88462",
   "metadata": {},
   "outputs": [],
   "source": [
    "#searching subject data for participants with more than one fmap\n",
    "\n",
    "df_fmap=df_files[df_files['Modality']=='fmap']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "5b2eea79-afac-4fee-b205-b35a674595e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fmap=df_fmap[['KeyParamGroup','KeyGroup','ParamGroup','Counts','FilePath']]\n",
    "df_fmap['Subject']=df_fmap['FilePath'].str.split('\\/').str[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "d93182f2-cbda-40f2-95e5-5395baf84bf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['datatype-fmap_fmap-phase1_suffix-phase1'\n",
      " 'datatype-fmap_fmap-phase2_suffix-phase2'\n",
      " 'datatype-fmap_fmap-magnitude1_suffix-magnitude1'\n",
      " 'datatype-fmap_fmap-magnitude2_suffix-magnitude2']\n"
     ]
    }
   ],
   "source": [
    "df_keygr=df_fmap['KeyGroup'].unique()\n",
    "print(df_keygr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "e03db797-d32f-4a55-92ba-111f8191c8fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/1386749628.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df_p1=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-phase1_suffix-phase1']\n",
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/1386749628.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_p1['Multiple_P1']=df_p1.duplicated(subset=['Subject'])\n"
     ]
    }
   ],
   "source": [
    "df_p1=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-phase1_suffix-phase1']\n",
    "df_p1['Multiple_P1']=df_p1.duplicated(subset=['Subject'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "7ce105c2-8a43-4f48-82a2-4347a9a037b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KeyParamGroup    0\n",
       "KeyGroup         0\n",
       "ParamGroup       0\n",
       "Counts           0\n",
       "FilePath         0\n",
       "Subject          0\n",
       "Multiple_P1      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p1r=df_p1[df_p1['Multiple_P1']==True]\n",
    "df_p1r.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "eb84e350-5a0c-47a2-a8e6-f74387c1b07c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/3665166110.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df_p2=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-phase2_suffix-phase2']\n",
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/3665166110.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_p2['Multiple_P2']=df_p2.duplicated(subset=['Subject'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "KeyParamGroup    0\n",
       "KeyGroup         0\n",
       "ParamGroup       0\n",
       "Counts           0\n",
       "FilePath         0\n",
       "Subject          0\n",
       "Multiple_P2      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p2=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-phase2_suffix-phase2']\n",
    "df_p2['Multiple_P2']=df_p2.duplicated(subset=['Subject'])\n",
    "df_p2r=df_p2[df_p2['Multiple_P2']==True]\n",
    "df_p2r.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "5272751b-113c-4ff0-be10-b8bccd8de9aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/1223881790.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df_m1=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-magnitude1_suffix-magnitude1']\n",
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/1223881790.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_m1['Multiple_M1']=df_m1.duplicated(subset=['Subject'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "KeyParamGroup    0\n",
       "KeyGroup         0\n",
       "ParamGroup       0\n",
       "Counts           0\n",
       "FilePath         0\n",
       "Subject          0\n",
       "Multiple_M1      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_m1=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-magnitude1_suffix-magnitude1']\n",
    "df_m1['Multiple_M1']=df_m1.duplicated(subset=['Subject'])\n",
    "df_m1r=df_m1[df_m1['Multiple_M1']==True]\n",
    "df_m1r.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "8184b472-b1be-41c7-acdd-2606257229a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/1802827136.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df_m2=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-magnitude2_suffix-magnitude2']\n",
      "/scratch/wolfsatterthwaitereward/tmp.WL1HnBKnz7/ipykernel_38573/1802827136.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_m2['Multiple_M2']=df_m2.duplicated(subset=['Subject'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "KeyParamGroup    0\n",
       "KeyGroup         0\n",
       "ParamGroup       0\n",
       "Counts           0\n",
       "FilePath         0\n",
       "Subject          0\n",
       "Multiple_M2      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_m2=df_fmap[df_files['KeyGroup']=='datatype-fmap_fmap-magnitude2_suffix-magnitude2']\n",
    "df_m2['Multiple_M2']=df_m2.duplicated(subset=['Subject'])\n",
    "df_m2r=df_m2[df_m2['Multiple_M2']==True]\n",
    "df_m2r.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "08483197-4965-4f5d-9bbc-183a21fb1bd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "124"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sub with phase1\n",
    "df_p1.Subject.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "30b62b72-6505-4c6d-902f-53c94e3bad06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "119"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sub with phase2\n",
    "df_p2.Subject.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "id": "6da0ca80-b6ad-4a5b-9aeb-7009772cd41c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "118"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sub with mag1\n",
    "df_m1.Subject.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "be0bfb80-ff04-428e-aa7f-70e55c7c92f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "118"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sub with mag2\n",
    "df_m2.Subject.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed9d4f2d-068d-4517-92cb-eed448ddebc1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}