datafreeze_notebooks/variability_task_audit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This script creates a binary spreadsheet of all participants enrolled in EF and whether or not their \n",
    "#varability data is complete\n",
    "#Inputs:\n",
    "    #enrollment sheets pulled from AXIS, stored at afp://saturn/Coordinators/Protocols/TED_PROTOCOLS/EXECUTIVE_829744/2022_data_freeze/inputs\n",
    "    #raw task files, stored on Penn+Box\n",
    "#Outputs:\n",
    "    #a binary spreadsheet of varability data collected for T1 participants enrolled prior to April 1st, 2022 \n",
    "    #a binary spreadsheet of varability data collected for T2 participants enrolled prior to April 1st, 2022 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read in T1 enrollment\n",
    "axis_t1=pd.read_csv('/Volumes/Coordinators/Protocols/TED_PROTOCOLS/EXECUTIVE_829744/2022_data_freeze/inputs/axis_enroll_t1.csv')\n",
    "axis_t1=axis_t1.drop(columns=['scan_1_date'])\n",
    "#axis_t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reformat \n",
    "t1_enroll=axis_t1['bblid']\n",
    "t1_enroll=t1_enroll.tolist()\n",
    "t1_enroll = [str(t) for t in t1_enroll]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set directory containing all variability files \n",
    "root_dir = '/Users/krmurtha/Box Sync/EXECUTIVE_FUNCTION/VARIABILITY/' #change file path for appropriate user \n",
    "bids_dir = 'TIMEPOINT 1/'\n",
    "all_files1=glob.glob(root_dir + bids_dir + '*/*')\n",
    "#print(all_files1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort out by task type \n",
    "has_dominant=[]\n",
    "has_reaction=[]\n",
    "has_cpt=[]\n",
    "has_cpt_g=[]\n",
    "has_simple_p=[]\n",
    "has_complex_p=[]\n",
    "has_complex_r=[]\n",
    "for f in all_files1:\n",
    "    if 'Dominant.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_dominant.append(x)\n",
    "    if 'Reaction.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_reaction.append(x)\n",
    "    if 'CPT.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_cpt.append(x)\n",
    "    if 'CPT_gordon.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_cpt_g.append(x)\n",
    "    if 'complex_reasoning.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_complex_r.append(x)\n",
    "    if 'complex_processing_speed.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_complex_p.append(x)\n",
    "    if 'simple_processing_speed.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        has_simple_p.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "audit = pd.DataFrame (t1_enroll, columns = ['BBLID'])\n",
    "#print(len(audit))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort out by task type \n",
    "dominant=[]\n",
    "reaction=[]\n",
    "cpt=[]\n",
    "cpt_g=[]\n",
    "simple_p=[]\n",
    "complex_p=[]\n",
    "complex_r=[]\n",
    "\n",
    "for t in t1_enroll:\n",
    "    if str(t) in [x for x in has_dominant]:\n",
    "        dominant.append(1)\n",
    "    if str(t) not in [x for x in has_dominant]:\n",
    "        dominant.append(0)\n",
    "    if str(t) in [x for x in has_reaction]:\n",
    "        reaction.append(1)\n",
    "    if str(t) not in [x for x in has_reaction]:\n",
    "        reaction.append(0)\n",
    "    if str(t) in [x for x in has_cpt]:\n",
    "        cpt.append(1)\n",
    "    if str(t) not in [x for x in has_cpt]:\n",
    "        cpt.append(0)\n",
    "    if str(t) in [x for x in has_cpt_g]:\n",
    "        cpt_g.append(1)\n",
    "    if str(t) not in [x for x in has_cpt_g]:\n",
    "        cpt_g.append(0)\n",
    "    if str(t) in [x for x in has_simple_p]:\n",
    "        simple_p.append(1)\n",
    "    if str(t) not in [x for x in has_simple_p]:\n",
    "        simple_p.append(0)\n",
    "    if str(t) in [x for x in has_complex_p]:\n",
    "        complex_p.append(1)\n",
    "    if str(t) not in [x for x in has_complex_p]:\n",
    "        complex_p.append(0)\n",
    "    if str(t) in [x for x in has_complex_r]:\n",
    "        complex_r.append(1)\n",
    "    if str(t) not in [x for x in has_complex_r]:\n",
    "        complex_r.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "audit['dominant']=dominant\n",
    "audit['reaction']=reaction\n",
    "audit['cpt']=cpt\n",
    "audit['cpt_g']=cpt_g\n",
    "audit['simple_p']=simple_p\n",
    "audit['complex_p']=complex_p\n",
    "audit['complex_r']=complex_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BBLID</th>\n",
       "      <th>dominant</th>\n",
       "      <th>reaction</th>\n",
       "      <th>cpt</th>\n",
       "      <th>cpt_g</th>\n",
       "      <th>simple_p</th>\n",
       "      <th>complex_p</th>\n",
       "      <th>complex_r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19861</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20124</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20125</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20139</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20141</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BBLID  dominant  reaction  cpt  cpt_g  simple_p  complex_p  complex_r\n",
       "0  19861         1         1    1      1         1          1          1\n",
       "1  20124         0         0    0      0         0          0          0\n",
       "2  20125         1         1    1      1         0          0          0\n",
       "3  20139         1         1    1      1         1          1          1\n",
       "4  20141         1         1    1      1         1          1          1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audit.head(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "audit.to_csv('/Volumes/Coordinators/Protocols/TED_PROTOCOLS/EXECUTIVE_829744/2022_data_freeze/audits/EF_T1_variability_audit.csv', sep = ',', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#and for T2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read in t2 enrollment\n",
    "axis_t2=pd.read_csv('/Volumes/Coordinators/Protocols/TED_PROTOCOLS/EXECUTIVE_829744/2022_data_freeze/inputs/axis_enroll_t2.csv')\n",
    "axis_t2=axis_t2.drop(columns=['scan_2_date'])\n",
    "#axis_t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "t2_enroll=axis_t2['bblid']\n",
    "t2_enroll=t2_enroll.tolist()\n",
    "t2_enroll = [str(t) for t in t2_enroll]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set directory containing all variability files \n",
    "root_dir = '/Users/krmurtha/Box Sync/EXECUTIVE_FUNCTION/VARIABILITY/' #change file path for appropriate user \n",
    "bids_dir = 'TIMEPOINT 2/'\n",
    "all_files2=glob.glob(root_dir + bids_dir + '*/*')\n",
    "#print(all_files2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort out by task type \n",
    "has_dominant=[]\n",
    "has_reaction=[]\n",
    "has_cpt=[]\n",
    "has_cpt_g=[]\n",
    "has_simple_p=[]\n",
    "has_complex_p=[]\n",
    "has_complex_r=[]\n",
    "for f in all_files2:\n",
    "    if 'Dominant.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_dominant.append(y)\n",
    "    if 'Reaction.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_reaction.append(y)\n",
    "    if 'CPT.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_cpt.append(y)\n",
    "    if 'CPT_gordon.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_cpt_g.append(y)\n",
    "    if 'complex_reasoning.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_complex_r.append(y)\n",
    "    if 'complex_processing_speed.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_complex_p.append(y)\n",
    "    if 'simple_processing_speed.log' in f:\n",
    "        x=f.split('/')[7]\n",
    "        y=x.split('_')[0]\n",
    "        has_simple_p.append(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "audit2 = pd.DataFrame (t2_enroll, columns = ['BBLID'])\n",
    "#print(len(audit))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort out by task type \n",
    "dominant=[]\n",
    "reaction=[]\n",
    "cpt=[]\n",
    "cpt_g=[]\n",
    "simple_p=[]\n",
    "complex_p=[]\n",
    "complex_r=[]\n",
    "\n",
    "for t in t2_enroll:\n",
    "    if str(t) in [x for x in has_dominant]:\n",
    "        dominant.append(1)\n",
    "    if str(t) not in [x for x in has_dominant]:\n",
    "        dominant.append(0)\n",
    "    if str(t) in [x for x in has_reaction]:\n",
    "        reaction.append(1)\n",
    "    if str(t) not in [x for x in has_reaction]:\n",
    "        reaction.append(0)\n",
    "    if str(t) in [x for x in has_cpt]:\n",
    "        cpt.append(1)\n",
    "    if str(t) not in [x for x in has_cpt]:\n",
    "        cpt.append(0)\n",
    "    if str(t) in [x for x in has_cpt_g]:\n",
    "        cpt_g.append(1)\n",
    "    if str(t) not in [x for x in has_cpt_g]:\n",
    "        cpt_g.append(0)\n",
    "    if str(t) in [x for x in has_simple_p]:\n",
    "        simple_p.append(1)\n",
    "    if str(t) not in [x for x in has_simple_p]:\n",
    "        simple_p.append(0)\n",
    "    if str(t) in [x for x in has_complex_p]:\n",
    "        complex_p.append(1)\n",
    "    if str(t) not in [x for x in has_complex_p]:\n",
    "        complex_p.append(0)\n",
    "    if str(t) in [x for x in has_complex_r]:\n",
    "        complex_r.append(1)\n",
    "    if str(t) not in [x for x in has_complex_r]:\n",
    "        complex_r.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "audit2['dominant']=dominant\n",
    "audit2['reaction']=reaction\n",
    "audit2['cpt']=cpt\n",
    "audit2['cpt_g']=cpt_g\n",
    "audit2['simple_p']=simple_p\n",
    "audit2['complex_p']=complex_p\n",
    "audit2['complex_r']=complex_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BBLID</th>\n",
       "      <th>dominant</th>\n",
       "      <th>reaction</th>\n",
       "      <th>cpt</th>\n",
       "      <th>cpt_g</th>\n",
       "      <th>simple_p</th>\n",
       "      <th>complex_p</th>\n",
       "      <th>complex_r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20139</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20149</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20188</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20214</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20238</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BBLID  dominant  reaction  cpt  cpt_g  simple_p  complex_p  complex_r\n",
       "0  20139         1         1    0      1         0          0          1\n",
       "1  20149         0         0    0      0         0          0          0\n",
       "2  20188         0         0    0      0         0          0          0\n",
       "3  20214         1         1    1      1         0          0          1\n",
       "4  20238         1         1    0      1         0          0          1"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audit2.head(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "audit2.to_csv('/Volumes/Coordinators/Protocols/TED_PROTOCOLS/EXECUTIVE_829744/2022_data_freeze/audits/EF_T2_variability_audit.csv', sep = ',', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}