Create_Alertness_Checker_PVT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  PVT Data Prep (Alertness Checker)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-08T20:51:18.756789Z",
     "start_time": "2018-11-08T20:51:16.206741Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "UPGRADE AVAILABLE\n",
      "\n",
      "A more recent version of the Synapse Client (2.0.0) is available. Your version (1.9.4) can be upgraded by typing:\n",
      "    pip install --upgrade synapseclient\n",
      "\n",
      "Python Synapse Client version 2.0.0 release notes\n",
      "\n",
      "https://python-docs.synapse.org/build/html/news.html\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome, Sean Deering!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import datetime as dt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "import synapseclient\n",
    "from synapseclient import Activity, Schema, Table, as_table_columns\n",
    "\n",
    "syn = synapseclient.Synapse()\n",
    "syn.login()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Raw PVT Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-08T20:51:03.792459Z",
     "start_time": "2018-11-08T20:50:50.988008Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participantId</th>\n",
       "      <th>result.VigilanceResponseResult</th>\n",
       "      <th>result.StimuliArray[0].InterStimuliTime</th>\n",
       "      <th>result.StimuliArray[0].WrongTaps</th>\n",
       "      <th>result.StimuliArray[0].ReactionTime</th>\n",
       "      <th>result.StimuliArray[0].StimulusNo</th>\n",
       "      <th>result.StimuliArray[1].InterStimuliTime</th>\n",
       "      <th>result.StimuliArray[1].WrongTaps</th>\n",
       "      <th>result.StimuliArray[1].ReactionTime</th>\n",
       "      <th>result.StimuliArray[1].StimulusNo</th>\n",
       "      <th>...</th>\n",
       "      <th>result.StimuliArray[45].ReactionTime</th>\n",
       "      <th>result.StimuliArray[45].StimulusNo</th>\n",
       "      <th>result.StimuliArray[46].InterStimuliTime</th>\n",
       "      <th>result.StimuliArray[46].WrongTaps</th>\n",
       "      <th>result.StimuliArray[46].ReactionTime</th>\n",
       "      <th>result.StimuliArray[46].StimulusNo</th>\n",
       "      <th>endDate</th>\n",
       "      <th>startDate</th>\n",
       "      <th>item</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93ceb9bc-5e6f-495d-8d2f-786bc694fc4f</td>\n",
       "      <td>251.77747</td>\n",
       "      <td>2.000023</td>\n",
       "      <td>0</td>\n",
       "      <td>397.026002</td>\n",
       "      <td>1</td>\n",
       "      <td>2.394613</td>\n",
       "      <td>0.0</td>\n",
       "      <td>370.995045</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>464.231968</td>\n",
       "      <td>46.0</td>\n",
       "      <td>3.894904</td>\n",
       "      <td>0.0</td>\n",
       "      <td>489.142001</td>\n",
       "      <td>47.0</td>\n",
       "      <td>2016-05-08T18:08:22-07:00</td>\n",
       "      <td>2016-05-08T18:08:22-07:00</td>\n",
       "      <td>VigilanceResponseResult</td>\n",
       "      <td>2016-05-08T18:08:22-07:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 194 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          participantId  result.VigilanceResponseResult  \\\n",
       "0  93ceb9bc-5e6f-495d-8d2f-786bc694fc4f                       251.77747   \n",
       "\n",
       "   result.StimuliArray[0].InterStimuliTime  result.StimuliArray[0].WrongTaps  \\\n",
       "0                                 2.000023                                 0   \n",
       "\n",
       "   result.StimuliArray[0].ReactionTime  result.StimuliArray[0].StimulusNo  \\\n",
       "0                           397.026002                                  1   \n",
       "\n",
       "   result.StimuliArray[1].InterStimuliTime  result.StimuliArray[1].WrongTaps  \\\n",
       "0                                 2.394613                               0.0   \n",
       "\n",
       "   result.StimuliArray[1].ReactionTime  result.StimuliArray[1].StimulusNo  \\\n",
       "0                           370.995045                                2.0   \n",
       "\n",
       "   ...  result.StimuliArray[45].ReactionTime  \\\n",
       "0  ...                            464.231968   \n",
       "\n",
       "   result.StimuliArray[45].StimulusNo  \\\n",
       "0                                46.0   \n",
       "\n",
       "   result.StimuliArray[46].InterStimuliTime  \\\n",
       "0                                  3.894904   \n",
       "\n",
       "   result.StimuliArray[46].WrongTaps  result.StimuliArray[46].ReactionTime  \\\n",
       "0                                0.0                            489.142001   \n",
       "\n",
       "   result.StimuliArray[46].StimulusNo                    endDate  \\\n",
       "0                                47.0  2016-05-08T18:08:22-07:00   \n",
       "\n",
       "                   startDate                     item  \\\n",
       "0  2016-05-08T18:08:22-07:00  VigilanceResponseResult   \n",
       "\n",
       "                   timestamp  \n",
       "0  2016-05-08T18:08:22-07:00  \n",
       "\n",
       "[1 rows x 194 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pvt_raw_id = 'syn7117882'\n",
    "pvt_raw = pd.read_csv(syn.get(pvt_raw_id).path)\n",
    "pvt_raw.head(n=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process PVT Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reshape \n",
    "pvt = pvt_raw.melt(id_vars = ['participantId' , 'endDate', 'startDate', 'item', 'timestamp'],\n",
    "         value_vars = pvt_raw.columns[pvt_raw.columns.str.startswith('result.StimuliArray')],\n",
    "          var_name  = 'feature',\n",
    "          value_name = 'value')\n",
    "\n",
    "#drop cols\n",
    "pvt = pvt.drop(['item', 'startDate', 'endDate'], 1)\n",
    "\n",
    "#Extract pvt stimuli block num\n",
    "pvt['block'] = pvt.feature.str.extract('.*\\[(\\d+)\\].*', expand=False)\n",
    "\n",
    "#Remove prefix in feature col \n",
    "pvt['feature'] = pvt.feature.replace(to_replace = 'result.*\\]\\.', value='', regex=True)\n",
    "\n",
    "### Reshape\n",
    "pvt = pvt.set_index(['participantId', 'timestamp', 'block', 'feature']).unstack(level=-1).reset_index()\n",
    "\n",
    "#Fix multiindex column mess \n",
    "first_3_cols = pvt.columns[0:3].droplevel(1).tolist()\n",
    "last_4_cols = pvt.columns[3:].droplevel(0).tolist()\n",
    "first_3_cols.extend(last_4_cols)\n",
    "new_colnames = first_3_cols\n",
    "pvt.columns = new_colnames\n",
    "\n",
    "\n",
    "#Remove block col as it is same as StimuluNo\n",
    "pvt = pvt.drop(['block'], axis=1)\n",
    "\n",
    "#Rename col\n",
    "pvt = pvt.rename(columns={\"StimulusNo\":\"StimulusNum\", \"InterStimuliTime\" : \"InterStimulusInterval\"})\n",
    "\n",
    "#deleting rows where StimulusNum is NaN\n",
    "pvt = pvt[pvt.StimulusNum.notnull()]\n",
    "\n",
    "#Change dtype of StimulusNum to INT\n",
    "pvt.StimulusNum = pvt.StimulusNum.astype(int)\n",
    "\n",
    "#Round up to 4 digits \n",
    "pvt.InterStimulusInterval = pvt.InterStimulusInterval.round(decimals=4)\n",
    "pvt.ReactionTime = pvt.ReactionTime.round(decimals=4)\n",
    "\n",
    "### Replace white space with NaN\n",
    "pvt = pvt.replace(r'^\\s*$', np.nan, regex=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter out underage participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "underage_participants = pd.read_csv( syn.get('syn21905452').path, sep=\"\\t\")\n",
    "pvt = pvt[~pvt.participantId.isin(underage_participants.participantId)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove test accounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accounts = pd.read_excel(syn.get('syn21958537').path)\n",
    "\n",
    "pvt = pvt[~pvt.participantId.isin(test_accounts.participantId)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Internal & External Copy of the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(825120, 6)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#download the data \n",
    "sharing_info = pd.read_excel(syn.get('syn21958546').path)\n",
    "healthCodes_with_broadsharing = sharing_info[sharing_info.sharing == 'all_qualified_researchers']\n",
    "\n",
    "EXTERNAL_PVT_DATA = pvt[pvt.participantId.isin(healthCodes_with_broadsharing['participant id'])]\n",
    "EXTERNAL_PVT_DATA.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of unique participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4031"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(EXTERNAL_PVT_DATA.participantId.unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total number of observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### External - Upload to Synapse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<synapseclient.table.CsvFileTable at 0x116dd1cd0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(EXTERNAL_PVT_DATA.participantId.unique())\n",
    "SH_EXTERNAL_PROJECT = 'syn18492837'\n",
    "table_schema_external = Schema(name='Alertness Checker - Psychomotor Vigilance Task',\n",
    "                               columns=as_table_columns(EXTERNAL_PVT_DATA),\n",
    "                               parent=SH_EXTERNAL_PROJECT)\n",
    "pvt_synTable_external = syn.store(Table(table_schema_external,EXTERNAL_PVT_DATA))\n",
    "pvt_synTable_external"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Internal - Upload to Synapse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-01T16:16:20.597420Z",
     "start_time": "2018-11-01T16:16:01.947714Z"
    }
   },
   "outputs": [],
   "source": [
    "SH_INTERNAL_PROJECT = 'syn7066726'\n",
    "table_schema_internal = Schema(name='Alertness Checker - Psychomotor Vigilance Task Internal',\n",
    "                               columns=as_table_columns(pvt), \n",
    "                               parent=SH_INTERNAL_PROJECT)\n",
    "pvt_synTable_internal = syn.store(Table(table_schema_internal,pvt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attach provenance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'createdBy': u'3334346',\n",
       " u'createdOn': u'2020-04-21T18:25:15.816Z',\n",
       " u'description': u'Process and convert raw data to table format',\n",
       " u'etag': u'7cca0632-ba87-4be4-b469-e97f67c479ca',\n",
       " u'id': u'10233844',\n",
       " u'modifiedBy': u'3334346',\n",
       " u'modifiedOn': u'2020-04-21T18:25:15.816Z',\n",
       " u'name': u'Psychomotor Vigilance Task data curation',\n",
       " u'used': [{u'concreteType': u'org.sagebionetworks.repo.model.provenance.UsedEntity',\n",
       "   u'reference': {u'targetId': u'syn7117882', u'targetVersionNumber': 753},\n",
       "   u'wasExecuted': False},\n",
       "  {u'concreteType': u'org.sagebionetworks.repo.model.provenance.UsedURL',\n",
       "   u'name': u'https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Alertness_Checker_PVT.ipynb',\n",
       "   u'url': u'https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Alertness_Checker_PVT.ipynb',\n",
       "   u'wasExecuted': True}]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "activity=Activity(name= 'Psychomotor Vigilance Task data curation', \n",
    "                  description='Process and convert raw data to table format', \n",
    "                  used=pvt_raw_id, \n",
    "                  executed='https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Alertness_Checker_PVT.ipynb')\n",
    "#Internal \n",
    "syn.setProvenance(pvt_synTable_internal, activity)\n",
    "\n",
    "#External\n",
    "syn.setProvenance(pvt_synTable_external, activity)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.17"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}