Create_Sleep_Quality_Checker.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Sleep Quality Checker Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-08T20:51:18.756789Z",
     "start_time": "2018-11-08T20:51:16.206741Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "UPGRADE AVAILABLE\n",
      "\n",
      "A more recent version of the Synapse Client (2.0.0) is available. Your version (1.9.4) can be upgraded by typing:\n",
      "    pip install --upgrade synapseclient\n",
      "\n",
      "Python Synapse Client version 2.0.0 release notes\n",
      "\n",
      "https://python-docs.synapse.org/build/html/news.html\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome, Sean Deering!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import datetime as dt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "import synapseclient\n",
    "from synapseclient import Activity, Schema, Table, as_table_columns\n",
    "\n",
    "InteractiveShell.ast_node_interactivity = 'all'\n",
    "syn = synapseclient.Synapse()\n",
    "syn.login()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Raw Sleep Quality Checker Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-08T20:51:03.792459Z",
     "start_time": "2018-11-08T20:50:50.988008Z"
    }
   },
   "outputs": [],
   "source": [
    "sleepquality_checker_raw_id = 'syn7117932'\n",
    "\n",
    "sleepquality_checker = pd.read_csv(syn.get(sleepquality_checker_raw_id).path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Sleep Quality Checker Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-08T20:51:04.422718Z",
     "start_time": "2018-11-08T20:51:03.852456Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#rename columns\n",
    "sleepquality_checker = sleepquality_checker.rename(columns={\n",
    "    'sleepQualityResult':'sq_score',})\n",
    "\n",
    "### Replace white space with NaN\n",
    "sleepquality_checker = sleepquality_checker.replace(r'^\\s*$', np.nan, regex=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove duplicated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66101"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sleepquality_checker)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "65552"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sleepquality_checker = sleepquality_checker.drop_duplicates(keep='first')\n",
    "len(sleepquality_checker)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter based on age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "underage_participants = pd.read_csv( syn.get('syn21905452').path, sep=\"\\t\")\n",
    "sleepquality_checker = sleepquality_checker[~sleepquality_checker.participantId.isin(underage_participants.participantId)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove test accounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accounts = pd.read_excel(syn.get('syn21958537').path)\n",
    "\n",
    "sleepquality_checker = sleepquality_checker[~sleepquality_checker.participantId.isin(test_accounts.participantId)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Internal & External Copies of the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42567, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#download the data \n",
    "sharing_info = pd.read_excel(syn.get('syn21958546').path)\n",
    "healthCodes_with_broadsharing = sharing_info[sharing_info.sharing == 'all_qualified_researchers']\n",
    "\n",
    "EXTERNAL_SLEEPQUALITY_CHECKER_DATA = sleepquality_checker[sleepquality_checker.participantId.isin(healthCodes_with_broadsharing['participant id'])]\n",
    "EXTERNAL_SLEEPQUALITY_CHECKER_DATA.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of unique participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4566"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(EXTERNAL_SLEEPQUALITY_CHECKER_DATA.participantId.unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total number of observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42567"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(EXTERNAL_SLEEPQUALITY_CHECKER_DATA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### External - Upload to Synapse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4566"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<synapseclient.table.CsvFileTable at 0x114d93550>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(EXTERNAL_SLEEPQUALITY_CHECKER_DATA.participantId.unique())\n",
    "SH_EXTERNAL_PROJECT = 'syn18492837'\n",
    "table_schema_external = Schema(name='Sleep Quality Checker',\n",
    "                               columns=as_table_columns(EXTERNAL_SLEEPQUALITY_CHECKER_DATA),\n",
    "                               parent=SH_EXTERNAL_PROJECT)\n",
    "sleepquality_checker_synTable_external = syn.store(Table(table_schema_external,EXTERNAL_SLEEPQUALITY_CHECKER_DATA))\n",
    "sleepquality_checker_synTable_external"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Internal - Upload to Synapse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "SH_INTERNAL_PROJECT = 'syn7066726'\n",
    "table_schema_internal = Schema(name='Sleep Quality Checker Internal',\n",
    "                               columns=as_table_columns(sleepquality_checker), \n",
    "                               parent=SH_INTERNAL_PROJECT)\n",
    "sleepquality_checker_synTable_internal = syn.store(Table(table_schema_internal,sleepquality_checker))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Provenance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'createdBy': u'3334346',\n",
       " u'createdOn': u'2020-07-20T19:24:27.589Z',\n",
       " u'description': u'Process and convert raw data to table format',\n",
       " u'etag': u'dfcf0056-3b70-48f8-b383-0f27ee94ae09',\n",
       " u'id': u'10282152',\n",
       " u'modifiedBy': u'3334346',\n",
       " u'modifiedOn': u'2020-07-20T19:24:27.589Z',\n",
       " u'name': u'Sleep Quality Checker data curation',\n",
       " u'used': [{u'concreteType': u'org.sagebionetworks.repo.model.provenance.UsedEntity',\n",
       "   u'reference': {u'targetId': u'syn7117932', u'targetVersionNumber': 761},\n",
       "   u'wasExecuted': False},\n",
       "  {u'concreteType': u'org.sagebionetworks.repo.model.provenance.UsedURL',\n",
       "   u'name': u'https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Sleep_Quality_Checker.ipynb',\n",
       "   u'url': u'https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Sleep_Quality_Checker.ipynb',\n",
       "   u'wasExecuted': True}]}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "{u'createdBy': u'3334346',\n",
       " u'createdOn': u'2020-07-20T19:24:51.301Z',\n",
       " u'description': u'Process and convert raw data to table format',\n",
       " u'etag': u'905a1568-92bd-4ca6-a18b-d075dd386e83',\n",
       " u'id': u'10282153',\n",
       " u'modifiedBy': u'3334346',\n",
       " u'modifiedOn': u'2020-07-20T19:24:51.301Z',\n",
       " u'name': u'Sleep Quality Checker data curation',\n",
       " u'used': [{u'concreteType': u'org.sagebionetworks.repo.model.provenance.UsedEntity',\n",
       "   u'reference': {u'targetId': u'syn7117932', u'targetVersionNumber': 761},\n",
       "   u'wasExecuted': False},\n",
       "  {u'concreteType': u'org.sagebionetworks.repo.model.provenance.UsedURL',\n",
       "   u'name': u'https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Sleep_Quality_Checker.ipynb',\n",
       "   u'url': u'https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Sleep_Quality_Checker.ipynb',\n",
       "   u'wasExecuted': True}]}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "activity=Activity(name= 'Sleep Quality Checker data curation', \n",
    "                  description='Process and convert raw data to table format', \n",
    "                  used=sleepquality_checker_raw_id, \n",
    "                  executed='https://github.com/apratap/SleepHealth_Data_Release/blob/master/Create_Sleep_Quality_Checker.ipynb')\n",
    "#Internal \n",
    "syn.setProvenance(sleepquality_checker_synTable_internal, activity)\n",
    "\n",
    "#External\n",
    "syn.setProvenance(sleepquality_checker_synTable_external, activity)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.17"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}