Initial Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "import os, sys\n",
    "import pandas as pd\n",
    "from importlib import reload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   2012    2012   27058\n",
      "   2012    2012   27058\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "ls ./../data_pp2/data/pp_2 | wc\n",
    "ls ./../data_pp1/data/pp_1 | wc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/sawer/src/dbmi/eeg\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = pkl.load(open(\"./../data_pp1/data/pp_1/seiz_2.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(320, 20, 24)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1.data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(x, y, z) where x is a time sample of size $w_l$, y is the number of eeg channels, z is the number of frequency bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = pkl.load(open(\"./../data_pp2/data/pp_2/seiz_3.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25, 900)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45270000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "25 * 900 * 2012"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(x, y) x is number of samples and where y is 30*30 (Number of features, should be diagonal of correlations coefficients matrix's eigenvalues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'util_funcs' from '/mnt/c/Users/sawer/src/dbmi/eeg/util_funcs.py'>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import util_funcs\n",
    "reload(util_funcs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting reading 0\n",
      "Starting reading 100\n"
     ]
    }
   ],
   "source": [
    "data = util_funcs.read_all(num_workers=None, num_files=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "shapes = [data[i].data.shape for i in range(len(data))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(18, 900),\n",
       " (45, 900),\n",
       " (159, 900),\n",
       " (25, 900),\n",
       " (476, 900),\n",
       " (344, 900),\n",
       " (97, 900),\n",
       " (51, 900),\n",
       " (75, 900),\n",
       " (121, 900),\n",
       " (37, 900),\n",
       " (332, 900),\n",
       " (27, 900),\n",
       " (132, 900),\n",
       " (158, 900),\n",
       " (180, 900),\n",
       " (24, 900),\n",
       " (66, 900),\n",
       " (65, 900),\n",
       " (27, 900),\n",
       " (46, 900),\n",
       " (150, 900),\n",
       " (42, 900),\n",
       " (280, 900),\n",
       " (12, 900),\n",
       " (14, 900),\n",
       " (15, 900),\n",
       " (13, 900),\n",
       " (185, 900),\n",
       " (16, 900),\n",
       " (57, 900),\n",
       " (46, 900),\n",
       " (71, 900),\n",
       " (233, 900),\n",
       " (125, 900),\n",
       " (4, 900),\n",
       " (96, 900),\n",
       " (26, 900),\n",
       " (45, 900),\n",
       " (38, 900),\n",
       " (27, 900),\n",
       " (483, 900),\n",
       " (64, 900),\n",
       " (71, 900),\n",
       " (6, 900),\n",
       " (36, 900),\n",
       " (52, 900),\n",
       " (27, 900),\n",
       " (52, 900),\n",
       " (21, 900),\n",
       " (155, 900),\n",
       " (10, 900),\n",
       " (49, 900),\n",
       " (18, 900),\n",
       " (55, 900),\n",
       " (42, 900),\n",
       " (34, 900),\n",
       " (74, 900),\n",
       " (32, 900),\n",
       " (60, 900),\n",
       " (9, 900),\n",
       " (12, 900),\n",
       " (208, 900),\n",
       " (30, 900),\n",
       " (36, 900),\n",
       " (8, 900),\n",
       " (46, 900),\n",
       " (38, 900),\n",
       " (21, 900),\n",
       " (18, 900),\n",
       " (651, 900),\n",
       " (31, 900),\n",
       " (246, 900),\n",
       " (79, 900),\n",
       " (135, 900),\n",
       " (40, 900),\n",
       " (212, 900),\n",
       " (14, 900),\n",
       " (81, 900),\n",
       " (13, 900),\n",
       " (238, 900),\n",
       " (121, 900),\n",
       " (627, 900),\n",
       " (77, 900),\n",
       " (33, 900),\n",
       " (173, 900),\n",
       " (277, 900),\n",
       " (11, 900),\n",
       " (91, 900),\n",
       " (73, 900),\n",
       " (70, 900),\n",
       " (13, 900),\n",
       " (243, 900),\n",
       " (546, 900),\n",
       " (38, 900),\n",
       " (31, 900),\n",
       " (87, 900),\n",
       " (31, 900),\n",
       " (119, 900),\n",
       " (407, 900),\n",
       " (29, 900),\n",
       " (285, 900),\n",
       " (39, 900),\n",
       " (59, 900),\n",
       " (42, 900),\n",
       " (9, 900),\n",
       " (95, 900),\n",
       " (93, 900),\n",
       " (191, 900),\n",
       " (128, 900),\n",
       " (24, 900),\n",
       " (339, 900),\n",
       " (17, 900),\n",
       " (89, 900),\n",
       " (100, 900),\n",
       " (12, 900),\n",
       " (140, 900),\n",
       " (207, 900),\n",
       " (129, 900),\n",
       " (73, 900),\n",
       " (8, 900),\n",
       " (176, 900),\n",
       " (123, 900),\n",
       " (14, 900),\n",
       " (299, 900),\n",
       " (56, 900),\n",
       " (21, 900),\n",
       " (24, 900),\n",
       " (30, 900),\n",
       " (141, 900),\n",
       " (66, 900),\n",
       " (16, 900),\n",
       " (229, 900),\n",
       " (173, 900),\n",
       " (52, 900),\n",
       " (47, 900),\n",
       " (169, 900),\n",
       " (269, 900),\n",
       " (120, 900),\n",
       " (364, 900),\n",
       " (53, 900),\n",
       " (65, 900),\n",
       " (483, 900),\n",
       " (312, 900),\n",
       " (58, 900),\n",
       " (187, 900),\n",
       " (116, 900),\n",
       " (2, 900),\n",
       " (729, 900),\n",
       " (60, 900)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = [data[i].data.shape[0] for i in range(len(data))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "width = 900\n",
    "paddedX = np.zeros((max(lengths), width))\n",
    "for i in range(len(data)):\n",
    "    paddedX[0:lengths[i]] = data[i].data\n",
    "paddedY = np.ndarray((max(lengths)))\n",
    "paddedY.fill(0)\n",
    "for i in range(len(data)):\n",
    "    paddedY[0:lengths[i]] = util_funcs.get_seizure_types().index(data[i].seizure_type) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(729, 900)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paddedX.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17137, 900)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allData.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "al"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "seizure_types = set([util_funcs.read_preproc_1(i).seizure_type for i in range(2012)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ABSZ', 'CPSZ', 'FNSZ', 'GNSZ', 'MYSZ', 'SPSZ', 'TCSZ', 'TNSZ'}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seizure_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([[1,2,3],[4,5,6\n",
    "                        ]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2\n",
       "0  1  2  3\n",
       "1  4  5  6"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.lo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = util_funcs.read_all(use_1=True, num_files=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "shapes = [data1[i].data.shape for i in range(len(data1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(37, 20, 24),\n",
       " (89, 20, 24),\n",
       " (320, 20, 24),\n",
       " (50, 20, 24),\n",
       " (959, 20, 24),\n",
       " (693, 20, 24),\n",
       " (194, 20, 24),\n",
       " (103, 20, 24),\n",
       " (150, 20, 24),\n",
       " (244, 20, 24),\n",
       " (74, 20, 24),\n",
       " (670, 20, 24),\n",
       " (53, 20, 24),\n",
       " (265, 20, 24),\n",
       " (319, 20, 24),\n",
       " (363, 20, 24),\n",
       " (47, 20, 24),\n",
       " (132, 20, 24),\n",
       " (131, 20, 24),\n",
       " (55, 20, 24),\n",
       " (92, 20, 24),\n",
       " (302, 20, 24),\n",
       " (84, 20, 24),\n",
       " (565, 20, 24),\n",
       " (25, 20, 24),\n",
       " (27, 20, 24),\n",
       " (30, 20, 24),\n",
       " (26, 20, 24),\n",
       " (372, 20, 24),\n",
       " (31, 20, 24),\n",
       " (115, 20, 24),\n",
       " (93, 20, 24),\n",
       " (143, 20, 24),\n",
       " (469, 20, 24),\n",
       " (252, 20, 24),\n",
       " (8, 20, 24),\n",
       " (193, 20, 24),\n",
       " (51, 20, 24),\n",
       " (91, 20, 24),\n",
       " (77, 20, 24),\n",
       " (54, 20, 24),\n",
       " (973, 20, 24),\n",
       " (128, 20, 24),\n",
       " (143, 20, 24),\n",
       " (12, 20, 24),\n",
       " (71, 20, 24),\n",
       " (105, 20, 24),\n",
       " (55, 20, 24),\n",
       " (104, 20, 24),\n",
       " (41, 20, 24),\n",
       " (312, 20, 24),\n",
       " (20, 20, 24),\n",
       " (97, 20, 24),\n",
       " (37, 20, 24),\n",
       " (111, 20, 24),\n",
       " (84, 20, 24),\n",
       " (68, 20, 24),\n",
       " (149, 20, 24),\n",
       " (63, 20, 24),\n",
       " (121, 20, 24),\n",
       " (18, 20, 24),\n",
       " (23, 20, 24),\n",
       " (420, 20, 24),\n",
       " (61, 20, 24),\n",
       " (71, 20, 24),\n",
       " (16, 20, 24),\n",
       " (92, 20, 24),\n",
       " (76, 20, 24),\n",
       " (43, 20, 24),\n",
       " (37, 20, 24),\n",
       " (1311, 20, 24),\n",
       " (62, 20, 24),\n",
       " (496, 20, 24),\n",
       " (158, 20, 24),\n",
       " (272, 20, 24),\n",
       " (80, 20, 24),\n",
       " (426, 20, 24),\n",
       " (27, 20, 24),\n",
       " (164, 20, 24),\n",
       " (25, 20, 24),\n",
       " (480, 20, 24),\n",
       " (243, 20, 24),\n",
       " (1264, 20, 24),\n",
       " (155, 20, 24),\n",
       " (65, 20, 24),\n",
       " (348, 20, 24),\n",
       " (559, 20, 24),\n",
       " (22, 20, 24),\n",
       " (184, 20, 24),\n",
       " (147, 20, 24),\n",
       " (142, 20, 24),\n",
       " (25, 20, 24),\n",
       " (489, 20, 24),\n",
       " (1100, 20, 24),\n",
       " (76, 20, 24),\n",
       " (62, 20, 24),\n",
       " (175, 20, 24),\n",
       " (63, 20, 24),\n",
       " (240, 20, 24),\n",
       " (820, 20, 24),\n",
       " (58, 20, 24),\n",
       " (574, 20, 24),\n",
       " (79, 20, 24),\n",
       " (119, 20, 24),\n",
       " (84, 20, 24),\n",
       " (17, 20, 24),\n",
       " (191, 20, 24),\n",
       " (186, 20, 24),\n",
       " (384, 20, 24),\n",
       " (257, 20, 24),\n",
       " (49, 20, 24),\n",
       " (683, 20, 24),\n",
       " (34, 20, 24),\n",
       " (178, 20, 24),\n",
       " (202, 20, 24),\n",
       " (24, 20, 24),\n",
       " (282, 20, 24),\n",
       " (417, 20, 24),\n",
       " (259, 20, 24),\n",
       " (146, 20, 24),\n",
       " (17, 20, 24),\n",
       " (355, 20, 24),\n",
       " (248, 20, 24),\n",
       " (28, 20, 24),\n",
       " (602, 20, 24),\n",
       " (112, 20, 24),\n",
       " (42, 20, 24),\n",
       " (47, 20, 24),\n",
       " (61, 20, 24),\n",
       " (285, 20, 24),\n",
       " (132, 20, 24),\n",
       " (31, 20, 24),\n",
       " (460, 20, 24),\n",
       " (347, 20, 24),\n",
       " (104, 20, 24),\n",
       " (94, 20, 24),\n",
       " (339, 20, 24),\n",
       " (541, 20, 24),\n",
       " (241, 20, 24),\n",
       " (734, 20, 24),\n",
       " (106, 20, 24),\n",
       " (130, 20, 24),\n",
       " (973, 20, 24),\n",
       " (629, 20, 24),\n",
       " (116, 20, 24),\n",
       " (377, 20, 24),\n",
       " (234, 20, 24),\n",
       " (3, 20, 24),\n",
       " (1469, 20, 24),\n",
       " (120, 20, 24)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-18:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/saqibm128/miniconda3/envs/dbmi/lib/python3.7/threading.py\", line 917, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/saqibm128/miniconda3/envs/dbmi/lib/python3.7/threading.py\", line 865, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/saqibm128/miniconda3/envs/dbmi/lib/python3.7/site-packages/multiprocess/pool.py\", line 470, in _handle_results\n",
      "    task = get()\n",
      "  File \"/home/saqibm128/miniconda3/envs/dbmi/lib/python3.7/site-packages/multiprocess/connection.py\", line 254, in recv\n",
      "    return _ForkingPickler.loads(buf.getbuffer())\n",
      "  File \"/home/saqibm128/miniconda3/envs/dbmi/lib/python3.7/site-packages/dill/_dill.py\", line 317, in loads\n",
      "    return load(file, ignore)\n",
      "  File \"/home/saqibm128/miniconda3/envs/dbmi/lib/python3.7/site-packages/dill/_dill.py\", line 305, in load\n",
      "    obj = pik.load()\n",
      "_pickle.UnpicklingError: invalid load key, '3'.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = np.vstack([instance.data.mean(axis=0, keepdims=True) for instance in data1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 480)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_data.reshape(x_data.shape[0], -1).shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}