filtering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "35d3278f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.signal import filtfilt, butter\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "14bf6e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "fs = 32\n",
    "n = 0.5*fs\n",
    "f_high = 0.5\n",
    "cut_off = f_high / n\n",
    "order = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "73319980",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filtering(signal):\n",
    "    b, a = butter(order, cut_off, btype='high')\n",
    "    signal = filtfilt(b, a, signal)\n",
    "    signal = (signal - np.min(signal)) / (np.max(signal) - np.min(signal))\n",
    "    return signal\n",
    "\n",
    "def helper(row):\n",
    "    row = np.array(row)\n",
    "    annotation = row[-1]\n",
    "    row = row[:-1]\n",
    "    row = np.reshape(row, (21, 256))\n",
    "    return np.append(np.array([filtering(x) for x in row]), annotation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "792be6d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2min 4s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5367</th>\n",
       "      <th>5368</th>\n",
       "      <th>5369</th>\n",
       "      <th>5370</th>\n",
       "      <th>5371</th>\n",
       "      <th>5372</th>\n",
       "      <th>5373</th>\n",
       "      <th>5374</th>\n",
       "      <th>5375</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.000009</td>\n",
       "      <td>-0.000019</td>\n",
       "      <td>-0.000015</td>\n",
       "      <td>-0.000017</td>\n",
       "      <td>-0.000013</td>\n",
       "      <td>-0.000013</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>-1.235876e-05</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>-1.084643e-05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>5.716124e-06</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>-0.000002</td>\n",
       "      <td>-0.000005</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>-1.639757e-05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.000010</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>-7.676535e-06</td>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-6.131463e-06</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000030</td>\n",
       "      <td>-3.172449e-05</td>\n",
       "      <td>-0.000033</td>\n",
       "      <td>-0.000033</td>\n",
       "      <td>-0.000032</td>\n",
       "      <td>-0.000029</td>\n",
       "      <td>-0.000026</td>\n",
       "      <td>-0.000021</td>\n",
       "      <td>-1.479180e-05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-8.693177e-06</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-7.987241e-06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>3.677891e-07</td>\n",
       "      <td>-0.000004</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.000013</td>\n",
       "      <td>-0.000017</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>-0.000021</td>\n",
       "      <td>-2.046998e-05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>4.829108e-06</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>5.385752e-06</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000002</td>\n",
       "      <td>-1.148686e-06</td>\n",
       "      <td>-0.000001</td>\n",
       "      <td>-0.000001</td>\n",
       "      <td>-0.000003</td>\n",
       "      <td>-0.000004</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-8.882271e-06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>-6.366012e-09</td>\n",
       "      <td>-0.000003</td>\n",
       "      <td>-8.108131e-07</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-6.797160e-06</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.000005</td>\n",
       "      <td>-0.000003</td>\n",
       "      <td>1.910313e-07</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47937</th>\n",
       "      <td>0.000001</td>\n",
       "      <td>-0.000002</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>1.155063e-05</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>-2.783687e-06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>3.993222e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>4.748940e-06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47938</th>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>6.353275e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>2.921822e-06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>3.083500e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>2.141669e-06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47939</th>\n",
       "      <td>-0.000009</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>-1.861646e-05</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>-2.006562e-05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>6.776134e-06</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>6.069041e-06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47940</th>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000028</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>1.869774e-05</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>1.222512e-05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>4.734669e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>4.132867e-06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47941</th>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>1.733479e-05</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>1.592725e-05</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>4.544308e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>4.532323e-06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>47876 rows × 5377 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0         1         2         3         4         5         6  \\\n",
       "0     -0.000009 -0.000019 -0.000015 -0.000017 -0.000013 -0.000013 -0.000011   \n",
       "1     -0.000010 -0.000010 -0.000011 -0.000011 -0.000012 -0.000012 -0.000010   \n",
       "2     -0.000007 -0.000007 -0.000006 -0.000006 -0.000006 -0.000007 -0.000006   \n",
       "3      0.000012  0.000012  0.000012  0.000012  0.000011  0.000009  0.000008   \n",
       "4      0.000011  0.000008  0.000010  0.000009  0.000009  0.000003  0.000003   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "47937  0.000001 -0.000002  0.000001  0.000009  0.000015  0.000018  0.000019   \n",
       "47938 -0.000007 -0.000016 -0.000018 -0.000006  0.000009  0.000011  0.000009   \n",
       "47939 -0.000009 -0.000012 -0.000012 -0.000014 -0.000014 -0.000016 -0.000018   \n",
       "47940  0.000024  0.000028  0.000022  0.000025  0.000023  0.000020  0.000019   \n",
       "47941  0.000023  0.000023  0.000025  0.000024  0.000024  0.000022  0.000019   \n",
       "\n",
       "                  7         8             9  ...      5367          5368  \\\n",
       "0     -1.235876e-05 -0.000010 -1.084643e-05  ...  0.000009  5.716124e-06   \n",
       "1     -7.676535e-06 -0.000007 -6.131463e-06  ... -0.000030 -3.172449e-05   \n",
       "2     -8.693177e-06 -0.000006 -7.987241e-06  ...  0.000003  3.677891e-07   \n",
       "3      4.829108e-06  0.000007  5.385752e-06  ... -0.000002 -1.148686e-06   \n",
       "4     -6.366012e-09 -0.000003 -8.108131e-07  ... -0.000006 -6.797160e-06   \n",
       "...             ...       ...           ...  ...       ...           ...   \n",
       "47937  1.155063e-05  0.000003 -2.783687e-06  ...  0.000004  3.993222e-06   \n",
       "47938  6.353275e-06  0.000004  2.921822e-06  ...  0.000004  3.083500e-06   \n",
       "47939 -1.861646e-05 -0.000020 -2.006562e-05  ...  0.000006  6.776134e-06   \n",
       "47940  1.869774e-05  0.000016  1.222512e-05  ...  0.000005  4.734669e-06   \n",
       "47941  1.733479e-05  0.000016  1.592725e-05  ...  0.000005  4.544308e-06   \n",
       "\n",
       "           5369      5370      5371      5372      5373      5374  \\\n",
       "0      0.000002 -0.000002 -0.000005 -0.000008 -0.000011 -0.000014   \n",
       "1     -0.000033 -0.000033 -0.000032 -0.000029 -0.000026 -0.000021   \n",
       "2     -0.000004 -0.000008 -0.000013 -0.000017 -0.000020 -0.000021   \n",
       "3     -0.000001 -0.000001 -0.000003 -0.000004 -0.000006 -0.000007   \n",
       "4     -0.000008 -0.000008 -0.000008 -0.000006 -0.000005 -0.000003   \n",
       "...         ...       ...       ...       ...       ...       ...   \n",
       "47937  0.000004  0.000004  0.000004  0.000004  0.000005  0.000005   \n",
       "47938  0.000004  0.000004  0.000004  0.000002  0.000002  0.000002   \n",
       "47939  0.000006  0.000006  0.000007  0.000006  0.000006  0.000006   \n",
       "47940  0.000004  0.000004  0.000004  0.000004  0.000005  0.000004   \n",
       "47941  0.000004  0.000004  0.000004  0.000004  0.000004  0.000004   \n",
       "\n",
       "               5375  annotation  \n",
       "0     -1.639757e-05           0  \n",
       "1     -1.479180e-05           0  \n",
       "2     -2.046998e-05           0  \n",
       "3     -8.882271e-06           0  \n",
       "4      1.910313e-07           0  \n",
       "...             ...         ...  \n",
       "47937  4.748940e-06           0  \n",
       "47938  2.141669e-06           0  \n",
       "47939  6.069041e-06           0  \n",
       "47940  4.132867e-06           0  \n",
       "47941  4.532323e-06           0  \n",
       "\n",
       "[47876 rows x 5377 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "df = pd.read_csv('babydf8sec.csv', index_col = 0)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "6e08347b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "e8b40eb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5367</th>\n",
       "      <th>5368</th>\n",
       "      <th>5369</th>\n",
       "      <th>5370</th>\n",
       "      <th>5371</th>\n",
       "      <th>5372</th>\n",
       "      <th>5373</th>\n",
       "      <th>5374</th>\n",
       "      <th>5375</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.670140</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.301372</td>\n",
       "      <td>0.248988</td>\n",
       "      <td>0.523084</td>\n",
       "      <td>0.528678</td>\n",
       "      <td>0.705319</td>\n",
       "      <td>0.625819</td>\n",
       "      <td>0.783455</td>\n",
       "      <td>0.766413</td>\n",
       "      <td>...</td>\n",
       "      <td>0.500981</td>\n",
       "      <td>0.461603</td>\n",
       "      <td>0.426803</td>\n",
       "      <td>0.397114</td>\n",
       "      <td>0.386804</td>\n",
       "      <td>0.385916</td>\n",
       "      <td>0.395171</td>\n",
       "      <td>0.413661</td>\n",
       "      <td>0.448250</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.437015</td>\n",
       "      <td>0.456724</td>\n",
       "      <td>0.398784</td>\n",
       "      <td>0.364049</td>\n",
       "      <td>0.289042</td>\n",
       "      <td>0.302564</td>\n",
       "      <td>0.366020</td>\n",
       "      <td>0.477623</td>\n",
       "      <td>0.522965</td>\n",
       "      <td>0.530133</td>\n",
       "      <td>...</td>\n",
       "      <td>0.259276</td>\n",
       "      <td>0.255251</td>\n",
       "      <td>0.252752</td>\n",
       "      <td>0.253213</td>\n",
       "      <td>0.256566</td>\n",
       "      <td>0.262274</td>\n",
       "      <td>0.272007</td>\n",
       "      <td>0.284573</td>\n",
       "      <td>0.299356</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.455561</td>\n",
       "      <td>0.426729</td>\n",
       "      <td>0.506361</td>\n",
       "      <td>0.468753</td>\n",
       "      <td>0.407573</td>\n",
       "      <td>0.289235</td>\n",
       "      <td>0.433850</td>\n",
       "      <td>0.130699</td>\n",
       "      <td>0.363518</td>\n",
       "      <td>0.156740</td>\n",
       "      <td>...</td>\n",
       "      <td>0.643495</td>\n",
       "      <td>0.627201</td>\n",
       "      <td>0.584046</td>\n",
       "      <td>0.539931</td>\n",
       "      <td>0.491986</td>\n",
       "      <td>0.455622</td>\n",
       "      <td>0.449974</td>\n",
       "      <td>0.478404</td>\n",
       "      <td>0.531860</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.541197</td>\n",
       "      <td>0.577599</td>\n",
       "      <td>0.559663</td>\n",
       "      <td>0.577550</td>\n",
       "      <td>0.487971</td>\n",
       "      <td>0.382396</td>\n",
       "      <td>0.281893</td>\n",
       "      <td>0.006463</td>\n",
       "      <td>0.197752</td>\n",
       "      <td>0.067282</td>\n",
       "      <td>...</td>\n",
       "      <td>0.403617</td>\n",
       "      <td>0.450442</td>\n",
       "      <td>0.472292</td>\n",
       "      <td>0.493083</td>\n",
       "      <td>0.487609</td>\n",
       "      <td>0.476824</td>\n",
       "      <td>0.458209</td>\n",
       "      <td>0.460850</td>\n",
       "      <td>0.437825</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.588300</td>\n",
       "      <td>0.470346</td>\n",
       "      <td>0.550646</td>\n",
       "      <td>0.555319</td>\n",
       "      <td>0.557234</td>\n",
       "      <td>0.237077</td>\n",
       "      <td>0.268543</td>\n",
       "      <td>0.116349</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.119071</td>\n",
       "      <td>...</td>\n",
       "      <td>0.272985</td>\n",
       "      <td>0.261871</td>\n",
       "      <td>0.254642</td>\n",
       "      <td>0.255609</td>\n",
       "      <td>0.257484</td>\n",
       "      <td>0.275836</td>\n",
       "      <td>0.293726</td>\n",
       "      <td>0.328193</td>\n",
       "      <td>0.365759</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47871</th>\n",
       "      <td>0.000001</td>\n",
       "      <td>-0.000002</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>-0.000003</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47872</th>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47873</th>\n",
       "      <td>-0.000009</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>-0.000019</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47874</th>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000028</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47875</th>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>47876 rows × 5377 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0         1         2         3         4         5         6  \\\n",
       "0      0.670140  0.000000  0.301372  0.248988  0.523084  0.528678  0.705319   \n",
       "1      0.437015  0.456724  0.398784  0.364049  0.289042  0.302564  0.366020   \n",
       "2      0.455561  0.426729  0.506361  0.468753  0.407573  0.289235  0.433850   \n",
       "3      0.541197  0.577599  0.559663  0.577550  0.487971  0.382396  0.281893   \n",
       "4      0.588300  0.470346  0.550646  0.555319  0.557234  0.237077  0.268543   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "47871  0.000001 -0.000002  0.000001  0.000009  0.000015  0.000018  0.000019   \n",
       "47872 -0.000007 -0.000016 -0.000018 -0.000006  0.000009  0.000011  0.000009   \n",
       "47873 -0.000009 -0.000012 -0.000012 -0.000014 -0.000014 -0.000016 -0.000018   \n",
       "47874  0.000024  0.000028  0.000022  0.000025  0.000023  0.000020  0.000019   \n",
       "47875  0.000023  0.000023  0.000025  0.000024  0.000024  0.000022  0.000019   \n",
       "\n",
       "              7         8         9  ...      5367      5368      5369  \\\n",
       "0      0.625819  0.783455  0.766413  ...  0.500981  0.461603  0.426803   \n",
       "1      0.477623  0.522965  0.530133  ...  0.259276  0.255251  0.252752   \n",
       "2      0.130699  0.363518  0.156740  ...  0.643495  0.627201  0.584046   \n",
       "3      0.006463  0.197752  0.067282  ...  0.403617  0.450442  0.472292   \n",
       "4      0.116349  0.000000  0.119071  ...  0.272985  0.261871  0.254642   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "47871  0.000012  0.000003 -0.000003  ...  0.000004  0.000004  0.000004   \n",
       "47872  0.000006  0.000004  0.000003  ...  0.000004  0.000003  0.000004   \n",
       "47873 -0.000019 -0.000020 -0.000020  ...  0.000006  0.000007  0.000006   \n",
       "47874  0.000019  0.000016  0.000012  ...  0.000005  0.000005  0.000004   \n",
       "47875  0.000017  0.000016  0.000016  ...  0.000005  0.000005  0.000004   \n",
       "\n",
       "           5370      5371      5372      5373      5374      5375  annotation  \n",
       "0      0.397114  0.386804  0.385916  0.395171  0.413661  0.448250           0  \n",
       "1      0.253213  0.256566  0.262274  0.272007  0.284573  0.299356           0  \n",
       "2      0.539931  0.491986  0.455622  0.449974  0.478404  0.531860           0  \n",
       "3      0.493083  0.487609  0.476824  0.458209  0.460850  0.437825           0  \n",
       "4      0.255609  0.257484  0.275836  0.293726  0.328193  0.365759           0  \n",
       "...         ...       ...       ...       ...       ...       ...         ...  \n",
       "47871  0.000004  0.000004  0.000004  0.000005  0.000005  0.000005           0  \n",
       "47872  0.000004  0.000004  0.000002  0.000002  0.000002  0.000002           0  \n",
       "47873  0.000006  0.000007  0.000006  0.000006  0.000006  0.000006           0  \n",
       "47874  0.000004  0.000004  0.000004  0.000005  0.000004  0.000004           0  \n",
       "47875  0.000004  0.000004  0.000004  0.000004  0.000004  0.000005           0  \n",
       "\n",
       "[47876 rows x 5377 columns]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "f0c6f978",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 7min 56s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        [0.6649237661014609, 0.0, 0.3028386054808248, ...\n",
       "1        [0.4407083781663414, 0.4590053742702377, 0.399...\n",
       "2        [0.45968698053695234, 0.43178341787212926, 0.5...\n",
       "3        [0.5252538065298004, 0.5604664046848968, 0.542...\n",
       "4        [0.5962124910305, 0.47625070220495735, 0.55426...\n",
       "                               ...                        \n",
       "47871    [0.4091193076783833, 0.314725876741884, 0.3771...\n",
       "47872    [0.6072781974101849, 0.4277924916112085, 0.390...\n",
       "47873    [0.610636635513714, 0.5780281940813419, 0.5786...\n",
       "47874    [0.4155364061108011, 0.5407113678062606, 0.423...\n",
       "47875    [0.5847153054963691, 0.5933823646814637, 0.685...\n",
       "Length: 47876, dtype: object"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_new = df.apply(lambda row: helper(row), axis = 1)\n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "997d7088",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5367</th>\n",
       "      <th>5368</th>\n",
       "      <th>5369</th>\n",
       "      <th>5370</th>\n",
       "      <th>5371</th>\n",
       "      <th>5372</th>\n",
       "      <th>5373</th>\n",
       "      <th>5374</th>\n",
       "      <th>5375</th>\n",
       "      <th>5376</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.664924</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.302839</td>\n",
       "      <td>0.252895</td>\n",
       "      <td>0.527977</td>\n",
       "      <td>0.535224</td>\n",
       "      <td>0.712620</td>\n",
       "      <td>0.634505</td>\n",
       "      <td>0.792404</td>\n",
       "      <td>0.775981</td>\n",
       "      <td>...</td>\n",
       "      <td>0.460246</td>\n",
       "      <td>0.421414</td>\n",
       "      <td>0.388102</td>\n",
       "      <td>0.360909</td>\n",
       "      <td>0.354249</td>\n",
       "      <td>0.358163</td>\n",
       "      <td>0.373431</td>\n",
       "      <td>0.399195</td>\n",
       "      <td>0.442411</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.440708</td>\n",
       "      <td>0.459005</td>\n",
       "      <td>0.399321</td>\n",
       "      <td>0.363132</td>\n",
       "      <td>0.286574</td>\n",
       "      <td>0.299242</td>\n",
       "      <td>0.362306</td>\n",
       "      <td>0.473967</td>\n",
       "      <td>0.519115</td>\n",
       "      <td>0.526008</td>\n",
       "      <td>...</td>\n",
       "      <td>0.259859</td>\n",
       "      <td>0.255532</td>\n",
       "      <td>0.252822</td>\n",
       "      <td>0.253228</td>\n",
       "      <td>0.256703</td>\n",
       "      <td>0.262711</td>\n",
       "      <td>0.273009</td>\n",
       "      <td>0.286394</td>\n",
       "      <td>0.302264</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.459687</td>\n",
       "      <td>0.431783</td>\n",
       "      <td>0.512529</td>\n",
       "      <td>0.475749</td>\n",
       "      <td>0.415288</td>\n",
       "      <td>0.297484</td>\n",
       "      <td>0.443092</td>\n",
       "      <td>0.139937</td>\n",
       "      <td>0.373754</td>\n",
       "      <td>0.166984</td>\n",
       "      <td>...</td>\n",
       "      <td>0.621134</td>\n",
       "      <td>0.604807</td>\n",
       "      <td>0.561696</td>\n",
       "      <td>0.518022</td>\n",
       "      <td>0.470900</td>\n",
       "      <td>0.435932</td>\n",
       "      <td>0.432481</td>\n",
       "      <td>0.463951</td>\n",
       "      <td>0.521185</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.525254</td>\n",
       "      <td>0.560466</td>\n",
       "      <td>0.542530</td>\n",
       "      <td>0.559965</td>\n",
       "      <td>0.472027</td>\n",
       "      <td>0.368519</td>\n",
       "      <td>0.270110</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.188318</td>\n",
       "      <td>0.060686</td>\n",
       "      <td>...</td>\n",
       "      <td>0.392979</td>\n",
       "      <td>0.447347</td>\n",
       "      <td>0.474633</td>\n",
       "      <td>0.500463</td>\n",
       "      <td>0.497595</td>\n",
       "      <td>0.488453</td>\n",
       "      <td>0.470204</td>\n",
       "      <td>0.474017</td>\n",
       "      <td>0.449265</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.596212</td>\n",
       "      <td>0.476251</td>\n",
       "      <td>0.554268</td>\n",
       "      <td>0.557270</td>\n",
       "      <td>0.557858</td>\n",
       "      <td>0.237756</td>\n",
       "      <td>0.268382</td>\n",
       "      <td>0.116216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.118620</td>\n",
       "      <td>...</td>\n",
       "      <td>0.270655</td>\n",
       "      <td>0.257617</td>\n",
       "      <td>0.248791</td>\n",
       "      <td>0.248765</td>\n",
       "      <td>0.249969</td>\n",
       "      <td>0.268813</td>\n",
       "      <td>0.287631</td>\n",
       "      <td>0.324390</td>\n",
       "      <td>0.365077</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47871</th>\n",
       "      <td>0.409119</td>\n",
       "      <td>0.314726</td>\n",
       "      <td>0.377140</td>\n",
       "      <td>0.580123</td>\n",
       "      <td>0.721367</td>\n",
       "      <td>0.805053</td>\n",
       "      <td>0.809710</td>\n",
       "      <td>0.599189</td>\n",
       "      <td>0.347885</td>\n",
       "      <td>0.192095</td>\n",
       "      <td>...</td>\n",
       "      <td>0.386580</td>\n",
       "      <td>0.445764</td>\n",
       "      <td>0.414810</td>\n",
       "      <td>0.399375</td>\n",
       "      <td>0.436431</td>\n",
       "      <td>0.488679</td>\n",
       "      <td>0.532678</td>\n",
       "      <td>0.578072</td>\n",
       "      <td>0.496491</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47872</th>\n",
       "      <td>0.607278</td>\n",
       "      <td>0.427792</td>\n",
       "      <td>0.390650</td>\n",
       "      <td>0.615235</td>\n",
       "      <td>0.886784</td>\n",
       "      <td>0.911227</td>\n",
       "      <td>0.875492</td>\n",
       "      <td>0.828515</td>\n",
       "      <td>0.792090</td>\n",
       "      <td>0.761711</td>\n",
       "      <td>...</td>\n",
       "      <td>0.661060</td>\n",
       "      <td>0.643501</td>\n",
       "      <td>0.695311</td>\n",
       "      <td>0.714864</td>\n",
       "      <td>0.716173</td>\n",
       "      <td>0.658541</td>\n",
       "      <td>0.644344</td>\n",
       "      <td>0.659286</td>\n",
       "      <td>0.679108</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47873</th>\n",
       "      <td>0.610637</td>\n",
       "      <td>0.578028</td>\n",
       "      <td>0.578676</td>\n",
       "      <td>0.563010</td>\n",
       "      <td>0.572841</td>\n",
       "      <td>0.541571</td>\n",
       "      <td>0.517731</td>\n",
       "      <td>0.509364</td>\n",
       "      <td>0.499299</td>\n",
       "      <td>0.503451</td>\n",
       "      <td>...</td>\n",
       "      <td>0.722861</td>\n",
       "      <td>0.752414</td>\n",
       "      <td>0.720291</td>\n",
       "      <td>0.704067</td>\n",
       "      <td>0.719866</td>\n",
       "      <td>0.703545</td>\n",
       "      <td>0.690849</td>\n",
       "      <td>0.661180</td>\n",
       "      <td>0.667561</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47874</th>\n",
       "      <td>0.415536</td>\n",
       "      <td>0.540711</td>\n",
       "      <td>0.423628</td>\n",
       "      <td>0.523628</td>\n",
       "      <td>0.509501</td>\n",
       "      <td>0.470432</td>\n",
       "      <td>0.472035</td>\n",
       "      <td>0.510317</td>\n",
       "      <td>0.467849</td>\n",
       "      <td>0.414331</td>\n",
       "      <td>...</td>\n",
       "      <td>0.726438</td>\n",
       "      <td>0.612115</td>\n",
       "      <td>0.531106</td>\n",
       "      <td>0.420183</td>\n",
       "      <td>0.478467</td>\n",
       "      <td>0.440619</td>\n",
       "      <td>0.583377</td>\n",
       "      <td>0.511047</td>\n",
       "      <td>0.458242</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47875</th>\n",
       "      <td>0.584715</td>\n",
       "      <td>0.593382</td>\n",
       "      <td>0.685937</td>\n",
       "      <td>0.637071</td>\n",
       "      <td>0.639815</td>\n",
       "      <td>0.594950</td>\n",
       "      <td>0.486056</td>\n",
       "      <td>0.433513</td>\n",
       "      <td>0.396899</td>\n",
       "      <td>0.399156</td>\n",
       "      <td>...</td>\n",
       "      <td>0.548029</td>\n",
       "      <td>0.502884</td>\n",
       "      <td>0.270958</td>\n",
       "      <td>0.268335</td>\n",
       "      <td>0.345789</td>\n",
       "      <td>0.439109</td>\n",
       "      <td>0.382391</td>\n",
       "      <td>0.411520</td>\n",
       "      <td>0.419557</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>47876 rows × 5377 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6     \\\n",
       "0      0.664924  0.000000  0.302839  0.252895  0.527977  0.535224  0.712620   \n",
       "1      0.440708  0.459005  0.399321  0.363132  0.286574  0.299242  0.362306   \n",
       "2      0.459687  0.431783  0.512529  0.475749  0.415288  0.297484  0.443092   \n",
       "3      0.525254  0.560466  0.542530  0.559965  0.472027  0.368519  0.270110   \n",
       "4      0.596212  0.476251  0.554268  0.557270  0.557858  0.237756  0.268382   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "47871  0.409119  0.314726  0.377140  0.580123  0.721367  0.805053  0.809710   \n",
       "47872  0.607278  0.427792  0.390650  0.615235  0.886784  0.911227  0.875492   \n",
       "47873  0.610637  0.578028  0.578676  0.563010  0.572841  0.541571  0.517731   \n",
       "47874  0.415536  0.540711  0.423628  0.523628  0.509501  0.470432  0.472035   \n",
       "47875  0.584715  0.593382  0.685937  0.637071  0.639815  0.594950  0.486056   \n",
       "\n",
       "           7         8         9     ...      5367      5368      5369  \\\n",
       "0      0.634505  0.792404  0.775981  ...  0.460246  0.421414  0.388102   \n",
       "1      0.473967  0.519115  0.526008  ...  0.259859  0.255532  0.252822   \n",
       "2      0.139937  0.373754  0.166984  ...  0.621134  0.604807  0.561696   \n",
       "3      0.000000  0.188318  0.060686  ...  0.392979  0.447347  0.474633   \n",
       "4      0.116216  0.000000  0.118620  ...  0.270655  0.257617  0.248791   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "47871  0.599189  0.347885  0.192095  ...  0.386580  0.445764  0.414810   \n",
       "47872  0.828515  0.792090  0.761711  ...  0.661060  0.643501  0.695311   \n",
       "47873  0.509364  0.499299  0.503451  ...  0.722861  0.752414  0.720291   \n",
       "47874  0.510317  0.467849  0.414331  ...  0.726438  0.612115  0.531106   \n",
       "47875  0.433513  0.396899  0.399156  ...  0.548029  0.502884  0.270958   \n",
       "\n",
       "           5370      5371      5372      5373      5374      5375  5376  \n",
       "0      0.360909  0.354249  0.358163  0.373431  0.399195  0.442411   0.0  \n",
       "1      0.253228  0.256703  0.262711  0.273009  0.286394  0.302264   0.0  \n",
       "2      0.518022  0.470900  0.435932  0.432481  0.463951  0.521185   0.0  \n",
       "3      0.500463  0.497595  0.488453  0.470204  0.474017  0.449265   0.0  \n",
       "4      0.248765  0.249969  0.268813  0.287631  0.324390  0.365077   0.0  \n",
       "...         ...       ...       ...       ...       ...       ...   ...  \n",
       "47871  0.399375  0.436431  0.488679  0.532678  0.578072  0.496491   0.0  \n",
       "47872  0.714864  0.716173  0.658541  0.644344  0.659286  0.679108   0.0  \n",
       "47873  0.704067  0.719866  0.703545  0.690849  0.661180  0.667561   0.0  \n",
       "47874  0.420183  0.478467  0.440619  0.583377  0.511047  0.458242   0.0  \n",
       "47875  0.268335  0.345789  0.439109  0.382391  0.411520  0.419557   0.0  \n",
       "\n",
       "[47876 rows x 5377 columns]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df = pd.DataFrame.from_dict(dict(zip(df_new.index, df_new.values))).T\n",
    "filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "ddcdb099",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 7min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "filtered_df.to_csv('filtered_babydf8sec.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}