plot-stall-duration-cdf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import pandas as pd\n",
    "import filterbyip as fbi\n",
    "import requests\n",
    "import asinfo\n",
    "import clusterbycolumn as cbc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>body {\n",
       "    margin: 0;\n",
       "    font-family: Helvetica;\n",
       "}\n",
       "table.dataframe {\n",
       "    border-collapse: collapse;\n",
       "    border: none;\n",
       "}\n",
       "table.dataframe tr {\n",
       "    border: none;\n",
       "}\n",
       "table.dataframe td, table.dataframe th {\n",
       "    margin: 0;\n",
       "    border: 1px solid white;\n",
       "    padding-left: 0.25em;\n",
       "    padding-right: 0.25em;\n",
       "}\n",
       "table.dataframe th:not(:empty) {\n",
       "    background-color: #fec;\n",
       "    text-align: left;\n",
       "    font-weight: normal;\n",
       "}\n",
       "table.dataframe tr:nth-child(2) th:empty {\n",
       "    border-left: none;\n",
       "    border-right: 1px dashed #888;\n",
       "}\n",
       "table.dataframe td {\n",
       "    border: 2px solid #ccf;\n",
       "    background-color: #f4f4ff;\n",
       "}\n",
       "h3 {\n",
       "    color: white;\n",
       "    background-color: black;\n",
       "    padding: 0.5em;\n",
       "}\n",
       "\n",
       ".CodeMirror {\n",
       "    font-family: 'Monaco for Powerline';\n",
       "}\n",
       "\n",
       "div.cell.code_cell {\n",
       "    font-family: 'Monaco for Powerline';\n",
       "}\n",
       "\n",
       "div.output_area pre {\n",
       "    font-family: 'Monaco for Powerline';\n",
       "}\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.display import HTML\n",
    "css = open('style-table.css').read() + open('style-notebook.css').read()\n",
    "HTML('<style>{}</style>'.format(css))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def asn_from_probe_name(probe_name):\n",
    "    base = 'http://leone-collector.eecs.jacobs-university.de/msmpoint'\n",
    "    msm_metadata = requests.get('%s/%s'%(base, probe_name))\n",
    "    asn4 = str(msm_metadata.json()['asn4'])\n",
    "    asn6 = str(msm_metadata.json()['asn6'])\n",
    "    return asn4,asn6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dailydb_location = '/data/vbajpai/datasets/youtube-data-collection-python3/youtube.db'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = '''select   msmpoint.name as probe\n",
    "                  , dtime\n",
    "                  , video_address\n",
    "                  , stall_duration_total\n",
    "                  \n",
    "           from youtube, msmpoint\n",
    "           where youtube.unit_id = msmpoint.skunitid\n",
    "           and msmpoint.skunitid <> ''\n",
    "           \n",
    "           and successes = 1\n",
    "           and failures = 0\n",
    "           \n",
    "           and stall_events <> 0\n",
    "           \n",
    "           order by dtime DESC\n",
    "        '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "con = sqlite3.connect(dailydb_location)\n",
    "df = pd.read_sql(query, con)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probe</th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_address</th>\n",
       "      <th>stall_duration_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:24:06</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:24:06</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:24:06</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:24:06</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:24:06</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       probe                dtime  video_address  stall_duration_total\n",
       "0  samsbox88  2017-04-26 23:24:06  173.194.5.201              31840282\n",
       "1  samsbox88  2017-04-26 23:24:06  173.194.5.201              31840282\n",
       "2  samsbox88  2017-04-26 23:24:06  173.194.5.201              31840282\n",
       "3  samsbox88  2017-04-26 23:24:06  173.194.5.201              31840282\n",
       "4  samsbox88  2017-04-26 23:24:06  173.194.5.201              31840282"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_probes = len(set(df['probe']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def round_time_to_nearest_hour(dtime):\n",
    "    import datetime;    \n",
    "    d = datetime.datetime.strptime(dtime, \"%Y-%m-%d %H:%M:%S\")\n",
    "    d = d.replace(minute=0, second=0)\n",
    "    dtime = d.strftime('%Y-%m-%d %H:%M:%S')\n",
    "    return dtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['dtime'] = df['dtime'].apply(round_time_to_nearest_hour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probe</th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_address</th>\n",
       "      <th>stall_duration_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       probe                dtime  video_address  stall_duration_total\n",
       "0  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "1  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "2  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "3  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "4  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probe</th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_address</th>\n",
       "      <th>stall_duration_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1868</th>\n",
       "      <td>samsbox10</td>\n",
       "      <td>2014-08-21 15:00:00</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>55735572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1869</th>\n",
       "      <td>samsbox07</td>\n",
       "      <td>2014-08-21 14:00:00</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>55867638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1870</th>\n",
       "      <td>samsbox10</td>\n",
       "      <td>2014-08-21 14:00:00</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>56905588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1871</th>\n",
       "      <td>samsbox07</td>\n",
       "      <td>2014-08-21 13:00:00</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>56793754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1872</th>\n",
       "      <td>samsbox10</td>\n",
       "      <td>2014-08-21 13:00:00</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>56835570</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          probe                dtime         video_address  \\\n",
       "1868  samsbox10  2014-08-21 15:00:00  2a00:1450:4009:36::e   \n",
       "1869  samsbox07  2014-08-21 14:00:00  2a00:1450:4009:36::e   \n",
       "1870  samsbox10  2014-08-21 14:00:00  2a00:1450:4009:36::e   \n",
       "1871  samsbox07  2014-08-21 13:00:00  2a00:1450:4009:36::e   \n",
       "1872  samsbox10  2014-08-21 13:00:00  2a00:1450:4009:36::e   \n",
       "\n",
       "      stall_duration_total  \n",
       "1868              55735572  \n",
       "1869              55867638  \n",
       "1870              56905588  \n",
       "1871              56793754  \n",
       "1872              56835570  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "probe                   1873\n",
       "dtime                   1873\n",
       "video_address           1873\n",
       "stall_duration_total    1873\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def filter_by_ip(df, column):\n",
    "  \"\"\"reads a DataFrame, calls fbi.filter_v{4,6} to get DataFrames with rows\n",
    "     containing only v4, v6 endpoints. Returns the two DataFrame as a tuple\"\"\"\n",
    "\n",
    "  dfv6 = fbi.filter_v6(df, column)\n",
    "  dfv4 = fbi.filter_v4(df, column)\n",
    "  return dfv4, dfv6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfv4, dfv6 = filter_by_ip(df, column = 'video_address')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probe</th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_address</th>\n",
       "      <th>stall_duration_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>samsbox88</td>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       probe                dtime  video_address  stall_duration_total\n",
       "0  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "1  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "2  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "3  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282\n",
       "4  samsbox88  2017-04-26 23:00:00  173.194.5.201              31840282"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfv4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probe</th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_address</th>\n",
       "      <th>stall_duration_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>samsbox83</td>\n",
       "      <td>2017-04-23 14:00:00</td>\n",
       "      <td>2a00:1450:401c:1c::6</td>\n",
       "      <td>33781852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>samsbox83</td>\n",
       "      <td>2017-04-23 14:00:00</td>\n",
       "      <td>2a00:1450:401c:1c::6</td>\n",
       "      <td>33781852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>samsbox83</td>\n",
       "      <td>2017-04-23 14:00:00</td>\n",
       "      <td>2a00:1450:401c:1c::6</td>\n",
       "      <td>33781852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>samsbox83</td>\n",
       "      <td>2017-04-23 14:00:00</td>\n",
       "      <td>2a00:1450:401c:1c::6</td>\n",
       "      <td>33781852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>samsbox83</td>\n",
       "      <td>2017-04-23 14:00:00</td>\n",
       "      <td>2a00:1450:401c:1c::6</td>\n",
       "      <td>33781852</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       probe                dtime         video_address  stall_duration_total\n",
       "5  samsbox83  2017-04-23 14:00:00  2a00:1450:401c:1c::6              33781852\n",
       "6  samsbox83  2017-04-23 14:00:00  2a00:1450:401c:1c::6              33781852\n",
       "7  samsbox83  2017-04-23 14:00:00  2a00:1450:401c:1c::6              33781852\n",
       "8  samsbox83  2017-04-23 14:00:00  2a00:1450:401c:1c::6              33781852\n",
       "9  samsbox83  2017-04-23 14:00:00  2a00:1450:401c:1c::6              33781852"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfv6.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfv4['name'] = dfv4['probe'].apply(lambda name: \"#%02d\"%int(name[7:]))\n",
    "dfv6['name'] = dfv6['probe'].apply(lambda name: \"#%02d\"%int(name[7:]))\n",
    "del dfv4['probe']; del dfv6['probe']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtime                   1522\n",
       "video_address           1522\n",
       "stall_duration_total    1522\n",
       "name                    1522\n",
       "dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfv6.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtime                   351\n",
       "video_address           351\n",
       "stall_duration_total    351\n",
       "name                    351\n",
       "dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfv4.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(dfv4, dfv6, on=[\"name\", \"dtime\"], how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtime                     1715\n",
       "video_address_x            351\n",
       "stall_duration_total_x     351\n",
       "name                      1715\n",
       "video_address_y           1522\n",
       "stall_duration_total_y    1522\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.rename(columns={  'video_address_x'             : 'video_ep_4'\n",
    "                   , 'stall_duration_total_x'      : 'stall_total_4'             \n",
    "                   , 'video_address_y'             : 'video_ep_6'\n",
    "                   , 'stall_duration_total_y'      : 'stall_total_6'                                \n",
    "                  } ,  inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_ep_4</th>\n",
       "      <th>stall_total_4</th>\n",
       "      <th>name</th>\n",
       "      <th>video_ep_6</th>\n",
       "      <th>stall_total_6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 dtime     video_ep_4  stall_total_4 name video_ep_6  \\\n",
       "0  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "1  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "2  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "3  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "4  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "\n",
       "   stall_total_6  \n",
       "0            0.0  \n",
       "1            0.0  \n",
       "2            0.0  \n",
       "3            0.0  \n",
       "4            0.0  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['46_stall_total']  = df.stall_total_4.map(str) + ',' + df.stall_total_6.map(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Absolute Difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_difference(v4v6_stall_total):\n",
    "    \n",
    "    v4, v6 = v4v6_stall_total.split(',')\n",
    "    try: v4 = float(v4); v6 = float(v6)\n",
    "    except Exception as e: print(e, v4v6_stall_total)\n",
    "    \n",
    "    difference = (v4 - v6)\n",
    "    difference = difference / 1000000            \n",
    "    return float('%0.2f'%difference)\n",
    "\n",
    "df['st_a'] = df['46_stall_total'].apply(get_difference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del df['46_stall_total']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_ep_4</th>\n",
       "      <th>stall_total_4</th>\n",
       "      <th>name</th>\n",
       "      <th>video_ep_6</th>\n",
       "      <th>stall_total_6</th>\n",
       "      <th>st_a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1710</th>\n",
       "      <td>2014-08-21 15:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>#10</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>55735572.0</td>\n",
       "      <td>-55.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1711</th>\n",
       "      <td>2014-08-21 14:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>#07</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>55867638.0</td>\n",
       "      <td>-55.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1712</th>\n",
       "      <td>2014-08-21 14:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>#10</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>56905588.0</td>\n",
       "      <td>-56.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1713</th>\n",
       "      <td>2014-08-21 13:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>#07</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>56793754.0</td>\n",
       "      <td>-56.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1714</th>\n",
       "      <td>2014-08-21 13:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>#10</td>\n",
       "      <td>2a00:1450:4009:36::e</td>\n",
       "      <td>56835570.0</td>\n",
       "      <td>-56.84</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    dtime video_ep_4  stall_total_4 name  \\\n",
       "1710  2014-08-21 15:00:00          0            0.0  #10   \n",
       "1711  2014-08-21 14:00:00          0            0.0  #07   \n",
       "1712  2014-08-21 14:00:00          0            0.0  #10   \n",
       "1713  2014-08-21 13:00:00          0            0.0  #07   \n",
       "1714  2014-08-21 13:00:00          0            0.0  #10   \n",
       "\n",
       "                video_ep_6  stall_total_6   st_a  \n",
       "1710  2a00:1450:4009:36::e     55735572.0 -55.74  \n",
       "1711  2a00:1450:4009:36::e     55867638.0 -55.87  \n",
       "1712  2a00:1450:4009:36::e     56905588.0 -56.91  \n",
       "1713  2a00:1450:4009:36::e     56793754.0 -56.79  \n",
       "1714  2a00:1450:4009:36::e     56835570.0 -56.84  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dtime</th>\n",
       "      <th>video_ep_4</th>\n",
       "      <th>stall_total_4</th>\n",
       "      <th>name</th>\n",
       "      <th>video_ep_6</th>\n",
       "      <th>stall_total_6</th>\n",
       "      <th>st_a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-04-26 23:00:00</td>\n",
       "      <td>173.194.5.201</td>\n",
       "      <td>31840282.0</td>\n",
       "      <td>#88</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 dtime     video_ep_4  stall_total_4 name video_ep_6  \\\n",
       "0  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "1  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "2  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "3  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "4  2017-04-26 23:00:00  173.194.5.201     31840282.0  #88          0   \n",
       "\n",
       "   stall_total_6   st_a  \n",
       "0            0.0  31.84  \n",
       "1            0.0  31.84  \n",
       "2            0.0  31.84  \n",
       "3            0.0  31.84  \n",
       "4            0.0  31.84  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtime            1715\n",
       "video_ep_4       1715\n",
       "stall_total_4    1715\n",
       "name             1715\n",
       "video_ep_6       1715\n",
       "stall_total_6    1715\n",
       "st_a             1715\n",
       "dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import Cdf\n",
    "import Pmf\n",
    "\n",
    "sample = df['st_a'].values\n",
    "c      = Cdf.MakeCdfFromList(sample)\n",
    "cc     = [1 - p for x, p in c.Items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAACdCAYAAABCdkjpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHCZJREFUeJzt3Xu8VGW9x/HPdyOkgIiGiEBsyryRlWlHq6MHL+XJY16C\n01FQ00OlgNihvGRJtT3lNU073u2gHPNeogipmXrS7KRo5i0VMgMERQVEELZcf+eP5xmdPcyeWTN7\nLmtm/96v17yYWWs9a/32sPdv1jzrWb9HZoZzzrnG1FLvAJxzzpXPk7hzzjUwT+LOOdfAPIk751wD\n8yTunHMNzJO4c841ME/iriFJukXScknrJR2fs25SXPeupGn1ibCyJP2vpEPrHYdLH0/iruokfUzS\nA5KWxOR6fc76aZLaStmnmR1lZv2BR/KsuyyuO6+MWH8nqV3S25L+LunMUvdRDklthT5wzGx/M5tZ\ni1hcY/Ek7mrhRuBJYBDwQeDH9Q2nqO+a2VbAwcDXJH2j3gE51xlP4q4WPgzcb2brzWyDmc1N2lDS\nkZLmxLNjk/S76oXZkZm9SPjA+UaMpcM3Bkn7SZqX9fp3kr4u6XxJi2O8w+O6zSVdGb+NrI3rji8W\ng6Sr47eXjZL2y7P+RElzJb0h6ZeSts2OTdKnJT0uaYWkC7La7RTjXRr3f29575KrN0/irhYuBK6T\ndET2QkkjJS0HxgJnxGSyXNKxcf2BwNXAZKAPML7GcQM8AXxKkhJu/11gDfAhYH7W8muAEcAngc2B\nPwBI+nB8D84Axma9B2cCmNmJsWtoQe6BJH0ZmAIcCgwFXgduy9qkP+H9Gwd8CviWpNa47sfAy8BA\nwrej0xP+fC5lNqt3AK75mdmPJL0IXCRpEjDBzP5qZg8B/WNf8Dwza8tp+l3gQjO7B0DSmlrGHa0i\n/J30Trj9n83sBwCS/gdYIelDwBjgo2a2KK5bD2Bmfye8B23AcDM7voTYTgQuM7M5cZ/fAZZL2iWu\n7wMcZ2bPxfWLgcGED5fFwEHAPwP3mtkzJRzXpYifibuaMLNfArsAs4FHJQ1J0Gwv8ly4rLEtgXVm\ntgpIUi1uVuaJmf3QzJYB/wAsMrP5nTcryzBgXtbxVgFL43LiMZ/L2n4D0Cs+P4Xw7eAqYL6kiRWO\nzdWIJ3FXM2a2zsy+B7xKuGiYsYHOfxc3ZD3/YrViK2BP4PH4fB3QI2vdQSXs572fQ9IgQrdK7vpS\n/x4XAMOz9tuX0DWySddLFsF7/xc/JVyvmAycL+lLJR7fpYAncVdVkvrGi5P9FYwEPgI8lbXZAuBA\nSX0k9Za0a1w+m9BP3FPSj4HPAB+oYey7AWcCl8VFC4G9JPWSNB4YnXBXTwLDJH1W0lDgTkI3TfbP\nsgD4rKRt48/7iQT7vQaYJGlnSb0IQyr/L16QLfazTZbUamYbgIeAZYS+etdgPIm7WjiOcBFtGXA5\nMN7MnshafxnwFqGfdi6hGwXgZOBzcfnWhD7gYWzqingxMN9QwMzFwv8pId5zJa0A7iL0Od8cl/8c\n2AZ4k9BF8q0kOzOzeXHbmcADhAu9v835WW4FHgX+Rviw+Jc8u5oVf5YPx/1OB84Ffg0sArYH/i3h\nz9gKPCDpbeAZYHp8uAYjnxTCOecal5+JO+dcA/Mk7pxzDcyTuHPONTBP4s4518A8iTvnXAPzJO6c\ncw3Mk7hzzjUwT+LOOdfAPIk751wDq1kSlzQmFqZvS7j9FpKmSlom6U1JF0hKFG8XCtybP/zhD39U\n4VE1NaknLukuQj3mR0to9iNgV0KVtb7AHwl1kC9P0HZAqTE651wjqtWkEGeY2fOFJoLNFiuyfR0Y\na2ZvSxoM/Bk4iWRJ3DnXbGYMh1UJSrKrB9iG4ttlth2zPtmx+rTC4fOS7beGapLEzez5EpsMA7YC\nHo7TYk0GTgNelNTTzNZVOkbnXJmSJtfOqAf0Hlr6Pvq0hjZjDW6Ks+dlno+NPRiZ5/nWm8HNLbBx\nPRBfZ3o/Vs2Hf8uZB+S2vrB+VXk/42Z9ymuXZNdV23PXbAu0m9k7kr4G3EyovyxC0fvFuQ0knQCc\nEF96d4pzubqabEt11Hq4pZMU8093wcOHhee2IcSVvSxj5Ex46ND8+8j8LItmvb8s8zx72cKZhde/\ndh9IhPSi+BxY8n/vL8tY8lj+WIoZdECHl5KuLaH1WjPrdH7ZtCbxJcDaOHP3x81sqqRPEz4Wl+Zr\nYGbXEIrkI+mJfNs41xRu3ix5d0E9vfabztcNzZOY8y0bkmCyoextMs+zl2XvN9/6IflKtwODPp9n\n2QGbLivPGMIkHkmcQoFJwtOaxBcQEvZ5hMlyAUYCc7wrxTW9Wp8xV0tnydEBvGtmZyXZUNLkQuvr\nnsQlbUG4aPkrM5sCYGZrJF0H7Au8G6e0mgT8tH6ROldFN6n4Ni6/7PfuJoU+9txluc8z/yp7ytQs\nfVo3/T/p09r1WN9XyrDDgtvWZGYfSbOBnQjDDDcC7wK/NrOjJfUG/gL80sxOz2rTmzAS5QjCJLLT\ngNPNbGOC4z1hZp8uI9TqvxnOQeN0idRCVy5spnC0SCc6fCJIesvMtk7UsMi2TTk9mydxl1rd4Yy7\nsZJrreQm8eFx7tXiDYtsW/fuFOe6hUom71LGQRfbT74x0q7qzGyepHXAE8AkM/tToW0L7cuTuHPV\nVumz79wE7sm4Ub0DXAz8UtJU4Bwro2vEu1M6ar43w9VXJRP4WP/1bGCb/CJIWmZm20j6IHAd4f6W\nr5rZS6Xs2M/EnauWriZw71vuFsxsKXCYpEnAbEl3A48Aq83s+mLt/Uy8o+Z7M1x9lDv6xM+2m1W+\nM/FNRp1I+hDwTWBvoI+Z7Vlsx34m7lw1lJLAPXF3V4/kLjCzVwh1ohKrZT3xPSQ9LmmlpOclHZSg\njST9RNKrkl6TdIXU2eh851IiaTdKn1ZP4N2YmXVSFKY0NUnisbTsTOBWoB9wDnBnrI1SyFjgs8BH\ngX8EjgS+WMVQnauNseb93a4gSb9Isl2tzsS/DPQALopDaIYBjwP/XqTdlsCzZrbazF4mFMZ6s6qR\nOtcV3eFmHlcrhyTZqFZ94h8HHjEzk7QrsA7437i8kBuA4ySNJZSgvcPMZlc3VOeqzLtQHCDpB0U2\n2TzJfmqVxLcF3ogTPHybMEPPaXF5IauAPwA7AAcDP+xsQ68n7uouyVm4J3D3vlMI006+1sn6RD0l\ntUriS4A+wDjgFjNbK2n7uLyQ/wRazOxHkq4C/ijpJDPbpFCx1xN3deXdKK5044F/N7O83cqSDk+y\nk1r1iT9L6DrZ3cweiMtGxuWFnAzcDmBmbxK6VyZVK0jnSnaTkidwPwt3WczsZqBd0i6dbPLXJPup\nVRK/AxgBLJTUIukYwoiT6wAkDZa0UNKJOe1eAU6UtIWkQcDhwIs1itm5zs0Y7mffrsvM7HAzy5vT\nzGzvJPuoSRI3szXAocBXgBXAmcCXzeyNuEkP4ANs2r1zNNAKvAo8DTwFJJoNw7mquUml1772s3BX\nJX7bfUfN92a4yunKmbcn8e6ual/b/LZ754rpareJJ3CXQ9IBwFwzW9jVfXkSdy5Xpfq6vQqh69z9\ngElqB14ys93L3ZEncde9VWuuSz/7doVdShikMSc+yuZJ3HU/1RxV4snbJWBm/1GpfXkSd82tVsMA\nvevE1UnNkrikPYCrgV0I478nm9l9CdqNAtqA4UA7MMHMplcvUtfQaj1225O3q7OaJPGsUrQXAxcR\nxn/fKak13onZWbv9CYl/DPAg4db9PtWP2DWMet1w490mLiWKJnFJd5nZYVmvjzazG0s8TodStJKy\nS9FeUKDdWcCPzOz++HplfLjuJg13R3ridimU5Ex8n5zXlwKlJvGSS9FKaiHMM9cm6bfAnsCfgfFm\nlqimgGtA1RotUipP2K5BlNOdUs4pUTmlaAcCvYArgQmEkrRtwK+AT24SlJeibSwzhpd+63o1ed+2\na1DlJPFyTlHKKUX7VjzWg2b2IICkc4EzJA3MqrsSgvJStOmStiSdzRO2ayJJknh/ScsKvMbMtimy\nj2eBbwA9zGxqXDaSUFo2LzNbI2kuYURKRl9CV8xbCeJ2tZDmZJ3Nu0dck0qSxPevwHHuIIxMuS/2\ndY8lpxQtMJtwEfPqrHZXAN+X9N/Ay4QJlq83s3UViMkl0ShJOpufabtupGgSN7OHunqQeFZ9KHAV\n8H3COPEkpWgvI0yq/HvgXWAWoU/dVUojJukMT9auyUgaWmpRrESlaCUdSOgO2QPYClhOOHO+3Mwe\nLSPWqvJStFkaOUlneLJ2jS/RgBBJyxJ0T3eQZJz414HzgMsJQwtXAVsShvzdJembZnZLKQd1XZCG\n8dLV5H3Xrnsr+Q+86Jm4pBeA483ssTzr9gGuMrPdSj1wNTXlmXgzJG8/o3bdV/3OxIFB+RI4gJk9\nImloKQd0JWrE5O3J2rlylfwHnySJ94wXJTvbeY9SD+oSSnMC924P56rhtFIbJOlOmUfhbgYzs4+U\neuBqavjulEreet6ntbwLm56knaukus6xOZEwfdDc3BWSdgJ2qHhU3V0la4fkJnD1gDHrK7d/51xd\nJUnilwKf62TdCsJY7qKJvNx64rHtYOBJwkXUtiRtGpLPpu6cK1FLgm22MbPX860ws8VA0SupWfXE\nbwX6Ee68vFNSoQJYmbabxXbNffpYTgJXj5C8PYE7120lSeJ/k7RvvhWSRhJuhy+mQz1xwl2YmXri\nxZwPPEeYHdpljDXvFnHOJUri5wC3SxonqR+ApK1i6dfbgbMT7KPkeuLxOKOA/YDJCY7RffiZt3Mu\nSlI7ZXrsDjkX+Lmk1UBvYD5wUsL5LkuuJy5pR+ASYL9Ye6XgARq6nnjSrhRP3s65HIlqp7y3sfRR\nQuJ908xeKqHd2YR64s8C88zsAUmXAf3N7JhO2pwJfI9w1g7hg2MjoRDWvmb2bIHjNc4Qw6QJ3EeV\nONfI6jrE8D0xcSdO3lnKqSd+NlldNZKmET4A2so4fmPzM3CXEhs3bmThwoWsWrWq3qGkSs+ePRk4\ncCD9+vXr0n4k/cLMji2lTU1mu6f8euLOE7hLkSVLliCJnXfemZaWJJfUmp+Z0d7ezqJFiwC6msgP\nKbVBTf4XzGwNcCjwFcLY8jNJVk/cOZciy5cvZ7vttvMEnkUSvXv3ZsiQIbzxxhvFG7zfbqOkDdkP\nwsxpGzLrkuynZknTzJ4A8vZTm9krFJ40GTM7vgph1Veaa6M4l8eGDRvo2bNnvcNIpS222IJ160qa\ndOxs4KuEuYdfIvSbPw18opSd+MdpmnlXikuhYiPFuqtS3xcz+z4wnjDB+55mNg/YaGbzM48k+/Hu\ni7TyBO5c0zOzeyQdANwWb54smZ+JO+eaxs0330y/fv1oa2tLtP3EiRPp379/h0dLSwuvvPIKbW1t\nHHHEEe9te/vtt7PDDjuwfPnyisYcz7j3BXoRrg2WxJO4c64pHHbYYUydOpXPfOYzidtcccUVLF++\n/L3Hvffey/bbb8+QIUM6bPfSSy8xYcIEbrvtNvr371/p0DGztWY2wcx6Z5ZJ+liStp7E06hPa70j\ncK7hnHfeedx///0MHjy4rPZr1qxh4sSJXHrppR1G37S3tzN69GjOOuss9txzz0qFm8Tvk2xUsyQu\naQ9Jj0taKel5SQclaDNa0kOSlkpaLOkGSb2LtWt4PrWZa0YzhocRWdmPGcMrtvsRI0Z0qf0pp5zC\n7rvvzqhRozosP+mkkxgxYgQTJkzo0v7LkOhKaU0ubGaVor0YuAg4mlCKttXM3izQ9F1gnJn9TdIw\n4BHgOODKasfsnEtgw1pYNS/Ztqvmw5fmdFw2a2dYscl8Mx31GQ49epUTXWIzZ87k7rvv5umnn+6w\nfN68ecyaNYsZM2ZU/JiSNtJ5qQ8VWNdBrUandChFGxNyphTtBZ01MrNfZz1fIGkZsLrawVadjw93\nLjXeeecdxo8fzw033MCWW27ZYd3w4cM55phjGDduHE8++eQmfeVddA/wJ2BqnnUCnkqyk1ol8bJK\n0WaTlClHe2sV4nPOlaNHL+i3U/Lt821bSvsqmDZtGq2trey///5515966qnMnj2b0aNH8/DDD9Or\nV8W+FYwjzFh2oZmtyF0pKdGZeK36xHNL0f6MkMiLzuwDIOk4wqD4g83s3U62OUHSE5KeoNFK0Trn\nqqq9vZ1ddtmFKVOmbLLuscceY5999inY/tprr2XlypWcfPLJFYspzph2JLBFJ5uMS7KfWp2JLyGU\noh0H3GJmayVtH5cXFBP4t4GRnU0TB2Bm1xDufCImcudcmvRp3bQrsYIjsfbaay/mzp3L6tWraWlp\n4ZJLLuGQQw7hxhtvxMxYs2YNa9eu3aTdO++8Q+/ehcdL9O3bl+nTp7PXXnux9957M25covxalJk9\nUmDdHUn2UVI98XJJOopQivZ5Mzs5LnsWuMHMzi/Q7gvAz4HPmdmrJRwv3fXEi/WJ+92aLqVeeOEF\ndt1113qHkVoF3p+qXQirVXfKHcAIYKGkFknHkFOKVtJCSSfmtJsCXFBKAm94PkbcOVeCmnSnxOnV\nDgWuAr4PvEKyUrTDgJ9IOidr2a1mlpvsm4OfgTvXLUh6GJibeZjZneXuK9WlaM3sw9WOyznn6mAf\nYGdgDvAikP4k7pxz7j3bmFlFKml57RTnXElqMRiiEZXyvlQqgYMn8drzuzVdA+vRo0eps9d0G+3t\n7XWZ9ciTeC15AncNrn///rz++uts3Lix3qGkhpmxevVqFi1axMCBAxO1iUOsK7Kt94nXiidw1wQG\nDBjAwoULmTNnTvGNu5GePXuy3XbblTLT/dASdl9wW0/izrnEWlpaGDZsWL3DaAalXFgouG3Nkrik\nPYCrgV0I48Qnm9l9RdoMINxKfyCwCvipmV1Y7Vgrzs/CnXMd9ZS0L8nu5CzY7Z32euJXET6FtgM+\nAjwm6YXsErWpVkryVo/qxeGcS5ulwPUJt11caGVq64lLGgwcAXzCzN6NBbMeAU4C6pfEZwwPxe2z\n9Wnt+mw8Y9Z3rb1zrmGY2fBK7SvN9cRHAMvN7HlJmwP/CpwP/KJqUS6aVXybVfNh5MyOyx469P22\n5Yyh9dvtnXNlqlUSz60nfhJwGoXriW8LZGqrnEboitm2szaSTgBOiC83LyvKIV9K0v9hebbLtyws\nL2aseYe5c65saa4nvgRYG8/czczmSvpEZ22y64lXVb6k21ki9gTtnKuyWt3s8yyh62R3M3sgLhsZ\nl3fmL8BA4HQgMyKlWBvnnOtWUltPPNYQfxTYJm6zG3AscEWNYnbOudRLez3xEwkz+7xOmOX+HDPL\nuapYOZLupfLzcw4gwTR0KeGxVofHWh2NFOvmZrZbNXZck+nZurMuTBVXcx5rdXis1eGxBl4Ayznn\nGpgnceeca2CexKuv+sMeK8djrQ6PtTo8VrxP3DnnGpqfiTvnXAPzJO6ccw3Mk3iVSBol6RlJKyS9\nLmlUXN4i6XxJb0h6S9K1krZIQbyDJS2W1Ja1LFWxShot6SFJS2OsN0jqncZYY0x7SHpc0kpJz0s6\nqJ7xZJM0RdKzkt6W9LKkKXH5AEnT4/JXJZ1a71gzJH1B0gZJ+8XXqYs1/h5OkTQ//u2/LGl4VWM1\nM39U+AHsD7wJfJ7wQbklMCiumwT8FRgMbA08AfykzvFuBvweWAi0ZS1PVazAIcAO8fkwYAEwIaWx\n9gIWAacSCv8fQ7hhbdt6/37G+I4GBsTn+xOKte0K/Aq4nVBEbgSwEjgkBfEOBV4G3gb2i8tSFyvw\nQ+BJYMf4ejChblTVYq37L1MzPoCHgW92su4FYGJ8vi1wI7Ac6FXHeC8CrgSm5STx1MWaE/dTwHFp\njBU4klDMPzN44HvAQ8Dp9X7f8sT6QUJ56H8E1gMj4vIDgXuAu+scX0/gj/E9nQfsF5NjqmIFtgJW\nEOZAyF5e1Vi9O6XCJLUAewPPSfqtpGWSHpC0o6SewM6EJA/wXeBbQF/gQ3WKdxThj2JyzvLUxZpN\nUibeW1Maazk19Gsu/r5eA0wFtiB/Df96x3wR8KSZ3Zq1rLP5BuoZ68cJ37b6x67UJZKmAntUM1ZP\n4pU3kPBV+krgXGB7YDbh69QAwlfrNyQdCDxtoX7MBgrXVq8KSTsClwBfMbM1OatTFWs2SccB44GD\nzexd0hlrbg39nxESeV3fu2wxtqsJeWAS+Wv41zVmSUcCnyW8h9lSFyuhy+cDhA+dIwhF/lrj66rF\n6rPdl0nSxYSqirkeJ/QvPmhmD8ZtzwXOICQVIxT8GmtmX1OYDLoXVSzkUyDWrYC1wJPh75newMZ4\nljsyZbH+wcwOjwn828BIM3s9rltaj1iLKKeGfs3EBH4N4dvKV8xsvaSSavjXyDjC5Oqvx9/RfsAs\nQt/4mymLdRnQH7jQzF4GkPRfhLPu6r2v9ezratYH8CLw06zXgwnJsmdcdxPwsbhuNCnpZ2bTPvFU\nxQp8gdAnOriT9zxNsR4FPABcmrXsWeA79f5/jrGcBfwG2Cxr2WDgVUKJ6M3jskuBe+odb1aM83i/\nTzxVscaYNgCHZS0bC8ytZqzenVIdVwDHShoR+8DOAa43s3Vx3eeAZZK2JpyhTzWztfULt1Npi3UK\ncIGFWvO50hZrwRr69RR/J78DfMvM3puh2xqohn8aY40xzQBOl7S1pEGESW3+s6qx1vuTtRkfhD7G\nCwlf8xcR+h37ZK37CWEI4luEP+re9Y45xjaNjmfiqYoV+DuwinCGnXlcncZYY0yfJgx1fIcweuaL\n9f4/jnG1Erqf3s55L48l9NPeGde9RspG0xDPxOPz1MWaFdNK4G/A5GrH6rVTnHOugXl3inPONTBP\n4s4518A8iTvnXAPzJO6ccw3Mk7hzzjUwT+LOOdfAPIm7pqVgkKSalpeI9aOvkzRP0j8n2P6SWG/8\nO/FGHOcS8yTumtk+hBsr8tVi6ZSk/STNK+eA8W682YQ6Gnua2W+KtTGzycAY4EvALE/krhSexF0z\nG0OY6GJsLQ4Wz/hvAi4zs1PMbGnStmb2NHAQYQKRM6sUomtCnsRdU4oJdRRwHHBArCCYWTdQ0qxY\n7/ltSU/FmuRd9a+ESpDn5Imn6DHNrJ1QEvYUSf0qEI/rBjyJu2b1eeA5C+WAnyHMCpNxKqFM7VBC\n6dCvEqrPIekvhFKnwyQtj497Eh5zNHCrZRWVSnLMbGb2ODCfMBWdc0V5EnfNagxhijaAmwlzSmYs\nBnYjFO7vaWbPmNlGADP7GKFveoGZ9Y+PgxMecydCoat8Oj1mHs8Tamg7V5Qncdd04oXBLxImpoWQ\nxPeUtFN8fTGh3/mHwKuSflChESwqsK6UY4pQZdC5ojyJu2Z0CPCwma0AMLNXgEeIFzgtuM7MdiV0\ns5xImO4tYwPl/W3MIdQQ30SCY2YbQZhIwLmiPIm7ZnQUcF8cIz4oFue/l5jEJZ0oaec4SfCjhJrv\n2cP6FgCDJf1DHGu+R8LjTgeOzHeRNMExM9vtTaj3PSvxT+u6NU/irhntSJg/8rWsx9lxOcDWhJl3\n3iJMNPEiWbOsmNl84HuERPoWYUahJH4FtHeyfcFjAkjqDVwGXJz5FuFcMT4phHMVJOmThLk1bwDO\nNrM3E7b7FHA5sAb4lzjc0Lmi/EzcuQqKN+3sTTjz/lPC2+5/RrhJ6Nd4Ancl8jNx55xrYH4m7pxz\nDcyTuHPONTBP4s4518A8iTvnXAPzJO6ccw3Mk7hzzjUwT+LOOdfA/h/l42LoDgJK3wAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f40070157f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize = (5,1.5))\n",
    "plt.rcParams['font.size'] = 12\n",
    "ax = plt.gca()\n",
    "yticks = np.arange(0,1.1,0.2)\n",
    "\n",
    "plt.plot(c.xs, c.ps, marker='s', linewidth=0.3, markersize=5, fillstyle='none', color = 'orange')\n",
    "\n",
    "ax = fig.axes[0]\n",
    "ax.grid(False)\n",
    "ax.set_yticks(yticks)\n",
    "ax.set_xlim([-70,60])\n",
    "ax.set_ylim([0,1.05])\n",
    "ax.set_xscale('linear')\n",
    "ax.set_xlabel('∆st (s)', fontsize=12)\n",
    "ax.set_ylabel('CDF', fontsize=12)\n",
    "\n",
    "plt.legend([\"1.7K\"], fontsize=12, loc='lower right')\n",
    "\n",
    "ax.spines['right'].set_color('none')\n",
    "ax.spines['top'].set_color('none')\n",
    "ax.yaxis.set_ticks_position('left')\n",
    "ax.xaxis.set_ticks_position('bottom')\n",
    "ax.spines['bottom'].set_position(('axes', -0.05))\n",
    "ax.spines['left'].set_position(('axes', -0.05))\n",
    "\n",
    "ax3 = ax.twinx()\n",
    "ax3.spines['right'].set_color('none')\n",
    "ax3.spines['top'].set_color('none')\n",
    "ax3.spines['left'].set_color('none')\n",
    "ax3.spines['bottom'].set_color('none')\n",
    "ax3.yaxis.set_ticks_position('none')\n",
    "ax3.set_ylabel('[\\'14 - \\'17]')\n",
    "plt.setp(ax3.get_yticklabels(), visible=False)\n",
    "\n",
    "plt.title('Stall Durations', fontsize=12, y=1.05)\n",
    "plt.savefig('yt-stall-durations-absolute-difference.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CDF of absolute difference of stall duration between IPv4 and IPv6 over the entire duration. 14% of the samples experience shorter stall durations over IPv6 with 10% of them being atleast 1s shorter and 4% being atleast 10s shorter. On the contrary, 80% of the samples experience stall durations that are atleast 1s longer over IPv6 with half of them being atleast 50s longer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('yt-stall-durations.csv', 'w') as f: \n",
    "    f.write('values; percentile\\n')\n",
    "    for x, y in zip(c.xs, c.ps): f.write('%.2f; %.2f\\n'%(x,y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}