analyses/hello_world.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and have a quick look at the data with pandas, dask and spark\n",
    "\n",
    "\n",
    "* [Pandas](#Pandas)\n",
    "* [Dask](#Dask)\n",
    "* [Spark](#Spark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tldextract\n",
    "\n",
    "DATA_DIR = 'path to where you have extracted the data'\n",
    "PARQUET_FILE = DATA_DIR + 'sample'  # I ran this with sample data*\n",
    "\n",
    "def extract_domain(url):\n",
    "    \"\"\"Use tldextract to return the base domain from a url\"\"\"\n",
    "    try:\n",
    "        extracted = tldextract.extract(url)\n",
    "        return '{}.{}'.format(extracted.domain, extracted.suffix)\n",
    "    except Exception as e:\n",
    "        return 'ERROR'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<small>*could also be run with full data (but beware of pandas blowing up RAM)</small>\n",
    "\n",
    "\n",
    "# Pandas\n",
    "\n",
    "http://pandas.pydata.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>argument_0</th>\n",
       "      <th>argument_1</th>\n",
       "      <th>argument_2</th>\n",
       "      <th>argument_3</th>\n",
       "      <th>argument_4</th>\n",
       "      <th>argument_5</th>\n",
       "      <th>argument_6</th>\n",
       "      <th>argument_7</th>\n",
       "      <th>argument_8</th>\n",
       "      <th>arguments</th>\n",
       "      <th>...</th>\n",
       "      <th>script_line</th>\n",
       "      <th>script_loc_eval</th>\n",
       "      <th>script_url</th>\n",
       "      <th>symbol</th>\n",
       "      <th>time_stamp</th>\n",
       "      <th>value</th>\n",
       "      <th>value_1000</th>\n",
       "      <th>value_len</th>\n",
       "      <th>valid</th>\n",
       "      <th>errors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>57</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.name</td>\n",
       "      <td>2017-12-16 02:54:10.079</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>57</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.name</td>\n",
       "      <td>2017-12-16 02:54:10.080</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>57</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.document.cookie</td>\n",
       "      <td>2017-12-16 02:54:10.086</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>49</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.navigator.userAgent</td>\n",
       "      <td>2017-12-16 02:54:10.088</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>68</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>25</td>\n",
       "      <td></td>\n",
       "      <td>https://ajax.googleapis.com/ajax/libs/webfont/...</td>\n",
       "      <td>window.navigator.userAgent</td>\n",
       "      <td>2017-12-16 07:12:07.104</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>68</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  argument_0 argument_1 argument_2 argument_3 argument_4 argument_5  \\\n",
       "0                                                                     \n",
       "1                                                                     \n",
       "2                                                                     \n",
       "3                                                                     \n",
       "4                                                                     \n",
       "\n",
       "  argument_6 argument_7 argument_8 arguments  ...    script_line  \\\n",
       "0                                         {}  ...             57   \n",
       "1                                         {}  ...             57   \n",
       "2                                         {}  ...             57   \n",
       "3                                         {}  ...             49   \n",
       "4                                         {}  ...             25   \n",
       "\n",
       "  script_loc_eval                                         script_url  \\\n",
       "0                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "1                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "2                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "3                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "4                  https://ajax.googleapis.com/ajax/libs/webfont/...   \n",
       "\n",
       "                       symbol              time_stamp  \\\n",
       "0                 window.name 2017-12-16 02:54:10.079   \n",
       "1                 window.name 2017-12-16 02:54:10.080   \n",
       "2      window.document.cookie 2017-12-16 02:54:10.086   \n",
       "3  window.navigator.userAgent 2017-12-16 02:54:10.088   \n",
       "4  window.navigator.userAgent 2017-12-16 07:12:07.104   \n",
       "\n",
       "                                               value  \\\n",
       "0                                 fb_xdm_frame_https   \n",
       "1                                 fb_xdm_frame_https   \n",
       "2                                                      \n",
       "3  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...   \n",
       "4  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...   \n",
       "\n",
       "                                          value_1000 value_len valid errors  \n",
       "0                                 fb_xdm_frame_https        18  True         \n",
       "1                                 fb_xdm_frame_https        18  True         \n",
       "2                                                            0  True         \n",
       "3  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...        68  True         \n",
       "4  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...        68  True         \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_parquet(PARQUET_FILE, engine='pyarrow')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['argument_0', 'argument_1', 'argument_2', 'argument_3', 'argument_4',\n",
       "       'argument_5', 'argument_6', 'argument_7', 'argument_8', 'arguments',\n",
       "       'arguments_n_keys', 'call_id', 'call_stack', 'crawl_id', 'file_name',\n",
       "       'func_name', 'in_iframe', 'location', 'operation', 'script_col',\n",
       "       'script_line', 'script_loc_eval', 'script_url', 'symbol', 'time_stamp',\n",
       "       'value', 'value_1000', 'value_len', 'valid', 'errors'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['location_domain'] = df.location.apply(extract_domain)\n",
    "df['script_domain'] = df.script_url.apply(extract_domain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gap.com               736\n",
       "officeworks.com.au    708\n",
       "ufc.ca                518\n",
       "doubleclick.net       517\n",
       "disqus.com            479\n",
       "Name: location_domain, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.location_domain.value_counts().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "google-analytics.com    1317\n",
       "optimizely.com           573\n",
       "yandex.ru                422\n",
       "baidu.com                416\n",
       "doubleclick.net          407\n",
       "cloudfront.net           385\n",
       "moatads.com              287\n",
       "disquscdn.com            273\n",
       "fbcdn.net                256\n",
       "gap.com                  233\n",
       "Name: script_domain, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.script_domain.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many domains have google analytics and yandex.ru?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "google_analytics = df[df.script_domain == 'google-analytics.com']\n",
    "yandex = df[df.script_domain == 'yandex.ru']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vjav.com\n",
      "newchic.com\n",
      "zona.mobi\n"
     ]
    }
   ],
   "source": [
    "for location_domain in google_analytics.location_domain.unique():\n",
    "    if location_domain in list(yandex.location_domain.unique()):\n",
    "        print(location_domain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dask\n",
    "\n",
    "http://dask.pydata.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as dd\n",
    "from dask.diagnostics import ProgressBar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>argument_0</th>\n",
       "      <th>argument_1</th>\n",
       "      <th>argument_2</th>\n",
       "      <th>argument_3</th>\n",
       "      <th>argument_4</th>\n",
       "      <th>argument_5</th>\n",
       "      <th>argument_6</th>\n",
       "      <th>argument_7</th>\n",
       "      <th>argument_8</th>\n",
       "      <th>arguments</th>\n",
       "      <th>...</th>\n",
       "      <th>script_line</th>\n",
       "      <th>script_loc_eval</th>\n",
       "      <th>script_url</th>\n",
       "      <th>symbol</th>\n",
       "      <th>time_stamp</th>\n",
       "      <th>value</th>\n",
       "      <th>value_1000</th>\n",
       "      <th>value_len</th>\n",
       "      <th>valid</th>\n",
       "      <th>errors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>57</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.name</td>\n",
       "      <td>2017-12-16 02:54:10.079</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>57</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.name</td>\n",
       "      <td>2017-12-16 02:54:10.080</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>fb_xdm_frame_https</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>57</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.document.cookie</td>\n",
       "      <td>2017-12-16 02:54:10.086</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>49</td>\n",
       "      <td></td>\n",
       "      <td>https://staticxx.facebook.com/connect/xd_arbit...</td>\n",
       "      <td>window.navigator.userAgent</td>\n",
       "      <td>2017-12-16 02:54:10.088</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>68</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{}</td>\n",
       "      <td>...</td>\n",
       "      <td>25</td>\n",
       "      <td></td>\n",
       "      <td>https://ajax.googleapis.com/ajax/libs/webfont/...</td>\n",
       "      <td>window.navigator.userAgent</td>\n",
       "      <td>2017-12-16 07:12:07.104</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...</td>\n",
       "      <td>68</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  argument_0 argument_1 argument_2 argument_3 argument_4 argument_5  \\\n",
       "0                                                                     \n",
       "1                                                                     \n",
       "2                                                                     \n",
       "3                                                                     \n",
       "4                                                                     \n",
       "\n",
       "  argument_6 argument_7 argument_8 arguments  ...    script_line  \\\n",
       "0                                         {}  ...             57   \n",
       "1                                         {}  ...             57   \n",
       "2                                         {}  ...             57   \n",
       "3                                         {}  ...             49   \n",
       "4                                         {}  ...             25   \n",
       "\n",
       "  script_loc_eval                                         script_url  \\\n",
       "0                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "1                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "2                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "3                  https://staticxx.facebook.com/connect/xd_arbit...   \n",
       "4                  https://ajax.googleapis.com/ajax/libs/webfont/...   \n",
       "\n",
       "                       symbol              time_stamp  \\\n",
       "0                 window.name 2017-12-16 02:54:10.079   \n",
       "1                 window.name 2017-12-16 02:54:10.080   \n",
       "2      window.document.cookie 2017-12-16 02:54:10.086   \n",
       "3  window.navigator.userAgent 2017-12-16 02:54:10.088   \n",
       "4  window.navigator.userAgent 2017-12-16 07:12:07.104   \n",
       "\n",
       "                                               value  \\\n",
       "0                                 fb_xdm_frame_https   \n",
       "1                                 fb_xdm_frame_https   \n",
       "2                                                      \n",
       "3  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...   \n",
       "4  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...   \n",
       "\n",
       "                                          value_1000 value_len valid errors  \n",
       "0                                 fb_xdm_frame_https        18  True         \n",
       "1                                 fb_xdm_frame_https        18  True         \n",
       "2                                                            0  True         \n",
       "3  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...        68  True         \n",
       "4  Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko...        68  True         \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddf = dd.read_parquet(PARQUET_FILE, engine='pyarrow')\n",
    "ddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['argument_0', 'argument_1', 'argument_2', 'argument_3', 'argument_4',\n",
       "       'argument_5', 'argument_6', 'argument_7', 'argument_8', 'arguments',\n",
       "       'arguments_n_keys', 'call_id', 'call_stack', 'crawl_id', 'file_name',\n",
       "       'func_name', 'in_iframe', 'location', 'operation', 'script_col',\n",
       "       'script_line', 'script_loc_eval', 'script_url', 'symbol', 'time_stamp',\n",
       "       'value', 'value_1000', 'value_len', 'valid', 'errors'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddf.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What are people putting in the canvas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddf['location_domain'] = ddf.location.apply(extract_domain, meta=('x', 'str'))\n",
    "ddf['script_domain'] = ddf.script_url.apply(extract_domain, meta=('x', 'str'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.4s\n"
     ]
    }
   ],
   "source": [
    "fillTexts = ddf[ddf.symbol == 'CanvasRenderingContext2D.fillText']\n",
    "\n",
    "with ProgressBar():\n",
    "    fillTexts = fillTexts.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>argument_0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ð</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð§ââï¸</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð§ââï¸</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ðºð³</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ClientJS,org &lt;canvas&gt; 1.0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ðºâð³</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>!H71JCaj)]# 1@#</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Soft Ruddy Foothold 2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           argument_0\n",
       "ð                                8\n",
       "ð§ââï¸                       2\n",
       "ð§ââï¸                       2\n",
       "45                                  2\n",
       "38                                  2\n",
       "ðºð³                            2\n",
       "ClientJS,org <canvas> 1.0           2\n",
       "ðºâð³                         2\n",
       "!H71JCaj)]# 1@#                     1\n",
       "Soft Ruddy Foothold 2               1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What's being written to canvas\n",
    "pd.DataFrame(fillTexts.argument_0.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location_domain</th>\n",
       "      <th>script_domain</th>\n",
       "      <th>argument_0</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">bongacams.com</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">bongacams.com</th>\n",
       "      <th>ðºâð³</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ðºð³</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð§ââï¸</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð§ââï¸</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">facebook.com</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">fbcdn.net</th>\n",
       "      <th>38</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">gap.com</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">gap.com</th>\n",
       "      <th>!H71JCaj)]# 1@#</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Soft Ruddy Foothold 2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">syracuse.edu</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">syracuse.edu</th>\n",
       "      <th>ðºâð³</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ðºð³</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð§ââï¸</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ð§ââï¸</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>urcosme.com</th>\n",
       "      <th>amazonaws.com</th>\n",
       "      <th>ClientJS,org &lt;canvas&gt; 1.0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         0\n",
       "location_domain script_domain argument_0                  \n",
       "bongacams.com   bongacams.com ðºâð³                1\n",
       "                              ðºð³                   1\n",
       "                              ð§ââï¸              1\n",
       "                              ð§ââï¸              1\n",
       "facebook.com    fbcdn.net     38                         2\n",
       "                              45                         2\n",
       "                              ð                       8\n",
       "gap.com         gap.com       !H71JCaj)]# 1@#            1\n",
       "                              Soft Ruddy Foothold 2      1\n",
       "syracuse.edu    syracuse.edu  ðºâð³                1\n",
       "                              ðºð³                   1\n",
       "                              ð§ââï¸              1\n",
       "                              ð§ââï¸              1\n",
       "urcosme.com     amazonaws.com ClientJS,org <canvas> 1.0  2"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How does it vary by domain?\n",
    "pd.DataFrame(fillTexts.groupby(['location_domain', 'script_domain', 'argument_0']).size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spark\n",
    "\n",
    "https://spark.apache.org/docs/latest/api/python/pyspark.html\n",
    "\n",
    "We use [findspark](https://github.com/minrk/findspark) to set up spark."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "\n",
    "findspark.init('/opt/spark')  # Adjust for the location where you installed spark\n",
    "\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "sc = SparkContext(appName=\"Overscripted\")\n",
    "spark = SparkSession(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in parquet and display a row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0-----------------------------------------------------------------------------------------------------------------------------------------------------\n",
      " argument_0       |                                                                                                                                           \n",
      " argument_1       |                                                                                                                                           \n",
      " argument_2       |                                                                                                                                           \n",
      " argument_3       |                                                                                                                                           \n",
      " argument_4       |                                                                                                                                           \n",
      " argument_5       |                                                                                                                                           \n",
      " argument_6       |                                                                                                                                           \n",
      " argument_7       |                                                                                                                                           \n",
      " argument_8       |                                                                                                                                           \n",
      " arguments        | {}                                                                                                                                        \n",
      " arguments_n_keys | 0                                                                                                                                         \n",
      " call_id          | 1_028048bbce3f7816a5f1277ac3ac2372d6607581a77a4bfb7a1873ab.json__0                                                                        \n",
      " call_stack       |                                                                                                                                           \n",
      " crawl_id         | 1                                                                                                                                         \n",
      " file_name        | 1_028048bbce3f7816a5f1277ac3ac2372d6607581a77a4bfb7a1873ab.json                                                                           \n",
      " func_name        | a/<                                                                                                                                       \n",
      " in_iframe        | true                                                                                                                                      \n",
      " location         | https://staticxx.facebook.com/connect/xd_arbiter/r/lY4eZXm_YWu.js?version=42#channel=f30ef17b61f384&origin=http%3A%2F%2Fwww.ubitennis.com \n",
      " operation        | get                                                                                                                                       \n",
      " script_col       | 1802                                                                                                                                      \n",
      " script_line      | 57                                                                                                                                        \n",
      " script_loc_eval  |                                                                                                                                           \n",
      " script_url       | https://staticxx.facebook.com/connect/xd_arbiter/r/lY4eZXm_YWu.js?version=42#channel=f30ef17b61f384&origin=http%3A%2F%2Fwww.ubitennis.com \n",
      " symbol           | window.name                                                                                                                               \n",
      " time_stamp       | 2017-12-16 04:54:10.079                                                                                                                   \n",
      " value            | fb_xdm_frame_https                                                                                                                        \n",
      " value_1000       | fb_xdm_frame_https                                                                                                                        \n",
      " value_len        | 18                                                                                                                                        \n",
      " valid            | true                                                                                                                                      \n",
      " errors           |                                                                                                                                           \n",
      "only showing top 1 row\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sdf = spark.read.parquet(DATA_DIR + 'sample')\n",
    "sdf.show(1, vertical=True, truncate=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the distinct symbols and show highest counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------------------------------------------+\n",
      "|symbol                                                       |\n",
      "+-------------------------------------------------------------+\n",
      "|window.navigator.appVersion                                  |\n",
      "|window.navigator.product                                     |\n",
      "|window.screen.colorDepth                                     |\n",
      "|RTCPeerConnection.createOffer                                |\n",
      "|RTCPeerConnection.localDescription                           |\n",
      "|CanvasRenderingContext2D.font                                |\n",
      "|HTMLCanvasElement.nodeType                                   |\n",
      "|CanvasRenderingContext2D.createRadialGradient                |\n",
      "|RTCPeerConnection.onicecandidate                             |\n",
      "|window.navigator.buildID                                     |\n",
      "|CanvasRenderingContext2D.scale                               |\n",
      "|window.navigator.mimeTypes[application/futuresplash].suffixes|\n",
      "|RTCPeerConnection.setLocalDescription                        |\n",
      "|CanvasRenderingContext2D.fillText                            |\n",
      "|CanvasRenderingContext2D.fill                                |\n",
      "|window.navigator.appName                                     |\n",
      "|HTMLCanvasElement.style                                      |\n",
      "|HTMLCanvasElement.className                                  |\n",
      "|HTMLCanvasElement.getContext                                 |\n",
      "|window.navigator.doNotTrack                                  |\n",
      "+-------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sdf.select('symbol').distinct().show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------------------------+-----+\n",
      "|symbol                                               |count|\n",
      "+-----------------------------------------------------+-----+\n",
      "|window.document.cookie                               |3390 |\n",
      "|window.navigator.userAgent                           |1797 |\n",
      "|window.Storage.getItem                               |807  |\n",
      "|window.localStorage                                  |442  |\n",
      "|window.Storage.setItem                               |363  |\n",
      "|window.navigator.plugins[Shockwave Flash].description|238  |\n",
      "|window.sessionStorage                                |236  |\n",
      "|window.Storage.removeItem                            |185  |\n",
      "|window.name                                          |185  |\n",
      "|window.screen.colorDepth                             |166  |\n",
      "|window.navigator.appName                             |135  |\n",
      "|window.navigator.platform                            |120  |\n",
      "|window.navigator.language                            |96   |\n",
      "|window.navigator.plugins[Shockwave Flash].name       |90   |\n",
      "|window.Storage.length                                |78   |\n",
      "|window.navigator.appVersion                          |74   |\n",
      "|HTMLCanvasElement.width                              |53   |\n",
      "|window.Storage.key                                   |48   |\n",
      "|window.navigator.cookieEnabled                       |45   |\n",
      "|window.navigator.product                             |43   |\n",
      "+-----------------------------------------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sdf.groupBy('symbol').count().sort('count', ascending=False).show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}