Feb 18, 2020 Dask DataFrame and Bag Basic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Feb 18, 2020 Dask DataFrame and Bag Basic\n",
    "---\n",
    "* Name: Jikhan Jeong\n",
    "* Ref: https://docs.dask.org/en/latest/install.html (conda install dask)\n",
    "* Ref; https://datascienceschool.net/view-notebook/2282b75b2a63448087b77269885c27cb/ (Korean)\n",
    "* Ref: https://towardsdatascience.com/trying-out-dask-dataframes-in-python-for-fast-data-analysis-in-parallel-aa960c18a915 (English)\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/cahnrs/jikhan.jeong/amazon\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "# <font color =blue> Part 1: Basic </font>\n",
    "---\n",
    "* dask dataframe.compute() -> to pandas dataframe\n",
    "* Ref; https://datascienceschool.net/view-notebook/2282b75b2a63448087b77269885c27cb/ (Korean)\n",
    "----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as dd\n",
    "from dask.diagnostics import ProgressBar\n",
    "pbar = ProgressBar()\n",
    "pbar.register()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting data1.csv\n"
     ]
    }
   ],
   "source": [
    "%%writefile data1.csv\n",
    "time,temperature,humidity\n",
    "0,22,58\n",
    "1,21,57\n",
    "2,25,57\n",
    "3,26,55\n",
    "4,22,53\n",
    "5,23,59"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data1.csv  data2.csv  data3.csv\n"
     ]
    }
   ],
   "source": [
    "ls data*.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><strong>Dask DataFrame Structure:</strong></div>\n",
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>npartitions=1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>int64</td>\n",
       "      <td>int64</td>\n",
       "      <td>int64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "<div>Dask Name: from-delayed, 3 tasks</div>"
      ],
      "text/plain": [
       "Dask DataFrame Structure:\n",
       "                time temperature humidity\n",
       "npartitions=1                            \n",
       "               int64       int64    int64\n",
       "                 ...         ...      ...\n",
       "Dask Name: from-delayed, 3 tasks"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dd.read_csv(\"data1.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dask.dataframe.core.DataFrame"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time  temperature  humidity\n",
       "0     0           22        58"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dd.Scalar<series-..., dtype=float64>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.temperature.mean() # task so no results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "23.166666666666668"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.temperature.mean().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    22\n",
       "1    21\n",
       "2    25\n",
       "3    26\n",
       "4    22\n",
       "5    23\n",
       "Name: temperature, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.temperature.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    225\n",
       "1    215\n",
       "2    255\n",
       "3    265\n",
       "4    225\n",
       "5    235\n",
       "Name: temperature, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.temperature*10+5).compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> **assign** : change the input of dd.dataframe\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>220</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time  temperature  humidity\n",
       "0     0          220        58"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.assign(temperature = df.temperature*10)\n",
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>220</td>\n",
       "      <td>58</td>\n",
       "      <td>220jikhan is handsome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>210</td>\n",
       "      <td>57</td>\n",
       "      <td>210jikhan is handsome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>250</td>\n",
       "      <td>57</td>\n",
       "      <td>250jikhan is handsome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>260</td>\n",
       "      <td>55</td>\n",
       "      <td>260jikhan is handsome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>220</td>\n",
       "      <td>53</td>\n",
       "      <td>220jikhan is handsome</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time  temperature  humidity                  title\n",
       "0     0          220        58  220jikhan is handsome\n",
       "1     1          210        57  210jikhan is handsome\n",
       "2     2          250        57  250jikhan is handsome\n",
       "3     3          260        55  260jikhan is handsome\n",
       "4     4          220        53  220jikhan is handsome"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.assign(title = df.temperature.astype(str) + \"jikhan is handsome\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> Dataframe for multiple dataset </font>`\n",
    "---\n",
    "* read a file with while-card(*)\n",
    "* count().compute()\n",
    "* describe().compute()\n",
    "* E.g. df = dd.read_csv('data*.csv') # read data1, data2, data3\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting data2.csv\n"
     ]
    }
   ],
   "source": [
    "%%writefile data2.csv\n",
    "time,temperature,humidity\n",
    "0,22,58\n",
    "1,21,57\n",
    "2,25,57\n",
    "3,26,55\n",
    "4,22,53\n",
    "5,23,59"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting data3.csv\n"
     ]
    }
   ],
   "source": [
    "%%writefile data3.csv\n",
    "time,temperature,humidity\n",
    "0,22,58\n",
    "1,21,57\n",
    "2,25,57\n",
    "3,26,55\n",
    "4,22,53\n",
    "5,23,59"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data1.csv  data2.csv  data3.csv\n"
     ]
    }
   ],
   "source": [
    "ls data*.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/apps/anaconda3/19.10.0/lib/python3.7/site-packages/dask/dataframe/core.py:5738: UserWarning: Insufficient elements for `head`. 12 elements requested, only 6 elements available. Try passing larger `npartitions` to `head`.\n",
      "  warnings.warn(msg.format(n, len(r)))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time  temperature  humidity\n",
       "0     0           22        58\n",
       "1     1           21        57\n",
       "2     2           25        57\n",
       "3     3           26        55\n",
       "4     4           22        53\n",
       "5     5           23        59"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = dd.read_csv('data*.csv')\n",
    "df1.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time  temperature  humidity\n",
       "0     0           22        58\n",
       "1     1           21        57\n",
       "2     2           25        57\n",
       "3     3           26        55\n",
       "4     4           22        53\n",
       "5     5           23        59\n",
       "0     0           22        58\n",
       "1     1           21        57\n",
       "2     2           25        57\n",
       "3     3           26        55\n",
       "4     4           22        53\n",
       "5     5           23        59\n",
       "0     0           22        58\n",
       "1     1           21        57\n",
       "2     2           25        57\n",
       "3     3           26        55\n",
       "4     4           22        53\n",
       "5     5           23        59"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dask Series Structure:\n",
       "npartitions=1\n",
       "humidity    int64\n",
       "time          ...\n",
       "dtype: int64\n",
       "Dask Name: dataframe-count-agg, 13 tasks"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "time           18\n",
       "temperature    18\n",
       "humidity       18\n",
       "dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.count().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "count    18.000000\n",
       "mean     23.166667\n",
       "std       1.823055\n",
       "min      21.000000\n",
       "25%      22.000000\n",
       "50%      22.500000\n",
       "75%      24.500000\n",
       "max      26.000000\n",
       "Name: temperature, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.temperature.describe().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dask.dataframe.core.DataFrame"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1_pd = df1.compute()\n",
    "type(df1_pd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> Handle Bigdata with parallel approach with dask </font>\n",
    "---\n",
    "\n",
    "* Dataset: https://catalogtdatatgovudatasetucrimess2001stospresents398a4 (1.3GB) \n",
    "* (Window) * using **urllib.reques** instead of wget in Linux\n",
    "* (Linux) !wget -O crime.csv https://data.cityofchicago.org/api/views/ijzp-q8t2/rows.csv?accessType=DOWNLOAD (Linux)\n",
    "* Ref: https://stackoverflow.com/questions/57748687/downloading-files-in-jupyter-wget-on-windows\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('crime.csv', <http.client.HTTPMessage at 0x2b13fa0f8c90>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://data.cityofchicago.org/api/views/ijzp-q8t2/rows.csv?accessType=DOWNLOAD'\n",
    "filename = 'crime.csv'\n",
    "urllib.request.urlretrieve(url, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "crime.csv\n"
     ]
    }
   ],
   "source": [
    "ls crime.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><strong>Dask DataFrame Structure:</strong></div>\n",
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Case Number</th>\n",
       "      <th>Date</th>\n",
       "      <th>Block</th>\n",
       "      <th>IUCR</th>\n",
       "      <th>Primary Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>Location Description</th>\n",
       "      <th>Arrest</th>\n",
       "      <th>Domestic</th>\n",
       "      <th>Beat</th>\n",
       "      <th>District</th>\n",
       "      <th>Ward</th>\n",
       "      <th>Community Area</th>\n",
       "      <th>FBI Code</th>\n",
       "      <th>X Coordinate</th>\n",
       "      <th>Y Coordinate</th>\n",
       "      <th>Year</th>\n",
       "      <th>Updated On</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Location</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>npartitions=26</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "<div>Dask Name: from-delayed, 78 tasks</div>"
      ],
      "text/plain": [
       "Dask DataFrame Structure:\n",
       "                    ID Case Number    Date   Block    IUCR Primary Type Description Location Description  Arrest Domestic    Beat District    Ward Community Area FBI Code X Coordinate Y Coordinate    Year Updated On Latitude Longitude Location\n",
       "npartitions=26                                                                                                                                                                                                                                     \n",
       "                object      object  object  object  object       object      object               object  object   object  object   object  object         object   object       object       object  object     object   object    object   object\n",
       "                   ...         ...     ...     ...     ...          ...         ...                  ...     ...      ...     ...      ...     ...            ...      ...          ...          ...     ...        ...      ...       ...      ...\n",
       "...                ...         ...     ...     ...     ...          ...         ...                  ...     ...      ...     ...      ...     ...            ...      ...          ...          ...     ...        ...      ...       ...      ...\n",
       "                   ...         ...     ...     ...     ...          ...         ...                  ...     ...      ...     ...      ...     ...            ...      ...          ...          ...     ...        ...      ...       ...      ...\n",
       "                   ...         ...     ...     ...     ...          ...         ...                  ...     ...      ...     ...      ...     ...            ...      ...          ...          ...     ...        ...      ...       ...      ...\n",
       "Dask Name: from-delayed, 78 tasks"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dd.read_csv(\"crime.csv\", dtype= str, error_bad_lines=False, warn_bad_lines=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Case Number</th>\n",
       "      <th>Date</th>\n",
       "      <th>Block</th>\n",
       "      <th>IUCR</th>\n",
       "      <th>Primary Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>Location Description</th>\n",
       "      <th>Arrest</th>\n",
       "      <th>Domestic</th>\n",
       "      <th>...</th>\n",
       "      <th>Ward</th>\n",
       "      <th>Community Area</th>\n",
       "      <th>FBI Code</th>\n",
       "      <th>X Coordinate</th>\n",
       "      <th>Y Coordinate</th>\n",
       "      <th>Year</th>\n",
       "      <th>Updated On</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>267230</td>\n",
       "      <td>11700926</td>\n",
       "      <td>JC279725</td>\n",
       "      <td>05/26/2019 05:13:00 PM</td>\n",
       "      <td>036XX W DOUGLAS BLVD</td>\n",
       "      <td>2825</td>\n",
       "      <td>OTHER OFFENSE</td>\n",
       "      <td>HARASSMENT BY TELEPHONE</td>\n",
       "      <td>APARTMENT</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>1152126</td>\n",
       "      <td>1893208</td>\n",
       "      <td>2019</td>\n",
       "      <td>06/30/2019 03:56:27 PM</td>\n",
       "      <td>41.862830429</td>\n",
       "      <td>-87.717040084</td>\n",
       "      <td>(41.862830429, -87.717040084)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267231</td>\n",
       "      <td>24560</td>\n",
       "      <td>JC279072</td>\n",
       "      <td>05/26/2019 06:48:00 AM</td>\n",
       "      <td>013XX W HASTINGS ST</td>\n",
       "      <td>0110</td>\n",
       "      <td>HOMICIDE</td>\n",
       "      <td>FIRST DEGREE MURDER</td>\n",
       "      <td>CHA PARKING LOT</td>\n",
       "      <td>true</td>\n",
       "      <td>false</td>\n",
       "      <td>...</td>\n",
       "      <td>25</td>\n",
       "      <td>28</td>\n",
       "      <td>01A</td>\n",
       "      <td>1167752</td>\n",
       "      <td>1893853</td>\n",
       "      <td>2019</td>\n",
       "      <td>07/16/2019 04:17:29 PM</td>\n",
       "      <td>41.864278228</td>\n",
       "      <td>-87.659660218</td>\n",
       "      <td>(41.864278228, -87.659660218)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267232</td>\n",
       "      <td>24559</td>\n",
       "      <td>JC278908</td>\n",
       "      <td>05/26/2019 02:11:00 AM</td>\n",
       "      <td>013XX W HASTINGS ST</td>\n",
       "      <td>0110</td>\n",
       "      <td>HOMICIDE</td>\n",
       "      <td>FIRST DEGREE MURDER</td>\n",
       "      <td>STREET</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>...</td>\n",
       "      <td>25</td>\n",
       "      <td>28</td>\n",
       "      <td>01A</td>\n",
       "      <td>1167746</td>\n",
       "      <td>1893853</td>\n",
       "      <td>2019</td>\n",
       "      <td>06/02/2019 04:09:42 PM</td>\n",
       "      <td>41.864278357</td>\n",
       "      <td>-87.659682244</td>\n",
       "      <td>(41.864278357, -87.659682244)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267233</td>\n",
       "      <td>11707734</td>\n",
       "      <td>JC287730</td>\n",
       "      <td>07/01/2014 07:30:00 AM</td>\n",
       "      <td>063XX S NORMAL BLVD</td>\n",
       "      <td>1153</td>\n",
       "      <td>DECEPTIVE PRACTICE</td>\n",
       "      <td>FINANCIAL IDENTITY THEFT OVER $ 300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>68</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>06/02/2019 04:09:42 PM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267234</td>\n",
       "      <td>11707239</td>\n",
       "      <td>JC287563</td>\n",
       "      <td>11/30/2017 09:00:00 AM</td>\n",
       "      <td>022XX S KOSTNER AVE</td>\n",
       "      <td>1153</td>\n",
       "      <td>DECEPTIVE PRACTICE</td>\n",
       "      <td>FINANCIAL IDENTITY THEFT OVER $ 300</td>\n",
       "      <td>RESIDENCE</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>...</td>\n",
       "      <td>22</td>\n",
       "      <td>29</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>06/02/2019 04:09:42 PM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              ID Case Number                    Date                 Block  \\\n",
       "267230  11700926    JC279725  05/26/2019 05:13:00 PM  036XX W DOUGLAS BLVD   \n",
       "267231     24560    JC279072  05/26/2019 06:48:00 AM   013XX W HASTINGS ST   \n",
       "267232     24559    JC278908  05/26/2019 02:11:00 AM   013XX W HASTINGS ST   \n",
       "267233  11707734    JC287730  07/01/2014 07:30:00 AM   063XX S NORMAL BLVD   \n",
       "267234  11707239    JC287563  11/30/2017 09:00:00 AM   022XX S KOSTNER AVE   \n",
       "\n",
       "        IUCR        Primary Type                          Description  \\\n",
       "267230  2825       OTHER OFFENSE              HARASSMENT BY TELEPHONE   \n",
       "267231  0110            HOMICIDE                  FIRST DEGREE MURDER   \n",
       "267232  0110            HOMICIDE                  FIRST DEGREE MURDER   \n",
       "267233  1153  DECEPTIVE PRACTICE  FINANCIAL IDENTITY THEFT OVER $ 300   \n",
       "267234  1153  DECEPTIVE PRACTICE  FINANCIAL IDENTITY THEFT OVER $ 300   \n",
       "\n",
       "       Location Description Arrest Domestic  ... Ward Community Area FBI Code  \\\n",
       "267230            APARTMENT  false     true  ...   24             29       26   \n",
       "267231      CHA PARKING LOT   true    false  ...   25             28      01A   \n",
       "267232               STREET  false    false  ...   25             28      01A   \n",
       "267233                  NaN  false    false  ...   20             68       11   \n",
       "267234            RESIDENCE  false    false  ...   22             29       11   \n",
       "\n",
       "       X Coordinate Y Coordinate  Year              Updated On      Latitude  \\\n",
       "267230      1152126      1893208  2019  06/30/2019 03:56:27 PM  41.862830429   \n",
       "267231      1167752      1893853  2019  07/16/2019 04:17:29 PM  41.864278228   \n",
       "267232      1167746      1893853  2019  06/02/2019 04:09:42 PM  41.864278357   \n",
       "267233          NaN          NaN  2014  06/02/2019 04:09:42 PM           NaN   \n",
       "267234          NaN          NaN  2017  06/02/2019 04:09:42 PM           NaN   \n",
       "\n",
       "            Longitude                       Location  \n",
       "267230  -87.717040084  (41.862830429, -87.717040084)  \n",
       "267231  -87.659660218  (41.864278228, -87.659660218)  \n",
       "267232  -87.659682244  (41.864278357, -87.659682244)  \n",
       "267233            NaN                            NaN  \n",
       "267234            NaN                            NaN  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> ProgressBar </font> \n",
    "---\n",
    "* To know the progress of work in dask task\n",
    "* Show the progress of dask work as a bar\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask.diagnostics import ProgressBar\n",
    "pbar = ProgressBar()\n",
    "pbar.register()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dask.dataframe.core.DataFrame"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed | 49.2s\n",
      "[########################################] | 100% Completed | 49.3s\n",
      "CPU times: user 40.8 s, sys: 4.65 s, total: 45.4 s\n",
      "Wall time: 49.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ID                      7054819\n",
       "Case Number             7054815\n",
       "Date                    7054819\n",
       "Block                   7054819\n",
       "IUCR                    7054819\n",
       "Primary Type            7054819\n",
       "Description             7054819\n",
       "Location Description    7048684\n",
       "Arrest                  7054819\n",
       "Domestic                7054819\n",
       "Beat                    7054819\n",
       "District                7054772\n",
       "Ward                    6439989\n",
       "Community Area          6441324\n",
       "FBI Code                7054819\n",
       "X Coordinate            6987226\n",
       "Y Coordinate            6987226\n",
       "Year                    7054819\n",
       "Updated On              7054819\n",
       "Latitude                6987226\n",
       "Longitude               6987226\n",
       "Location                6987226\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df.count().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> Task Scheduler </font>\n",
    "---\n",
    "* dask.get: single thread\n",
    "* dask.threaded.get: multiple thread pool\n",
    "* dask.multiprocessing.get : multiprocess pool\n",
    "* distributed.Client.get: multiple computer\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 20 CPUs Demo\n",
    "* 0.6s faster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed | 48.2s\n",
      "[########################################] | 100% Completed | 48.3s\n",
      "CPU times: user 841 ms, sys: 335 ms, total: 1.18 s\n",
      "Wall time: 48.6 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ID                      7054819\n",
       "Case Number             7054815\n",
       "Date                    7054819\n",
       "Block                   7054819\n",
       "IUCR                    7054819\n",
       "Primary Type            7054819\n",
       "Description             7054819\n",
       "Location Description    7048684\n",
       "Arrest                  7054819\n",
       "Domestic                7054819\n",
       "Beat                    7054819\n",
       "District                7054772\n",
       "Ward                    6439989\n",
       "Community Area          6441324\n",
       "FBI Code                7054819\n",
       "X Coordinate            6987226\n",
       "Y Coordinate            6987226\n",
       "Year                    7054819\n",
       "Updated On              7054819\n",
       "Latitude                6987226\n",
       "Longitude               6987226\n",
       "Location                6987226\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df.count().compute(scheduler='processes', num_workers=20) # 20Cpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Part 2: Basic 2\n",
    "* Ref: https://towardsdatascience.com/parallelizing-feature-engineering-with-dask-3db88aec33b7\n",
    "* Ref: https://towardsdatascience.com/trying-out-dask-dataframes-in-python-for-fast-data-analysis-in-parallel-aa960c18a915 (English)\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Dask Bags: https://examples.dask.org/bag.html\n",
    "* Dask Bags: https://docs.dask.org/en/latest/bag.html\n",
    "* Dask JSON data: https://examples.dask.org/applications/json-data-on-the-web.html\n",
    "* Dask Tutorial: https://people.duke.edu/~ccc14/sta-663-2017/18A_Dask.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> Data Structure of Dask </font>\n",
    "* Dask Tutorial: https://people.duke.edu/~ccc14/sta-663-2017/18A_Dask.html\n",
    "---\n",
    "#### Dask provides 3 data structures\n",
    "* dask array ~ numpy array\n",
    "* dask bag ~ Python dictionary\n",
    "* dask dataframe ~ pandas dataframe\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask\n",
    "import dask.array as da\n",
    "import dask.bag as db\n",
    "import dask.dataframe as dd\n",
    "from dask import delayed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### dask arrays\n",
    "* but break a massive job into tasks that are then executed by a scheduler. \n",
    "* we can tell the dask array how to break the data into chunks for processing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### dask data frames\n",
    "* treat multiple pandas dataframes that might not simultaneously fit into memory like a single dataframe."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### dask bags\n",
    "* multiset is a set that allows repeats. Unlike lists, order is not preserved."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Conversion from bag to dataframe\n",
    "* dask_df = dic.to_dataframe(columns=['word', 'n'])\n",
    "* dask_df.head(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analyze web-hosted JSON data\n",
    "* Ref: https://examples.dask.org/applications/json-data-on-the-web.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read file as a dask bag = db\n",
    "* import dask.bag as db\n",
    "* dic= db.read_text(filenames).map(json.loads)\n",
    "* dic.take(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dask Gabs\n",
    "* map, filter, groupby\n",
    "* Ref: https://examples.dask.org/bag.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read json as a dask bag\n",
    "* import dask.bag as db\n",
    "* import json\n",
    "\n",
    "* b = db.read_text('data/*.json').map(json.loads)\n",
    "* b.take(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dask bag : Map, Filter, Aggregate\n",
    "* b.filter(lambda record: record['age_variable'] > 30).take(2)  # Select only people over 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* b.map(lambda record: record['occupation']).take(2)  # Select the occupation field"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* b.count().compute()  # Count total number of records"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Chain computations\n",
    "* result = (b.filter(lambda record: record['age'] > 30)\n",
    "           .map(lambda record: record['occupation'])\n",
    "           .frequencies(sort=True)\n",
    "           .topk(10, key=1))\n",
    "\n",
    "* result.compute() # we need to call compute to actually evaluate our result."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dask Bag Tutorial 2\n",
    "* https://docs.dask.org/en/latest/bag.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# <font color = blue> Dask dataframe Stack, Concatenate, and Block <font> \n",
    "---\n",
    "* da.stack, da.concatenate, and da.block\n",
    "* Ref: https://docs.dask.org/en/latest/array-stack.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.array as da"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr0 = da.from_array(np.zeros((3, 4)), chunks=(1, 2))\n",
    "arr1 = da.from_array(np.ones((3, 4)), chunks=(1, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr>\n",
       "<td>\n",
       "<table>\n",
       "  <thead>\n",
       "    <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><th> Bytes </th><td> 96 B </td> <td> 16 B </td></tr>\n",
       "    <tr><th> Shape </th><td> (3, 4) </td> <td> (1, 2) </td></tr>\n",
       "    <tr><th> Count </th><td> 7 Tasks </td><td> 6 Chunks </td></tr>\n",
       "    <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</td>\n",
       "<td>\n",
       "<svg width=\"170\" height=\"140\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"0\" y1=\"30\" x2=\"120\" y2=\"30\" />\n",
       "  <line x1=\"0\" y1=\"60\" x2=\"120\" y2=\"60\" />\n",
       "  <line x1=\"0\" y1=\"90\" x2=\"120\" y2=\"90\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"90\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"60\" y1=\"0\" x2=\"60\" y2=\"90\" />\n",
       "  <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"90\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,90.000000 0.000000,90.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Text -->\n",
       "  <text x=\"60.000000\" y=\"110.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >4</text>\n",
       "  <text x=\"140.000000\" y=\"45.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,140.000000,45.000000)\">3</text>\n",
       "</svg>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "dask.array<array, shape=(3, 4), dtype=float64, chunksize=(1, 2), chunktype=numpy.ndarray>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[dask.array<array, shape=(3, 4), dtype=float64, chunksize=(1, 2), chunktype=numpy.ndarray>,\n",
       " dask.array<array, shape=(3, 4), dtype=float64, chunksize=(1, 2), chunktype=numpy.ndarray>]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = [arr0, arr1]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3, 4)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = da.stack(data, axis=0)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr>\n",
       "<td>\n",
       "<table>\n",
       "  <thead>\n",
       "    <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><th> Bytes </th><td> 192 B </td> <td> 16 B </td></tr>\n",
       "    <tr><th> Shape </th><td> (2, 3, 4) </td> <td> (1, 1, 2) </td></tr>\n",
       "    <tr><th> Count </th><td> 26 Tasks </td><td> 12 Chunks </td></tr>\n",
       "    <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</td>\n",
       "<td>\n",
       "<svg width=\"215\" height=\"175\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"45\" y2=\"35\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"10\" y1=\"30\" x2=\"45\" y2=\"65\" />\n",
       "  <line x1=\"10\" y1=\"60\" x2=\"45\" y2=\"95\" />\n",
       "  <line x1=\"10\" y1=\"90\" x2=\"45\" y2=\"125\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"90\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"27\" y1=\"17\" x2=\"27\" y2=\"107\" />\n",
       "  <line x1=\"45\" y1=\"35\" x2=\"45\" y2=\"125\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"10.000000,0.000000 45.294118,35.294118 45.294118,125.294118 10.000000,90.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"130\" y2=\"0\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"27\" y1=\"17\" x2=\"147\" y2=\"17\" />\n",
       "  <line x1=\"45\" y1=\"35\" x2=\"165\" y2=\"35\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"45\" y2=\"35\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"70\" y1=\"0\" x2=\"105\" y2=\"35\" />\n",
       "  <line x1=\"130\" y1=\"0\" x2=\"165\" y2=\"35\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"10.000000,0.000000 130.000000,0.000000 165.294118,35.294118 45.294118,35.294118\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"45\" y1=\"35\" x2=\"165\" y2=\"35\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"45\" y1=\"65\" x2=\"165\" y2=\"65\" />\n",
       "  <line x1=\"45\" y1=\"95\" x2=\"165\" y2=\"95\" />\n",
       "  <line x1=\"45\" y1=\"125\" x2=\"165\" y2=\"125\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"45\" y1=\"35\" x2=\"45\" y2=\"125\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"105\" y1=\"35\" x2=\"105\" y2=\"125\" />\n",
       "  <line x1=\"165\" y1=\"35\" x2=\"165\" y2=\"125\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"45.294118,35.294118 165.294118,35.294118 165.294118,125.294118 45.294118,125.294118\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Text -->\n",
       "  <text x=\"105.294118\" y=\"145.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >4</text>\n",
       "  <text x=\"185.294118\" y=\"80.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,185.294118,80.294118)\">3</text>\n",
       "  <text x=\"17.647059\" y=\"127.647059\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,17.647059,127.647059)\">2</text>\n",
       "</svg>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "dask.array<stack, shape=(2, 3, 4), dtype=float64, chunksize=(1, 1, 2), chunktype=numpy.ndarray>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 2, 4)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "da.stack(data, axis=1).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Concatenate\n",
    "* concatenate existing arrays into a new array, extending them along an existing dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.array as da\n",
    "import numpy as np\n",
    "\n",
    "arr0 = da.from_array(np.zeros((3, 4)))\n",
    "arr1 = da.from_array(np.ones((3, 4)))\n",
    "data = [arr0, arr1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6, 4)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = da.concatenate(data, axis=0) \n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 8)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "da.concatenate(data, axis=1).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataframe concante example\n",
    "* import dask.dataframe as dd\n",
    "* df = dd.read_csv(['data/untermaederbrunnen_station1_xyz_intensity_rgb.txt'], delimiter=' ', header=None, names=['x', 'y', 'z', 'intensity', 'r', 'g', 'b'])\n",
    "* df_label = dd.read_csv(['data/untermaederbrunnen_station1_xyz_intensity_rgb.labels'], header=None, names=['label'])\n",
    "* df = df.repartition(npartitions=200)\n",
    "* df_label = df_label.repartition(npartitions=200)\n",
    "* df = dd.concat([df, df_label], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dask Groupby\n",
    "* Ref: https://examples.dask.org/dataframes/02-groupby.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><strong>Dask DataFrame Structure:</strong></div>\n",
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>npartitions=30</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2000-01-01</th>\n",
       "      <td>int64</td>\n",
       "      <td>object</td>\n",
       "      <td>float64</td>\n",
       "      <td>float64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-02</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-30</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-31</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "<div>Dask Name: make-timeseries, 30 tasks</div>"
      ],
      "text/plain": [
       "Dask DataFrame Structure:\n",
       "                   id    name        x        y\n",
       "npartitions=30                                 \n",
       "2000-01-01      int64  object  float64  float64\n",
       "2000-01-02        ...     ...      ...      ...\n",
       "...               ...     ...      ...      ...\n",
       "2000-01-30        ...     ...      ...      ...\n",
       "2000-01-31        ...     ...      ...      ...\n",
       "Dask Name: make-timeseries, 30 tasks"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import dask\n",
    "df = dask.datasets.timeseries()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed |  1.2s\n",
      "[########################################] | 100% Completed |  1.3s\n",
      "CPU times: user 1.05 s, sys: 112 ms, total: 1.17 s\n",
      "Wall time: 1.34 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "name\n",
       "Alice       0.001558\n",
       "Bob         0.001778\n",
       "Charlie     0.001967\n",
       "Dan         0.001146\n",
       "Edith       0.003717\n",
       "Frank       0.001993\n",
       "George     -0.000409\n",
       "Hannah      0.001473\n",
       "Ingrid      0.000094\n",
       "Jerry       0.002051\n",
       "Kevin       0.000493\n",
       "Laura      -0.002057\n",
       "Michael    -0.000478\n",
       "Norbert     0.000516\n",
       "Oliver     -0.001005\n",
       "Patricia    0.000314\n",
       "Quinn      -0.003217\n",
       "Ray        -0.000248\n",
       "Sarah      -0.004322\n",
       "Tim         0.002823\n",
       "Ursula     -0.001512\n",
       "Victor      0.002398\n",
       "Wendy       0.002392\n",
       "Xavier     -0.002486\n",
       "Yvonne      0.002242\n",
       "Zelda       0.002681\n",
       "Name: x, dtype: float64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df.groupby('name').x.mean().compute(scheduler='processes', num_workers=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}