015_Dask_Delayed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5aea82cb-6c9c-4146-92df-3b64a89ee923",
   "metadata": {},
   "source": [
    "# Parallelizing Python code with Dask\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea6fa9f9-cd57-4b8c-8760-a0f4c259a8c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import requests\n",
    "\n",
    "URLS = [\n",
    "    'https://jsonplaceholder.typicode.com/posts',\n",
    "    'https://jsonplaceholder.typicode.com/comments',\n",
    "    'https://jsonplaceholder.typicode.com/albums',\n",
    "    'https://jsonplaceholder.typicode.com/photos',\n",
    "    'https://jsonplaceholder.typicode.com/todos',\n",
    "    'https://jsonplaceholder.typicode.com/users'\n",
    "]\n",
    "\n",
    "def fetch_single(url: str) -> None:\n",
    "    print(f'Fetching: {url}...')\n",
    "    req = requests.get(url)\n",
    "    time.sleep(1)\n",
    "    print(f'Fetched {url}!')\n",
    "    return req.content"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b16986d-aa87-4548-be0a-d4cf3e6e07c4",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## Normal Python Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5e76f8ea-ac8a-474a-9599-0a26c147105f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching: https://jsonplaceholder.typicode.com/posts...\n",
      "Fetched https://jsonplaceholder.typicode.com/posts!\n",
      "Fetching: https://jsonplaceholder.typicode.com/comments...\n",
      "Fetched https://jsonplaceholder.typicode.com/comments!\n",
      "Fetching: https://jsonplaceholder.typicode.com/albums...\n",
      "Fetched https://jsonplaceholder.typicode.com/albums!\n",
      "Fetching: https://jsonplaceholder.typicode.com/photos...\n",
      "Fetched https://jsonplaceholder.typicode.com/photos!\n",
      "Fetching: https://jsonplaceholder.typicode.com/todos...\n",
      "Fetched https://jsonplaceholder.typicode.com/todos!\n",
      "Fetching: https://jsonplaceholder.typicode.com/users...\n",
      "Fetched https://jsonplaceholder.typicode.com/users!\n",
      "CPU times: user 138 ms, sys: 16.1 ms, total: 155 ms\n",
      "Wall time: 7.79 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "fetch_normal = []\n",
    "\n",
    "for url in URLS:\n",
    "    single = fetch_single(url)\n",
    "    fetch_normal.append(single)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ad09d76-7c17-42df-9287-4f9c3591710f",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## Parallel Execution with Dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4499dc39-8ac4-4a30-804a-775607d95519",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask import delayed, compute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "21c04d24-8870-468d-a0ff-ad1f9770faa0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching: https://jsonplaceholder.typicode.com/posts...Fetching: https://jsonplaceholder.typicode.com/users...Fetching: https://jsonplaceholder.typicode.com/comments...\n",
      "\n",
      "Fetching: https://jsonplaceholder.typicode.com/albums...\n",
      "Fetching: https://jsonplaceholder.typicode.com/photos...\n",
      "Fetching: https://jsonplaceholder.typicode.com/todos...\n",
      "\n",
      "Fetched https://jsonplaceholder.typicode.com/todos!\n",
      "Fetched https://jsonplaceholder.typicode.com/albums!\n",
      "Fetched https://jsonplaceholder.typicode.com/comments!\n",
      "Fetched https://jsonplaceholder.typicode.com/users!\n",
      "Fetched https://jsonplaceholder.typicode.com/posts!\n",
      "Fetched https://jsonplaceholder.typicode.com/photos!\n",
      "CPU times: user 121 ms, sys: 17.4 ms, total: 138 ms\n",
      "Wall time: 1.43 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "fetch_dask = []\n",
    "\n",
    "for url in URLS:\n",
    "    single = delayed(fetch_single)(url)\n",
    "    fetch_dask.append(single)\n",
    "    \n",
    "results_dask = compute(*fetch_dask)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3769dbe-1d2b-4d37-a7c4-a46d3e4f54b0",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "## Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "12513b81-0ab6-4bdf-9c89-efca328ceab2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(list, tuple)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(fetch_normal), type(results_dask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e373d248-ec10-4055-a414-50450bd57caf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fetch_normal == list(results_dask)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}