examples/crossing-boundaries.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02f7cf5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import dias.rewriter\n",
    "import urllib.request\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d82189ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the NYC Taxi dataset. Enable this only in the first run (i.e., if you don't have the file).\n",
    "url = 'https://dask-data.s3.amazonaws.com/nyc-taxi/2015/yellow_tripdata_2015-01.csv'\n",
    "filename = 'yellow_tripdata_2015-01.csv'\n",
    "urllib.request.urlretrieve(url, filename)\n",
    "assert os.path.isfile(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9b1ae907",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(filename, low_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b47a85e",
   "metadata": {},
   "source": [
    "## Original\n",
    "\n",
    "Figure 3 from [our paper](https://baziotis.cs.illinois.edu/papers/dias.pdf) that crosses library boundaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a8dee780",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.44 s, sys: 409 ms, total: 1.85 s\n",
      "Wall time: 1.84 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# DIAS_DISABLE\n",
    "orig = pd.Series(df['pickup_longitude'].tolist() + df['pickup_latitude'].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09970aa9",
   "metadata": {},
   "source": [
    "## With Dias\n",
    "\n",
    "Dias rewrites the code to be about **69x faster**. Note that using Dias is so lightweight that even when using it in cells that are somewhat fast already, it is probably worth it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7981de4e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.7 ms, sys: 12.8 ms, total: 27.5 ms\n",
      "Wall time: 26.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rewr = pd.Series(df['pickup_longitude'].tolist() + df['pickup_latitude'].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ace91520",
   "metadata": {},
   "source": [
    "## Correctness check\n",
    "\n",
    "We drop the NaNs because comparisons are always false."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ad0929fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (orig == rewr).all()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}