examples/parallel_benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.7.7\n"
     ]
    }
   ],
   "source": [
    "from platform import python_version\n",
    "\n",
    "print(python_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.18.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import fklearn\n",
    "\n",
    "fklearn.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fklearn.data.datasets import make_tutorial_data\n",
    "from fklearn.preprocessing.splitting import space_time_split_dataset\n",
    "from concurrent.futures import ThreadPoolExecutor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_START_DATE = '2015-01-01' \n",
    "TRAIN_END_DATE = '2015-03-01' \n",
    "HOLDOUT_END_DATE = '2015-04-01'\n",
    "\n",
    "split_fn = space_time_split_dataset(train_start_date=TRAIN_START_DATE,\n",
    "                                train_end_date=TRAIN_END_DATE,\n",
    "                                holdout_end_date=HOLDOUT_END_DATE,\n",
    "                                space_holdout_percentage=.5,\n",
    "                                split_seed=42, \n",
    "                                space_column=\"id\",\n",
    "                                time_column=\"date\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def measure_time_split_fn(sample_size):\n",
    "    result = %timeit -r 1 -o split_fn(data[:sample_size])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_sizes = [10000, 100000, 250000, 500000, 750000, 1000000]\n",
    "data = make_tutorial_data(sample_sizes[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.12 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 100 loops each)\n",
      "26.5 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 10 loops each)\n",
      "54.4 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 10 loops each)\n",
      "102 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 10 loops each)\n",
      "152 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 10 loops each)\n",
      "204 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "old_function_times = []\n",
    "\n",
    "for sample_size in sample_sizes:\n",
    "    time_old = %timeit -r 1 -o split_fn(data[:sample_size])\n",
    "    old_function_times.append(time_old.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.007119678000000001,\n",
       " 0.026497800000000016,\n",
       " 0.05441616999999992,\n",
       " 0.10246849999999999,\n",
       " 0.15187872999999996,\n",
       " 0.2037241999999999]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "old_function_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "985 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "1.01 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)939 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "\n",
      "947 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "876 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "149 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "with ThreadPoolExecutor() as executor:\n",
    "        result = executor.map(measure_time_split_fn, sample_sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<TimeitResult : 149 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 10 loops each)>,\n",
       " <TimeitResult : 985 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>,\n",
       " <TimeitResult : 1.01 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>,\n",
       " <TimeitResult : 939 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>,\n",
       " <TimeitResult : 947 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>,\n",
       " <TimeitResult : 876 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import nan\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def new_make_tutorial_data(n: int) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Generates fake data for a tutorial. There are 3 numerical features (\"num1\", \"num3\" and \"num3\")\n",
    "    and tow categorical features (\"cat1\" and \"cat2\")\n",
    "    sex, age and severity, the treatment is a binary variable, medication and the response\n",
    "    days until recovery.\n",
    "    Parameters\n",
    "    ----------\n",
    "    n : int\n",
    "        The number of samples to generate\n",
    "    Returns\n",
    "    ----------\n",
    "    df : pd.DataFrame\n",
    "        A tutorial dataset\n",
    "    \"\"\"\n",
    "    np.random.seed(1111)\n",
    "\n",
    "    dataset = pd.DataFrame({\n",
    "        \"id\": list(map(lambda x: \"id%d\" % x, np.random.randint(0, 1000000, n))),\n",
    "        \"date\": np.random.choice(pd.date_range(\"2015-01-01\", periods=100), n),\n",
    "        \"feature1\": np.random.gamma(20, size=n),\n",
    "        \"feature2\": np.random.normal(40, size=n),\n",
    "        \"feature3\": np.random.choice([\"a\", \"b\", \"c\"], size=n)})\n",
    "\n",
    "    dataset[\"target\"] = (dataset[\"feature1\"]\n",
    "                         + dataset[\"feature2\"]\n",
    "                         + dataset[\"feature3\"].apply(lambda x: 0 if x == \"a\" else 30 if x == \"b\" else 10)\n",
    "                         + np.random.normal(0, 5, size=n))\n",
    "\n",
    "    # insert some NANs\n",
    "    dataset.loc[np.random.randint(0, n, 100), \"feature1\"] = nan\n",
    "    dataset.loc[np.random.randint(0, n, 100), \"feature3\"] = nan\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample_sizes = [10000, 50000, 100000, 200000, 250000,]\n",
    "sample_sizes = [10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000]\n",
    "data = new_make_tutorial_data(sample_sizes[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.47 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "2.45 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "2.46 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "2.47 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "2.48 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "2.47 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "2.47 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)2.48 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with ThreadPoolExecutor() as executor:\n",
    "        result = executor.map(measure_time_split_fn, sample_sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}