DataFrames/Pandas_Alternatives/pandas_alternatives_POC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pandas alternatives Proof of Concept\n",
    "In this notebook we will explore, if it's worth using pandas alternatives - vaex, spark, dask or modin if our dataset fits comfortably into the memory. We will load two files, join them, group by and aggregate and sort the results. The file `Performance_test.py` contains all these steps executable as a performance test. \n",
    "\n",
    "Article: https://towardsdatascience.com/is-something-better-than-pandas-when-the-dataset-fits-the-memory-7e8e983c4fe5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:14:45.083313Z",
     "start_time": "2020-11-17T23:14:44.818195Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UserWarning: The Dask Engine for Modin is experimental.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import gc\n",
    "from time import time, sleep, strftime, localtime\n",
    "\n",
    "import pandas as pd\n",
    "import dask.dataframe as dd\n",
    "import modin.pandas as mpd\n",
    "import vaex\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import sum, avg\n",
    "# pandas on ray has moved to Modin\n",
    "# import ray.dataframe as rpd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:55:27.722264Z",
     "start_time": "2020-11-17T22:55:27.699399Z"
    }
   },
   "outputs": [],
   "source": [
    "# data based on https://www.kaggle.com/c/ieee-fraud-detection/data\n",
    "folder = \"/Data/Kaggle/EEE-CIS_Fraud_Detection\"\n",
    "files = [\"train_transaction.csv\", \"train_identity.csv\"]\n",
    "paths = [os.path.join(folder, f) for f in files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:55:27.733967Z",
     "start_time": "2020-11-17T22:55:27.727006Z"
    }
   },
   "outputs": [],
   "source": [
    "stats = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:55:27.750077Z",
     "start_time": "2020-11-17T22:55:27.737957Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.1.4'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:56:19.465362Z",
     "start_time": "2020-11-17T22:55:27.767439Z"
    }
   },
   "outputs": [],
   "source": [
    "stats[\"pandas\"] = {}\n",
    "s = stats[\"pandas\"]\n",
    "\n",
    "ts = time()\n",
    "df = pd.read_csv(paths[0])\n",
    "te = time()\n",
    "s[\"load_transactions\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "df2 = pd.read_csv(paths[1])\n",
    "te = time()\n",
    "s[\"load_identity\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "dff = df.merge(df2, on=\"TransactionID\")\n",
    "te = time()\n",
    "s[\"merge\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "grp = dff.groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"])[\"TransactionAmt\"].agg([\"mean\",\"sum\"])\n",
    "te = time()\n",
    "s[\"aggregation\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "dff.sort_values(by=[\"card1\",\"addr1\",\"D9\"], inplace=True)\n",
    "dff.sort_values(by=[\"addr1\",\"D9\",\"card1\"], inplace=True)\n",
    "dff.sort_values(by=[\"D9\",\"card1\",\"addr1\"], inplace=True)\n",
    "te = time()\n",
    "s[\"sorting\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "dff[\"card2\"].value_counts()\n",
    "te = time()\n",
    "s[\"value_counts\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:56:19.519391Z",
     "start_time": "2020-11-17T22:56:19.472412Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pandas</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>aggregation</th>\n",
       "      <td>0.062498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_identity</th>\n",
       "      <td>0.501993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_transactions</th>\n",
       "      <td>20.143912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merge</th>\n",
       "      <td>2.207136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sorting</th>\n",
       "      <td>1.258234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value_counts</th>\n",
       "      <td>0.002657</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      pandas\n",
       "aggregation         0.062498\n",
       "load_identity       0.501993\n",
       "load_transactions  20.143912\n",
       "merge               2.207136\n",
       "sorting             1.258234\n",
       "value_counts        0.002657"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dff.to_pickle(\"data/dff.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4553, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Because julia groups by including N\\A, let's just check that number of groups matches\n",
    "grp = dff[[\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\",\"TransactionAmt\"]].fillna(\"~U~\")\\\n",
    ".groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"])[\"TransactionAmt\"].agg([\"mean\",\"sum\"])\n",
    "grp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:56:19.550559Z",
     "start_time": "2020-11-17T22:56:19.524963Z"
    }
   },
   "outputs": [],
   "source": [
    "def clean(wait_time: int=15):\n",
    "    \"\"\"Cleans created DataFrames and call the garbage collector to actions. Wait for 15s by default\"\"\"\n",
    "    df, df2, dff, grp = None, None, None, None\n",
    "    gc.collect()\n",
    "    sleep(wait_time)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:56:34.718818Z",
     "start_time": "2020-11-17T22:56:19.559830Z"
    }
   },
   "outputs": [],
   "source": [
    "clean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:56:34.813222Z",
     "start_time": "2020-11-17T22:56:34.780299Z"
    }
   },
   "outputs": [],
   "source": [
    "def list_variables_memory_usage() -> dict:\n",
    "    \"\"\"Memory of existing local variables\"\"\"\n",
    "    local_vars = list(locals().items())\n",
    "    return {var: sys.getsizeof(obj) for var, obj in local_vars}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dask\n",
    "When to use dask - https://docs.dask.org/en/latest/dataframe.html#common-uses-and-anti-uses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:57:00.280745Z",
     "start_time": "2020-11-17T22:56:34.844985Z"
    }
   },
   "outputs": [],
   "source": [
    "stats[\"dask\"] = {}\n",
    "s = stats[\"dask\"]\n",
    "\n",
    "ts = time()\n",
    "df = dd.read_csv(paths[0])\n",
    "te = time()\n",
    "s[\"load_transactions\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "df2 = dd.read_csv(paths[1])\n",
    "te = time()\n",
    "s[\"load_identity\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "dff = df.merge(df2, on=\"TransactionID\")\n",
    "te = time()\n",
    "s[\"merge\"] = te-ts\n",
    "\n",
    "# the difference is that we call compute method, which runs all the computations at this point\n",
    "ts = time()\n",
    "grp = dff.groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"])[\"TransactionAmt\"]\\\n",
    "    .agg([\"mean\",\"sum\"])\\\n",
    "    .compute()\n",
    "te = time()\n",
    "s[\"aggregation\"] = te-ts\n",
    "\n",
    "# parallel soring is tricky that is why there are only work arounds in dask. \n",
    "ts = time()\n",
    "dfs = dff.set_index(\"card1\").compute()\n",
    "te = time()\n",
    "s[\"sorting\"] = te-ts\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results of the `.compute()` operation can be pandas dataframe rather than dask.dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(dask.dataframe.core.DataFrame,\n",
       " pandas.core.frame.DataFrame,\n",
       " pandas.core.frame.DataFrame)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(dff), type(grp), type(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:57:15.539155Z",
     "start_time": "2020-11-17T22:57:00.286799Z"
    }
   },
   "outputs": [],
   "source": [
    "clean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:57:15.580380Z",
     "start_time": "2020-11-17T22:57:15.546567Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pandas</th>\n",
       "      <th>dask</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>load_transactions</th>\n",
       "      <td>20.143912</td>\n",
       "      <td>0.090847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_identity</th>\n",
       "      <td>0.501993</td>\n",
       "      <td>0.017655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merge</th>\n",
       "      <td>2.207136</td>\n",
       "      <td>0.075063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aggregation</th>\n",
       "      <td>0.062498</td>\n",
       "      <td>16.052712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sorting</th>\n",
       "      <td>1.258234</td>\n",
       "      <td>43.264110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value_counts</th>\n",
       "      <td>0.002657</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      pandas       dask\n",
       "load_transactions  20.143912   0.090847\n",
       "load_identity       0.501993   0.017655\n",
       "merge               2.207136   0.075063\n",
       "aggregation         0.062498  16.052712\n",
       "sorting             1.258234  43.264110\n",
       "value_counts        0.002657        NaN"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats[\"dask_indexed\"] = {}\n",
    "s = stats[\"dask_indexed\"]\n",
    "\n",
    "ts = time()\n",
    "df = dd.read_csv(paths[0]).set_index(\"TransactionID\")\n",
    "te = time()\n",
    "s[\"load_transactions\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "df2 = dd.read_csv(paths[1]).set_index(\"TransactionID\")\n",
    "te = time()\n",
    "s[\"load_identity\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "dff = df.merge(df2, left_index=True, right_index=True)\n",
    "te = time()\n",
    "s[\"merge\"] = te-ts\n",
    "\n",
    "# the difference is that we call compute method, which runs all the computations at this point\n",
    "ts = time()\n",
    "grp = dff.groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"])[\"TransactionAmt\"]\\\n",
    "    .agg([\"mean\",\"sum\"])\\\n",
    "    .compute()\n",
    "te = time()\n",
    "s[\"aggregation\"] = te-ts\n",
    "\n",
    "# parallel soring is tricky that is why there are only work arounds in dask. \n",
    "ts = time()\n",
    "dff.set_index(\"card1\").compute()\n",
    "te = time()\n",
    "s[\"sorting\"] = te-ts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pandas</th>\n",
       "      <th>dask</th>\n",
       "      <th>dask_indexed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>load_transactions</th>\n",
       "      <td>20.143912</td>\n",
       "      <td>0.090847</td>\n",
       "      <td>10.722161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_identity</th>\n",
       "      <td>0.501993</td>\n",
       "      <td>0.017655</td>\n",
       "      <td>0.582975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merge</th>\n",
       "      <td>2.207136</td>\n",
       "      <td>0.075063</td>\n",
       "      <td>0.048098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aggregation</th>\n",
       "      <td>0.062498</td>\n",
       "      <td>16.052712</td>\n",
       "      <td>14.059959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sorting</th>\n",
       "      <td>1.258234</td>\n",
       "      <td>43.264110</td>\n",
       "      <td>40.348259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value_counts</th>\n",
       "      <td>0.002657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      pandas       dask  dask_indexed\n",
       "load_transactions  20.143912   0.090847     10.722161\n",
       "load_identity       0.501993   0.017655      0.582975\n",
       "merge               2.207136   0.075063      0.048098\n",
       "aggregation         0.062498  16.052712     14.059959\n",
       "sorting             1.258234  43.264110     40.348259\n",
       "value_counts        0.002657        NaN           NaN"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean()\n",
    "pd.DataFrame(stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:29:19.438715Z",
     "start_time": "2020-11-17T22:29:19.429209Z"
    }
   },
   "source": [
    "# Vaex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:03:06.560013Z",
     "start_time": "2020-11-17T23:03:06.545427Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vaex-core': '2.0.3',\n",
       " 'vaex-viz': '0.4.0',\n",
       " 'vaex-hdf5': '0.6.0',\n",
       " 'vaex-server': '0.3.1',\n",
       " 'vaex-astro': '0.7.0',\n",
       " 'vaex-jupyter': '0.5.2',\n",
       " 'vaex-ml': '0.9.0',\n",
       " 'vaex-arrow': '0.5.1'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vaex.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:01.165275Z",
     "start_time": "2020-11-17T23:03:06.562006Z"
    }
   },
   "outputs": [],
   "source": [
    "tool = \"vaex\"\n",
    "stats[tool] = {}\n",
    "s = stats[tool]\n",
    "\n",
    "\n",
    "ts = time()\n",
    "#df = vaex.open(paths[0])\n",
    "df = vaex.from_csv(paths[0])\n",
    "te = time()\n",
    "s[\"load_transactions\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "df2 = vaex.open(paths[1])\n",
    "te = time()\n",
    "s[\"load_identity\"] = te-ts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:01.642707Z",
     "start_time": "2020-11-17T23:04:01.176085Z"
    }
   },
   "outputs": [],
   "source": [
    "ts = time()\n",
    "dff = df.join(df2, on=\"TransactionID\")\n",
    "te = time()\n",
    "s[\"merge\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:03.395316Z",
     "start_time": "2020-11-17T23:04:01.645742Z"
    }
   },
   "outputs": [],
   "source": [
    "# the difference is that we call compute method, which runs all the computations at this point\n",
    "ts = time()\n",
    "\n",
    "grp = dff.groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"], \n",
    "                  agg={'TransactionAmt':[\"mean\",\"sum\"]})\n",
    "\n",
    "# alternatively\n",
    "#grp = dff.groupby([dff[\"isFraud\"],dff[\"ProductCD\"],dff[\"card4\"],dff[\"card6\"],dff[\"id_15\"],dff[\"id_31\"]], \n",
    "#                  agg=[vaex.agg.mean('TransactionAmt'), vaex.agg.sum('TransactionAmt')])\n",
    "\n",
    "\n",
    "te = time()\n",
    "s[\"aggregation\"] = te-ts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the difference is that we call compute method, which runs all the computations at this point\n",
    "ts = time()\n",
    "dff_s = dff.sort(by=[\"card1\",\"addr1\",\"D9\"])\n",
    "dff_s = dff.sort(by=[\"addr1\",\"D9\",\"card1\"])\n",
    "dff_s = dff.sort(by=[\"D9\",\"card1\",\"addr1\"])\n",
    "te = time()\n",
    "s[\"sorting\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = time()\n",
    "dff[\"card2\"].value_counts()\n",
    "te = time()\n",
    "s[\"value_counts\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:03.469428Z",
     "start_time": "2020-11-17T23:04:03.423857Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pandas</th>\n",
       "      <th>dask</th>\n",
       "      <th>dask_indexed</th>\n",
       "      <th>vaex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>load_transactions</th>\n",
       "      <td>20.143912</td>\n",
       "      <td>0.090847</td>\n",
       "      <td>10.722161</td>\n",
       "      <td>16.696253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_identity</th>\n",
       "      <td>0.501993</td>\n",
       "      <td>0.017655</td>\n",
       "      <td>0.582975</td>\n",
       "      <td>0.749975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merge</th>\n",
       "      <td>2.207136</td>\n",
       "      <td>0.075063</td>\n",
       "      <td>0.048098</td>\n",
       "      <td>0.120835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aggregation</th>\n",
       "      <td>0.062498</td>\n",
       "      <td>16.052712</td>\n",
       "      <td>14.059959</td>\n",
       "      <td>0.455176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sorting</th>\n",
       "      <td>1.258234</td>\n",
       "      <td>43.264110</td>\n",
       "      <td>40.348259</td>\n",
       "      <td>1.124968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value_counts</th>\n",
       "      <td>0.002657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.021019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      pandas       dask  dask_indexed       vaex\n",
       "load_transactions  20.143912   0.090847     10.722161  16.696253\n",
       "load_identity       0.501993   0.017655      0.582975   0.749975\n",
       "merge               2.207136   0.075063      0.048098   0.120835\n",
       "aggregation         0.062498  16.052712     14.059959   0.455176\n",
       "sorting             1.258234  43.264110     40.348259   1.124968\n",
       "value_counts        0.002657        NaN           NaN   0.021019"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:18.930053Z",
     "start_time": "2020-11-17T23:04:03.543914Z"
    }
   },
   "outputs": [],
   "source": [
    "clean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PySpark\n",
    "Java must be installed. (e.g. Ubuntu 18 - https://www.digitalocean.com/community/tutorials/how-to-install-java-with-apt-on-ubuntu-18-04)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:27.010489Z",
     "start_time": "2020-11-17T23:04:18.932048Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "sc = SparkContext()\n",
    "sc.version\n",
    "sc.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:04:29.416261Z",
     "start_time": "2020-11-17T23:04:27.011485Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create my_spark\n",
    "my_spark = SparkSession.builder \\\n",
    "    .master(\"local\") \\\n",
    "    .appName(\"Pandas Alternative\") \\\n",
    "    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:05:33.528531Z",
     "start_time": "2020-11-17T23:04:29.419253Z"
    }
   },
   "outputs": [],
   "source": [
    "tool = \"spark\"\n",
    "stats[tool] = {}\n",
    "s = stats[tool]\n",
    "\n",
    "\n",
    "ts = time()\n",
    "df = my_spark.read.csv(paths[0],inferSchema = True,header= True) \n",
    "te = time()\n",
    "s[\"load_transactions\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "df2 = my_spark.read.csv(paths[1],inferSchema = True,header= True) \n",
    "te = time()\n",
    "s[\"load_identity\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:05:33.849489Z",
     "start_time": "2020-11-17T23:05:33.534687Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "ts = time()\n",
    "dff = df.join(df2, \"TransactionID\")\n",
    "te = time()\n",
    "s[\"merge\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:06:01.148952Z",
     "start_time": "2020-11-17T23:05:33.851490Z"
    }
   },
   "outputs": [],
   "source": [
    "# the difference is that we call collect method, which runs all the computations at this point\n",
    "#ts = time()\n",
    "#grp = dff.groupby([dff[\"isFraud\"],dff[\"ProductCD\"],dff[\"card4\"],dff[\"card6\"],dff[\"id_15\"],dff[\"id_31\"]]) \\\n",
    "#        .agg(avg(\"TransactionAmt\"), sum(\"TransactionAmt\"))\\\n",
    "#        .collect()\n",
    "#te = time()\n",
    "#s[\"aggregation\"] = te-ts\n",
    "#s[\"all\"] = te-tss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:13:24.332254Z",
     "start_time": "2020-11-17T23:13:03.641149Z"
    }
   },
   "outputs": [],
   "source": [
    "# the difference is that we call collect method, which runs all the computations at this point\n",
    "ts = time()\n",
    "grp = dff.groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"]) \\\n",
    "        .agg(avg(\"TransactionAmt\"), sum(\"TransactionAmt\"))\\\n",
    "        .collect()\n",
    "te = time()\n",
    "s[\"aggregation\"] = te-ts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = time()\n",
    "dff.orderBy(\"card1\",\"addr1\",\"D9\").collect()\n",
    "# alternatively\n",
    "# dff.sort(\"card1\",\"addr1\",\"D9\").collect()\n",
    "te = time()\n",
    "s[\"sorting\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = time()\n",
    "dff.select(\"card2\").distinct().collect()\n",
    "te = time()\n",
    "s[\"value_counts\"] = te-ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:06:25.157340Z",
     "start_time": "2020-11-17T23:06:25.118349Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pandas</th>\n",
       "      <th>dask</th>\n",
       "      <th>dask_indexed</th>\n",
       "      <th>vaex</th>\n",
       "      <th>spark</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>load_transactions</th>\n",
       "      <td>20.143912</td>\n",
       "      <td>0.090847</td>\n",
       "      <td>10.722161</td>\n",
       "      <td>16.696253</td>\n",
       "      <td>32.811776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_identity</th>\n",
       "      <td>0.501993</td>\n",
       "      <td>0.017655</td>\n",
       "      <td>0.582975</td>\n",
       "      <td>0.749975</td>\n",
       "      <td>1.250784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merge</th>\n",
       "      <td>2.207136</td>\n",
       "      <td>0.075063</td>\n",
       "      <td>0.048098</td>\n",
       "      <td>0.120835</td>\n",
       "      <td>0.302898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aggregation</th>\n",
       "      <td>0.062498</td>\n",
       "      <td>16.052712</td>\n",
       "      <td>14.059959</td>\n",
       "      <td>0.455176</td>\n",
       "      <td>14.660436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sorting</th>\n",
       "      <td>1.258234</td>\n",
       "      <td>43.264110</td>\n",
       "      <td>40.348259</td>\n",
       "      <td>1.124968</td>\n",
       "      <td>104.669759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value_counts</th>\n",
       "      <td>0.002657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.021019</td>\n",
       "      <td>9.147272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>24.176429</td>\n",
       "      <td>59.500387</td>\n",
       "      <td>65.761452</td>\n",
       "      <td>19.168226</td>\n",
       "      <td>162.842925</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      pandas       dask  dask_indexed       vaex       spark\n",
       "load_transactions  20.143912   0.090847     10.722161  16.696253   32.811776\n",
       "load_identity       0.501993   0.017655      0.582975   0.749975    1.250784\n",
       "merge               2.207136   0.075063      0.048098   0.120835    0.302898\n",
       "aggregation         0.062498  16.052712     14.059959   0.455176   14.660436\n",
       "sorting             1.258234  43.264110     40.348259   1.124968  104.669759\n",
       "value_counts        0.002657        NaN           NaN   0.021019    9.147272\n",
       "Total              24.176429  59.500387     65.761452  19.168226  162.842925"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_df = pd.DataFrame(stats)\n",
    "stats_df.loc['Total'] = stats_df.sum()\n",
    "stats_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:16:19.569545Z",
     "start_time": "2020-11-17T22:16:19.559625Z"
    }
   },
   "source": [
    "# Modin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:57:15.609009Z",
     "start_time": "2020-11-17T22:57:15.586070Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.8.2'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mpd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:00:06.292260Z",
     "start_time": "2020-11-17T22:58:42.702035Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UserWarning: `DataFrame.groupby_on_multiple_columns` defaulting to pandas implementation.\n",
      "To request implementation, send an email to feature_requests@modin.org.\n",
      "FutureWarning: The `squeeze` parameter is deprecated and will be removed in a future version.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'isFraud'\n"
     ]
    }
   ],
   "source": [
    "tool = \"modin\"\n",
    "stats[tool] = {}\n",
    "s = stats[tool]\n",
    "\n",
    "\n",
    "ts = time()\n",
    "df = mpd.read_csv(paths[0])\n",
    "te = time()\n",
    "s[\"load_transactions\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "df2 = mpd.read_csv(paths[1])\n",
    "te = time()\n",
    "s[\"load_identity\"] = te-ts\n",
    "\n",
    "ts = time()\n",
    "dff = df.merge(df2, on=\"TransactionID\")\n",
    "te = time()\n",
    "s[\"merge\"] = te-ts\n",
    "\n",
    "# modin defaults to pandas for multiple column aggregation and then fails on KeyError, though the key is available\n",
    "ts = time()\n",
    "try:\n",
    "    grp = dff.groupby([\"isFraud\",\"ProductCD\",\"card4\",\"card6\",\"id_15\",\"id_31\"])[\"TransactionAmt\"].agg([\"mean\",\"sum\"])\n",
    "except Exception as e:\n",
    "    print(e)\n",
    "te = time()\n",
    "s[\"aggregation\"] = te-ts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T22:58:20.241434Z",
     "start_time": "2020-11-17T22:55:17.025Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pandas</th>\n",
       "      <th>dask</th>\n",
       "      <th>dask_indexed</th>\n",
       "      <th>vaex</th>\n",
       "      <th>spark</th>\n",
       "      <th>modin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>load_transactions</th>\n",
       "      <td>20.143912</td>\n",
       "      <td>0.090847</td>\n",
       "      <td>10.722161</td>\n",
       "      <td>16.696253</td>\n",
       "      <td>32.811776</td>\n",
       "      <td>14.306013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>load_identity</th>\n",
       "      <td>0.501993</td>\n",
       "      <td>0.017655</td>\n",
       "      <td>0.582975</td>\n",
       "      <td>0.749975</td>\n",
       "      <td>1.250784</td>\n",
       "      <td>0.777875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merge</th>\n",
       "      <td>2.207136</td>\n",
       "      <td>0.075063</td>\n",
       "      <td>0.048098</td>\n",
       "      <td>0.120835</td>\n",
       "      <td>0.302898</td>\n",
       "      <td>9.725335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aggregation</th>\n",
       "      <td>0.062498</td>\n",
       "      <td>16.052712</td>\n",
       "      <td>14.059959</td>\n",
       "      <td>0.455176</td>\n",
       "      <td>14.660436</td>\n",
       "      <td>0.341296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sorting</th>\n",
       "      <td>1.258234</td>\n",
       "      <td>43.264110</td>\n",
       "      <td>40.348259</td>\n",
       "      <td>1.124968</td>\n",
       "      <td>104.669759</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value_counts</th>\n",
       "      <td>0.002657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.021019</td>\n",
       "      <td>9.147272</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      pandas       dask  dask_indexed       vaex       spark  \\\n",
       "load_transactions  20.143912   0.090847     10.722161  16.696253   32.811776   \n",
       "load_identity       0.501993   0.017655      0.582975   0.749975    1.250784   \n",
       "merge               2.207136   0.075063      0.048098   0.120835    0.302898   \n",
       "aggregation         0.062498  16.052712     14.059959   0.455176   14.660436   \n",
       "sorting             1.258234  43.264110     40.348259   1.124968  104.669759   \n",
       "value_counts        0.002657        NaN           NaN   0.021019    9.147272   \n",
       "\n",
       "                       modin  \n",
       "load_transactions  14.306013  \n",
       "load_identity       0.777875  \n",
       "merge               9.725335  \n",
       "aggregation         0.341296  \n",
       "sorting                  NaN  \n",
       "value_counts             NaN  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-17T23:01:01.211911Z",
     "start_time": "2020-11-17T23:00:45.850513Z"
    }
   },
   "outputs": [],
   "source": [
    "clean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(stats).to_csv(\"statistics.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import psutil\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(62.9, 82.8)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use psutil to check for system resources\n",
    "psutil.cpu_percent(), psutil.virtual_memory().percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def system_resources(n, pause, cpu_threshold = 0.5, mem_threshold = 0.5):\n",
    "    \n",
    "    cpu_m = []\n",
    "    mem_m = []\n",
    "    cpu_treshold_breached = False\n",
    "    mem_treshold_breached = False\n",
    "    \n",
    "    for i in range(n):\n",
    "        cpu_m.append(psutil.cpu_percent())\n",
    "        mem_m.append(psutil.virtual_memory().percent)\n",
    "        sleep(pause)\n",
    "    cpu_m = np.mean(cpu_m)\n",
    "    mem_m = np.mean(mem_m)\n",
    "    \n",
    "    if cpu_m / 100 > cpu_threshold:\n",
    "        cpu_treshold_breached = True\n",
    "        \n",
    "    if mem_m / 100 > mem_threshold:\n",
    "        mem_treshold_breached = True\n",
    "    \n",
    "    return {\"cpu\": cpu_m, \"memory\": mem_m, \"cpu_threshold\": cpu_treshold_breached, \"mem_threshold\": mem_treshold_breached }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cpu': 25.599999999999998,\n",
       " 'memory': 82.76666666666667,\n",
       " 'cpu_threshold': False,\n",
       " 'mem_threshold': True}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "system_resources(3, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Events:\n",
    "    \n",
    "    def __init__(self, path):\n",
    "        self.file = open(path, 'a', encoding='utf-8')\n",
    "        \n",
    "    def log(self, time, tool, operation, duration):\n",
    "        print(\"|\".join([strftime('%Y-%m-%d %H:%M:%S', localtime(te)),tool,operation,str(duration)])+\"\\n\")\n",
    "        print(self.file)\n",
    "        self.file.write(\"|\".join([strftime('%Y-%m-%d %H:%M:%S', localtime(te)),tool,operation,str(duration)])+\"\\n\")\n",
    "        \n",
    "    def close(self):\n",
    "        self.file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = Events(\"example.log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-25 22:38:38|pandas|load|0.552954912185669\n",
      "\n",
      "<_io.TextIOWrapper name='example.log' mode='a' encoding='utf-8'>\n"
     ]
    }
   ],
   "source": [
    "ts = time()\n",
    "df = pd.read_csv(paths[1])\n",
    "te = time()\n",
    "logger.log(te, \"pandas\", \"load\",  te-ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logger.file.write(\"|\".join([strftime('%Y-%m-%d %H:%M:%S', localtime(te)),\"pandas\",\"load\",str(ts-te)])+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "te = time()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(filename='/Notebooks/Medium/DataFrames/lexample.log', encoding='utf-8', level=logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:MainThread:root:This is an info message\n"
     ]
    }
   ],
   "source": [
    "logging.info('This is an info message')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "big-tables",
   "language": "python",
   "name": "big-tables"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}