Automate EDA tools part1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-12T12:08:44.703611Z",
     "iopub.status.busy": "2022-06-12T12:08:44.702615Z",
     "iopub.status.idle": "2022-06-12T12:08:49.367935Z",
     "shell.execute_reply": "2022-06-12T12:08:49.367935Z",
     "shell.execute_reply.started": "2022-06-12T12:08:44.703611Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.options.display.float_format = '{:,.2f}'.format\n",
    "\n",
    "from IPython.core import display as ICD\n",
    "\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-12T12:08:49.369937Z",
     "iopub.status.busy": "2022-06-12T12:08:49.368940Z",
     "iopub.status.idle": "2022-06-12T12:08:49.587004Z",
     "shell.execute_reply": "2022-06-12T12:08:49.586011Z",
     "shell.execute_reply.started": "2022-06-12T12:08:49.369937Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>date</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>direction</th>\n",
       "      <th>is_expensive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-03</td>\n",
       "      <td>2022-03-03</td>\n",
       "      <td>8.33</td>\n",
       "      <td>41.00</td>\n",
       "      <td>6.98</td>\n",
       "      <td>1.02</td>\n",
       "      <td>322.00</td>\n",
       "      <td>2.56</td>\n",
       "      <td>37.88</td>\n",
       "      <td>-122.23</td>\n",
       "      <td>4.53</td>\n",
       "      <td>North</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-02-18</td>\n",
       "      <td>2022-02-18</td>\n",
       "      <td>8.30</td>\n",
       "      <td>21.00</td>\n",
       "      <td>6.24</td>\n",
       "      <td>0.97</td>\n",
       "      <td>2,401.00</td>\n",
       "      <td>2.11</td>\n",
       "      <td>37.86</td>\n",
       "      <td>-122.22</td>\n",
       "      <td>3.58</td>\n",
       "      <td>North</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>7.26</td>\n",
       "      <td>52.00</td>\n",
       "      <td>8.29</td>\n",
       "      <td>1.07</td>\n",
       "      <td>496.00</td>\n",
       "      <td>2.80</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.24</td>\n",
       "      <td>3.52</td>\n",
       "      <td>North</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-01-08</td>\n",
       "      <td>2022-01-08</td>\n",
       "      <td>5.64</td>\n",
       "      <td>52.00</td>\n",
       "      <td>5.82</td>\n",
       "      <td>1.07</td>\n",
       "      <td>558.00</td>\n",
       "      <td>2.55</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>3.41</td>\n",
       "      <td>North</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-02-14</td>\n",
       "      <td>2022-02-14</td>\n",
       "      <td>3.85</td>\n",
       "      <td>52.00</td>\n",
       "      <td>6.28</td>\n",
       "      <td>1.08</td>\n",
       "      <td>565.00</td>\n",
       "      <td>2.18</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>3.42</td>\n",
       "      <td>North</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    datetime        date  MedInc  HouseAge  AveRooms  AveBedrms  Population  \\\n",
       "0 2022-03-03  2022-03-03    8.33     41.00      6.98       1.02      322.00   \n",
       "1 2022-02-18  2022-02-18    8.30     21.00      6.24       0.97    2,401.00   \n",
       "2 2022-03-09  2022-03-09    7.26     52.00      8.29       1.07      496.00   \n",
       "3 2022-01-08  2022-01-08    5.64     52.00      5.82       1.07      558.00   \n",
       "4 2022-02-14  2022-02-14    3.85     52.00      6.28       1.08      565.00   \n",
       "\n",
       "   AveOccup  Latitude  Longitude  MedHouseVal direction  is_expensive  \n",
       "0      2.56     37.88    -122.23         4.53     North          True  \n",
       "1      2.11     37.86    -122.22         3.58     North          True  \n",
       "2      2.80     37.85    -122.24         3.52     North          True  \n",
       "3      2.55     37.85    -122.25         3.41     North          True  \n",
       "4      2.18     37.85    -122.25         3.42     North          True  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn.datasets\n",
    "\n",
    "data = sklearn.datasets.fetch_california_housing(return_X_y=False, as_frame=True)\n",
    "df = data[\"frame\"]\n",
    "\n",
    "def random_date(start, end, random_amount):\n",
    "    delta = (end - start).days\n",
    "    return [start + timedelta(days=random.randrange(delta)) for i in range(random_amount)]\n",
    "\n",
    "d1 = datetime.strptime('2022-01-01', '%Y-%m-%d')\n",
    "d2 = datetime.strptime('2022-03-31', '%Y-%m-%d')\n",
    "\n",
    "df['datetime'] = random_date(d1,d2,df.shape[0])\n",
    "df['date'] = df['datetime'].apply(lambda x: x.date())\n",
    "df = df.reindex(columns= ['datetime', 'date'] + list(df.columns[:-2]))\n",
    "df['direction'] = df['Latitude'].apply(lambda x: 'North' if x >= 36 else 'South')\n",
    "df['is_expensive'] = df['MedHouseVal'].apply(lambda x: True if x >= 2.5 else False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-12T08:01:30.125924Z",
     "iopub.status.busy": "2022-06-12T08:01:30.125924Z",
     "iopub.status.idle": "2022-06-12T08:01:30.162648Z",
     "shell.execute_reply": "2022-06-12T08:01:30.162648Z",
     "shell.execute_reply.started": "2022-06-12T08:01:30.125924Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 20640 entries, 0 to 20639\n",
      "Data columns (total 13 columns):\n",
      " #   Column        Non-Null Count  Dtype         \n",
      "---  ------        --------------  -----         \n",
      " 0   datetime      20640 non-null  datetime64[ns]\n",
      " 1   date          20640 non-null  object        \n",
      " 2   MedInc        20640 non-null  float64       \n",
      " 3   HouseAge      20640 non-null  float64       \n",
      " 4   AveRooms      20640 non-null  float64       \n",
      " 5   AveBedrms     20640 non-null  float64       \n",
      " 6   Population    20640 non-null  float64       \n",
      " 7   AveOccup      20640 non-null  float64       \n",
      " 8   Latitude      20640 non-null  float64       \n",
      " 9   Longitude     20640 non-null  float64       \n",
      " 10  MedHouseVal   20640 non-null  float64       \n",
      " 11  direction     20640 non-null  object        \n",
      " 12  is_expensive  20640 non-null  bool          \n",
      "dtypes: bool(1), datetime64[ns](1), float64(9), object(2)\n",
      "memory usage: 1.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pandas_profiling\n",
    "- https://pypi.org/project/pandas-profiling/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-14T15:24:52.207623Z",
     "iopub.status.busy": "2022-06-14T15:24:52.206623Z",
     "iopub.status.idle": "2022-06-14T15:24:52.212872Z",
     "shell.execute_reply": "2022-06-14T15:24:52.212623Z",
     "shell.execute_reply.started": "2022-06-14T15:24:52.207623Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pandas_profiling import ProfileReport\n",
    "# create profile report\n",
    "profile = ProfileReport(df, title=\"Pandas Profiling Report\")\n",
    "# save report\n",
    "profile.to_file(\"pandas_profiling.html\")\n",
    "# show on jupyter lab\n",
    "profile.to_widgets()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Describe\n",
    "- Show duplicate row from iris dataset example\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SweetViz\n",
    "- https://pypi.org/project/sweetviz/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-14T15:25:11.239381Z",
     "iopub.status.busy": "2022-06-14T15:25:11.239381Z",
     "iopub.status.idle": "2022-06-14T15:25:11.252094Z",
     "shell.execute_reply": "2022-06-14T15:25:11.251313Z",
     "shell.execute_reply.started": "2022-06-14T15:25:11.239381Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sweetviz as sv\n",
    "analyze_report = sv.analyze(df)\n",
    "analyze_report.show_html('report.html', open_browser=True)\n",
    "# analyze_report.show_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-14T15:25:01.716075Z",
     "iopub.status.busy": "2022-06-14T15:25:01.715075Z",
     "iopub.status.idle": "2022-06-14T15:25:01.723075Z",
     "shell.execute_reply": "2022-06-14T15:25:01.722075Z",
     "shell.execute_reply.started": "2022-06-14T15:25:01.716075Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "analyze_report_2 = sv.compare_intra(df, df[\"direction\"] == \"North\", [\"North\", \"South\"])\n",
    "analyze_report_2.show_html('report_2.html', open_browser=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AutoViz\n",
    "- https://github.com/AutoViML/AutoViz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-12T08:03:55.582420Z",
     "iopub.status.busy": "2022-06-12T08:03:55.582420Z",
     "iopub.status.idle": "2022-06-12T08:03:55.594275Z",
     "shell.execute_reply": "2022-06-12T08:03:55.593780Z",
     "shell.execute_reply.started": "2022-06-12T08:03:55.582420Z"
    }
   },
   "outputs": [],
   "source": [
    "# from autoviz.AutoViz_Class import AutoViz_Class\n",
    "# AV = AutoViz_Class()\n",
    "# df_av = AV.AutoViz(filename=\"\"\n",
    "#                    , dfte=df\n",
    "#                    # , depVar='label'\n",
    "#                    # , verbose=0\n",
    "#                    # , chart_format=\"html\"\n",
    "#                    # , save_plot_dir='autoviz.html'\n",
    "#                       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-14T15:25:24.007844Z",
     "iopub.status.busy": "2022-06-14T15:25:24.007844Z",
     "iopub.status.idle": "2022-06-14T15:25:24.024891Z",
     "shell.execute_reply": "2022-06-14T15:25:24.024390Z",
     "shell.execute_reply.started": "2022-06-14T15:25:24.007844Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from autoviz.AutoViz_Class import AutoViz_Class\n",
    "AV = AutoViz_Class()\n",
    "df_av = AV.AutoViz(filename=\"\"\n",
    "                    , dfte=df\n",
    "                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-14T15:23:54.146788Z",
     "iopub.status.busy": "2022-06-14T15:23:54.146788Z",
     "iopub.status.idle": "2022-06-14T15:23:54.159524Z",
     "shell.execute_reply": "2022-06-14T15:23:54.158522Z",
     "shell.execute_reply.started": "2022-06-14T15:23:54.146788Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from autoviz.AutoViz_Class import AutoViz_Class\n",
    "AV = AutoViz_Class()\n",
    "df_av = AV.AutoViz(filename=\"\"\n",
    "                   , dfte=df\n",
    "                   , chart_format='bokeh'\n",
    "                      )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dtale\n",
    "- https://pypi.org/project/dtale/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-14T15:24:01.296107Z",
     "iopub.status.busy": "2022-06-14T15:24:01.295636Z",
     "iopub.status.idle": "2022-06-14T15:24:01.299459Z",
     "shell.execute_reply": "2022-06-14T15:24:01.298960Z",
     "shell.execute_reply.started": "2022-06-14T15:24:01.296107Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import dtale\n",
    "d = dtale.show(df)\n",
    "d.open_browser() # <-- To show in new browser (Suggested)\n",
    "# dtale.show(df) <-- This for showing in jupyter"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}