develop_main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ebda291c-1f37-49af-bbee-f66d04b4c375",
   "metadata": {},
   "source": [
    "# Notebook developing main.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7c3a11e7-d3b9-47b8-9288-8c1978926935",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pandas as pd\n",
    "import os\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.io as pio\n",
    "import json\n",
    "\n",
    "### import scripts with codes to do eda\n",
    "from scripts import auxiliar as aux\n",
    "from scripts import ydata_profiling as dp\n",
    "from scripts import univariate_analysis as uv\n",
    "from scripts import bivariate_analysis as bv\n",
    "from scripts import segmentation_analysis as se\n",
    "from scripts import categorical_analysis as ca\n",
    "\n",
    "# auxiliar function for dataset example\n",
    "def transform_strings_to_save(var_string):\n",
    "    \"\"\" Replace characters that can be saved in windows \"\"\"\n",
    "    var_string= var_string.replace('/', '_') # replace element bad name windows\n",
    "    var_string= var_string.replace('**', '_') # replace element bad name windows\n",
    "    return var_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00c07a8c-8695-4b6e-8125-c04f1a835cf4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "81ae0c58-c5af-49a2-9187-76b125b4c5ea",
   "metadata": {},
   "source": [
    "### 0. Define parameters of the report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41b4d9d6-7fd6-4962-9ea0-f53627fa1b80",
   "metadata": {},
   "source": [
    "### read json config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "42b4eb16-2e90-4023-9158-f64793f0985c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" read json config \"\"\"\n",
    "path_json = 'config.json'\n",
    "with open(path_json, 'r') as archivo_json:\n",
    "    config = json.load(archivo_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c204833-0441-4ae9-bd2c-504804558c76",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "800c93f3-77ba-48ce-ba94-77878cfc19bb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3b9d90c3-6bad-4488-be11-54988e2069b0",
   "metadata": {},
   "source": [
    "#### read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dc650637-a2d9-43cd-912e-3c5f17892234",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CMPC.SN</th>\n",
       "      <th>CHILE.SN</th>\n",
       "      <th>COPEC.SN</th>\n",
       "      <th>ANDINA-B.SN</th>\n",
       "      <th>MSFT</th>\n",
       "      <th>AAPL</th>\n",
       "      <th>GOOG</th>\n",
       "      <th>TSLA</th>\n",
       "      <th>VOO</th>\n",
       "      <th>QQQ</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2022-01-03</th>\n",
       "      <td>1405.541748</td>\n",
       "      <td>57.923576</td>\n",
       "      <td>6479.199707</td>\n",
       "      <td>1508.288452</td>\n",
       "      <td>328.727661</td>\n",
       "      <td>179.953873</td>\n",
       "      <td>145.074493</td>\n",
       "      <td>399.926666</td>\n",
       "      <td>425.171265</td>\n",
       "      <td>396.005310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-01-04</th>\n",
       "      <td>1357.653931</td>\n",
       "      <td>60.247246</td>\n",
       "      <td>6429.206055</td>\n",
       "      <td>1528.332275</td>\n",
       "      <td>323.090912</td>\n",
       "      <td>177.669983</td>\n",
       "      <td>144.416504</td>\n",
       "      <td>383.196655</td>\n",
       "      <td>424.997040</td>\n",
       "      <td>390.868866</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                CMPC.SN   CHILE.SN     COPEC.SN  ANDINA-B.SN        MSFT  \\\n",
       "Date                                                                       \n",
       "2022-01-03  1405.541748  57.923576  6479.199707  1508.288452  328.727661   \n",
       "2022-01-04  1357.653931  60.247246  6429.206055  1528.332275  323.090912   \n",
       "\n",
       "                  AAPL        GOOG        TSLA         VOO         QQQ  \n",
       "Date                                                                    \n",
       "2022-01-03  179.953873  145.074493  399.926666  425.171265  396.005310  \n",
       "2022-01-04  177.669983  144.416504  383.196655  424.997040  390.868866  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" read data \"\"\"\n",
    "name_data_pkl = config['config_report']['name_data_pkl']\n",
    "path_data_pkl = 'data/' + name_data_pkl\n",
    "data = pd.read_pickle(path_data_pkl)\n",
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "947ab53d-ad1c-4e96-9952-096ab32fe5f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(516, 10)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11327c84-47f8-4d3d-a773-c689416f9d47",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "34b18f78-5805-434d-b67b-6374ae02d3ab",
   "metadata": {},
   "source": [
    "#### read global params and do global actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3954b50b-ef60-4294-bfda-91101a3288e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" read global params and do global actions \"\"\"\n",
    "# define id report\n",
    "name_report = config['config_report']['name_report']\n",
    "datetime_report = datetime.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "id_report = name_report + '-' + datetime_report\n",
    "\n",
    "# define number of columns in the plots\n",
    "param_number_columns = config['config_report']['number_columns']\n",
    "\n",
    "# read params feature target\n",
    "param_target = config['config_report']['target']\n",
    "\n",
    "# read params list feautures\n",
    "param_list_features = config['config_report']['list_features']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9713025e-6eab-4532-8a82-c92b54ee9d6d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1de24f9-0c8d-419f-921e-5eb7a390cbd1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "4847652b-c3c9-4dd1-a1fb-804249fe4e8b",
   "metadata": {},
   "source": [
    "### 1. Create folders neccesary to save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1d6801fc-c223-4281-bf23-da0339d2c880",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Create folders neccesary to save results \"\"\"\n",
    "# create folders to save each kind of reports\n",
    "os.makedirs('output_eda/' + id_report)\n",
    "os.makedirs('output_eda/' + id_report + '/ydata_profiling')\n",
    "os.makedirs('output_eda/' + id_report + '/univariate_analysis')\n",
    "os.makedirs('output_eda/' + id_report + '/bivariate_analysis')\n",
    "os.makedirs('output_eda/' + id_report + '/segmentation_analysis')\n",
    "os.makedirs('output_eda/' + id_report + '/categorical_analysis')\n",
    "\n",
    "# create folder to save univariate analysis - trends\n",
    "os.makedirs('output_eda/' + id_report + '/univariate_analysis/trend') \n",
    "os.makedirs('output_eda/' + id_report + '/univariate_analysis/trend_zoom')\n",
    "\n",
    "\n",
    "# create folder to save bivariate_analysis - scatter plots\n",
    "os.makedirs('output_eda/' + id_report + '/bivariate_analysis/scatter-features-target')\n",
    "os.makedirs('output_eda/' + id_report + '/bivariate_analysis/scatter-features-features')\n",
    "\n",
    "\n",
    "# create folder to save segmentation_analysis - scatter plots\n",
    "os.makedirs('output_eda/' + id_report + '/segmentation_analysis/scatter-features-target')\n",
    "os.makedirs('output_eda/' + id_report + '/segmentation_analysis/scatter-features-features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dcaf26c-d8ac-4dca-a85e-fbb0b0f53ba1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a40006a3-61b2-4d84-9b33-4be2b0eca020",
   "metadata": {},
   "source": [
    "### 2. Define reports to show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c15bb53e-f684-4bfc-aa0e-ef478e300546",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- repots to show ---\n",
      "show_ydata_profiling:  True\n",
      "show_univariate_analysis:  True\n",
      "show_bivariate_analysis:  True\n",
      "show_segmentation_analysis:  True\n",
      "show_categorical_analysis: True\n",
      "--- --- --- --- --- ---\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Define reports to show \"\"\"\n",
    "### define reports to show\n",
    "show_ydata_profiling = config['reports_to_show']['ydata_profiling']\n",
    "show_univariate_analysis = config['reports_to_show']['univariate_analysis']\n",
    "show_bivariate_analysis = config['reports_to_show']['bivariate_analysis']\n",
    "show_segmentation_analysis = config['reports_to_show']['segmentation_analysis']\n",
    "show_categorical_analysis = config['reports_to_show']['categotical_analysis']\n",
    "\n",
    "\n",
    "print('--- repots to show ---')\n",
    "print('show_ydata_profiling: ', show_ydata_profiling)\n",
    "print('show_univariate_analysis: ', show_univariate_analysis)\n",
    "print('show_bivariate_analysis: ', show_bivariate_analysis)\n",
    "print('show_segmentation_analysis: ', show_segmentation_analysis)\n",
    "print('show_categorical_analysis:', show_categorical_analysis)\n",
    "print('--- --- --- --- --- ---')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee399b92-99e0-44e9-874c-3e013044ad22",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c28ea893-5924-4ede-a393-12050e03f115",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fdd3b36-ecf9-4a1f-bc57-d531a1d9eb74",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5f3d33a1-1ae5-42b3-abc9-ed4990c6a143",
   "metadata": {},
   "source": [
    "### 2. ydata-profiling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9bc4a1ae-14a4-4bea-a4ef-ef9fe1157279",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ydata-profiling... time:2024-01-22 00:59:52\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c48251e1dc3b49029cee11877774fe61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Summarize dataset:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "030bda921b214c3499c8eb40498badde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generate report structure:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f670c185fb4843cca2e02b7682090a40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Render HTML:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d54c2ce7d7554cfab54091d99f464ce8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Export report to file:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if show_ydata_profiling:\n",
    "    # read params\n",
    "    param_minimal = config['ydata_profiling']['minimal']\n",
    "\n",
    "    # generate report\n",
    "    print(f'ydata-profiling... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    dp.generate_report_ydata_profiling(df = data, \n",
    "                                       minimal = param_minimal, \n",
    "                                       id_report = id_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39dda89a-6c95-4ff7-8423-f8afb832634c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1e3a959c-bd7e-4f42-b006-fead09d1b129",
   "metadata": {},
   "source": [
    "### 3. univariate_analysis\n",
    "The code is divided in two parts:\n",
    "- read params to generate de plots\n",
    "- generate individual plolty figure of each plot\n",
    "\n",
    "obs: trend plots could be with zoom (subplots and individual plots) and without zoom (only plot individual plots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c04a9a7f-bb98-4ba0-9dfd-dbb7d0f8b480",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "statistics... time:2024-01-22 00:59:57\n",
      "histogram... time:2024-01-22 00:59:57\n",
      "trend data zoom... time:2024-01-22 01:00:00\n",
      "trend full data... time:2024-01-22 01:00:02\n",
      "boxplots... time:2024-01-22 01:00:03\n",
      "moving average... time:2024-01-22 01:00:03\n",
      "weighted moving average... time:2024-01-22 01:00:04\n",
      "exponential moving average... time:2024-01-22 01:00:04\n",
      "acf... time:2024-01-22 01:00:04\n",
      "acf stats models... time:2024-01-22 01:00:05\n",
      "pacf... time:2024-01-22 01:00:07\n",
      "pacf stats models... time:2024-01-22 01:00:08\n",
      "end... time:2024-01-22 01:00:10\n"
     ]
    }
   ],
   "source": [
    "if show_univariate_analysis:\n",
    "    \n",
    "    \"\"\" PARAMS \"\"\"\n",
    "    # read params zoom tendency\n",
    "    param_zoom_start_date = config['univariate_analysis']['zoom_tendency']['start_date']\n",
    "    param_zoom_end_date = config['univariate_analysis']['zoom_tendency']['end_date']\n",
    "    \n",
    "    # read params smooth data\n",
    "    param_smooth_ma_window = config['univariate_analysis']['smooth_data']['moving_average']['window']\n",
    "    param_smooth_wma_weights = config['univariate_analysis']['smooth_data']['weighted_moving_average']['weights']\n",
    "    param_smooth_ema_aplha = config['univariate_analysis']['smooth_data']['exponential_moving_average']['alpha']\n",
    "    \n",
    "    # read params acf/pacf\n",
    "    param_lags = config['univariate_analysis']['acf_pacf']['lags']\n",
    "\n",
    "\n",
    "    \"\"\" PLOTS \"\"\"\n",
    "    ################### fig histogram all features ###################\n",
    "    print(f'statistics... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_statistics = uv.generate_descriptive_statistics(df = data)\n",
    "    fig_statistics.write_html(f\"output_eda/{id_report}/univariate_analysis/statistics.html\")\n",
    "\n",
    "    \n",
    "    ################### fig histogram all features ###################\n",
    "    print(f'histogram... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_hist_all = uv.plot_multiple_hist(df = data, number_columns = param_number_columns)\n",
    "    fig_hist_all.write_html(f\"output_eda/{id_report}/univariate_analysis/histograms.html\")\n",
    "\n",
    "    fig_hist_kde_all = uv.plot_kde_hist(df = data, number_columns = param_number_columns)\n",
    "    fig_hist_kde_all.savefig(f\"output_eda/{id_report}/univariate_analysis/histograms_kde.png\", dpi = 300)\n",
    "\n",
    "\n",
    "    ################### zoom data to tendency plots (trend & moving averavge) - zoom to reduce cost to plot ###################\n",
    "    data_zoom = data.loc[param_zoom_start_date:param_zoom_end_date]\n",
    "\n",
    "    \n",
    "    # ################### data zoom - trend ###################\n",
    "    print(f'trend data zoom... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')  \n",
    "    # fig tendency all features in subplots\n",
    "    fig_tendency_all_subplots = uv.plot_multiple_tendency(df = data_zoom, number_columns = 1)\n",
    "    fig_tendency_all_subplots.write_html(f\"output_eda/{id_report}/univariate_analysis/trend_zoom/subplots_zoomtendency.html\")\n",
    "\n",
    "    # fig tendency all features in individual plots\n",
    "    for feature_ in param_list_features:\n",
    "        fig_tendency_all_individual = uv.plot_tendency(df = data_zoom, feature_plot = feature_)\n",
    "        feature_ = transform_strings_to_save(feature_) # replace bad characters to save name\n",
    "        fig_tendency_all_individual.write_html(f\"output_eda/{id_report}/univariate_analysis/trend_zoom/tendency_{feature_}.html\")\n",
    "    \n",
    "    # fig tendency all features in oneplot\n",
    "    fig_tendency_all_oneplot = uv.plot_all_trend_oneplot(df = data_zoom)\n",
    "    fig_tendency_all_oneplot.write_html(f\"output_eda/{id_report}/univariate_analysis/trend_zoom/oneplot_zoom_tendency.html\")\n",
    "\n",
    "    \n",
    "    # ################### full data - trend ###################\n",
    "    print(f'trend full data... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    for feature_ in param_list_features:\n",
    "        fig_tendency_all_individual = uv.plot_tendency(df = data_zoom, feature_plot = feature_)\n",
    "        feature_ = transform_strings_to_save(feature_) # replace bad characters to save name\n",
    "        fig_tendency_all_individual.write_html(f\"output_eda/{id_report}/univariate_analysis/trend/tendency_{feature_}.html\")    \n",
    "\n",
    "    \n",
    "    # ################### fig boxplot for each month and year ###################  \n",
    "    print(f'boxplots... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')    \n",
    "    fig_boxplot_all = uv.plot_multiple_boxplot_months(df = data, number_columns = 1)  # always 1 boxplot for column beacuse there are 12 months\n",
    "    fig_boxplot_all.write_html(f\"output_eda/{id_report}/univariate_analysis/boxplots.html\")\n",
    "    \n",
    "    # ################### fig smooth data ###################    \n",
    "    ## moving average\n",
    "    print(f'moving average... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    data_moving_average = uv.apply_moving_average(df = data_zoom.copy(), window_size = param_smooth_ma_window)\n",
    "    fig_moving_average = uv.plot_compare_tendencias(df_original = data_zoom, \n",
    "                                                    df_smoothed = data_moving_average,\n",
    "                                                    number_columns = param_number_columns,\n",
    "                                                    kind_smooth = f'moving average - window: {param_smooth_ma_window}'\n",
    "                                                )\n",
    "    fig_moving_average.write_html(f\"output_eda/{id_report}/univariate_analysis/moving_average.html\")\n",
    "    \n",
    "    ## weighted moving average\n",
    "    print(f'weighted moving average... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    data_weighted_moving_average = uv.apply_weighted_moving_average(df = data_zoom.copy(), weights = param_smooth_wma_weights)\n",
    "    fig_weighted_moving_average = uv.plot_compare_tendencias(df_original = data_zoom,\n",
    "                                                             df_smoothed = data_weighted_moving_average,\n",
    "                                                             number_columns = param_number_columns,\n",
    "                                                             kind_smooth = f'weighted moving average - weights: [{param_smooth_wma_weights}]'\n",
    "                                                            )\n",
    "    fig_weighted_moving_average.write_html(f\"output_eda/{id_report}/univariate_analysis/weighted_moving_average.html\")\n",
    "    \n",
    "    ## exponential moving average\n",
    "    print(f'exponential moving average... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    data_exponential_moving_average = uv.apply_exponential_moving_average(df = data_zoom.copy(), alpha = param_smooth_ema_aplha)\n",
    "    fig_exponential_moving_average = uv.plot_compare_tendencias(df_original = data_zoom,\n",
    "                                                                df_smoothed = data_exponential_moving_average,\n",
    "                                                                number_columns = param_number_columns,\n",
    "                                                                kind_smooth = f'exponential moving average - alpha: {param_smooth_ema_aplha}'\n",
    "                                                               )\n",
    "    fig_exponential_moving_average.write_html(f\"output_eda/{id_report}/univariate_analysis/exponential_moving_average.html\")\n",
    "    \n",
    "    \n",
    "    # ################### fig acf ###################\n",
    "    print(f'acf... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_acf = uv.plot_all_acf(df = data, lags = param_lags, number_columns = param_number_columns)\n",
    "    fig_acf.write_html(f\"output_eda/{id_report}/univariate_analysis/acf.html\")\n",
    "\n",
    "    print(f'acf stats models... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_acf_stats = uv.plot_all_acf_stats(df = data, lags = param_lags, number_columns = param_number_columns) # v2 statsmodels\n",
    "    fig_acf_stats.savefig(f\"output_eda/{id_report}/univariate_analysis/acf_stats.png\", dpi = 300)\n",
    "    \n",
    "    \n",
    "    # ################### fig pacf ###################\n",
    "    print(f'pacf... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_pacf = uv.plot_all_pacf(df = data, lags = param_lags, number_columns = param_number_columns)\n",
    "    fig_pacf.write_html(f\"output_eda/{id_report}/univariate_analysis/pacf.html\")\n",
    "\n",
    "    print(f'pacf stats models... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_acf_stats = uv.plot_all_pacf_stats(df = data, lags = param_lags, number_columns = param_number_columns) # v2 statsmodels\n",
    "    fig_acf_stats.savefig(f\"output_eda/{id_report}/univariate_analysis/pacf_stats.png\", dpi = 300)\n",
    "\n",
    "\n",
    "    # ################### end ###################\n",
    "    print(f'end... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "348ca436-0b93-4bc8-b484-54134b9d557a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "899d6764-65b8-40fe-a017-73a74d9b36d9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "60cfd466-1fe4-45b0-935f-a8deb5648196",
   "metadata": {},
   "source": [
    "### 4. bivariate_analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "174d508e-361c-495c-9f51-995a427ca4ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "correlations... time:2024-01-22 01:00:10\n",
      "scatters plots... time:2024-01-22 01:00:11\n",
      "correlations features lagged vs target... time:2024-01-22 01:00:19\n",
      "calculating corr with lag: 0\n",
      "calculating corr with lag: 20\n",
      "parallel continuous ... time:2024-01-22 01:00:19\n",
      "end... time:2024-01-22 01:00:19\n"
     ]
    }
   ],
   "source": [
    "if show_bivariate_analysis:\n",
    "    \"\"\" PARAMS \"\"\"\n",
    "    # read param correlations\n",
    "    param_theshold_corr_all_features = config['bivariate_analysis']['correlations']['threshold_corr_all_features']  # threshold in correlations between each feature \n",
    "    param_theshold_corr_target = config['bivariate_analysis']['correlations']['threshold_corr_target']  # threshold in correlations between a target\n",
    "    \n",
    "    # read param scatter plot individual\n",
    "    param_individual_scatter_marginal = config['bivariate_analysis']['scatter_plot']['individual_scatter']['marginal']\n",
    "        \n",
    "    # read param corr features lagged vs target\n",
    "    param_lag_features = config['bivariate_analysis']['correlations_features_lagged_target']['lags']\n",
    "\n",
    "    # read params parallel\n",
    "    param_features_parallel = config['bivariate_analysis']['parallel']['list_features']\n",
    "    param_features_target_parallel = param_features_parallel + [param_target]\n",
    "    \n",
    "    \n",
    "    \"\"\" PLOTS \"\"\"\n",
    "    ################### fig correlations ###################\n",
    "    print(f'correlations... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    \n",
    "    # correlations all features\n",
    "    _, df_corr_upper = bv.calculate_correlations_triu(data)\n",
    "    df_corr_upper_filtered = bv.filter_correlations_by_threshold(df_corr_upper, param_theshold_corr_all_features)\n",
    "    fig_corr_all = bv.plot_heatmap(df_corr = df_corr_upper_filtered)\n",
    "    fig_corr_all.write_html(f\"output_eda/{id_report}/bivariate_analysis/corr_all.html\")\n",
    "    \n",
    "    # correlations against the target\n",
    "    corr_target = bv.calculate_correlations_target(data, param_target)\n",
    "    corr_target_filtered = bv.filter_correlations_by_threshold(corr_target, param_theshold_corr_target)\n",
    "    fig_corr_target = bv.plot_heatmap(df_corr = corr_target_filtered)\n",
    "    fig_corr_target.write_html(f\"output_eda/{id_report}/bivariate_analysis/corr_target.html\")\n",
    "    \n",
    "    \n",
    "    ################### fig scatter plots ###################\n",
    "    print(f'scatters plots... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "\n",
    "    # --- plots features-target - subplots. save in scatter-features-target\n",
    "    fig_scatter_features_target = bv.plot_features_to_target_scatter_plot_low(df = data, target = param_target, number_columns = param_number_columns)\n",
    "    fig_scatter_features_target.write_html(f\"output_eda/{id_report}/bivariate_analysis/scatter-features-target/scatter_features_target.html\")\n",
    "    pio.write_image(fig_scatter_features_target, f\"output_eda/{id_report}/bivariate_analysis/scatter-features-target/scatter_features_target.png\")  # -----> save as png because a lot of plots could be generated and freeze the pc in a html file\n",
    "\n",
    "    # --- plots features-target - individual. save in scatter-features-target\n",
    "    for feature_ in param_list_features:\n",
    "        fig_individual_scatter_features_target = bv.plot_individual_scatter_plot(df = data, feature_x = feature_, feature_y = param_target,\n",
    "                                                                                   marginal_hist = param_individual_scatter_marginal)\n",
    "        feature_ = transform_strings_to_save(feature_) # replace bad characters to save name\n",
    "        fig_individual_scatter_features_target.write_html(f\"output_eda/{id_report}/bivariate_analysis/scatter-features-target/scatter-{feature_}.html\")\n",
    "        \n",
    "    # --- plots features-features - scatter matrix. save in scatter-features-features\n",
    "    #fig_scatter_all_features = bv.plot_all_features_scatter_plot_mine(df = data, number_columns = param_number_columns) ## mine old\n",
    "    fig_scatter_features_features = bv.plot_all_features_scatter_plot(df = data[param_list_features])\n",
    "    fig_scatter_features_features.write_html(f\"output_eda/{id_report}/bivariate_analysis/scatter-features-features/scatter_matrix_features_features.html\")\n",
    "    \n",
    "    # --- plots features-features - individual. save in scatter-features-features\n",
    "    list_features_features = bv.list_map_features_features(df = data[param_list_features])\n",
    "    for feature_x, feature_y in list_features_features:\n",
    "        fig_individual_scatter_features_features = bv.plot_individual_scatter_plot(df = data, feature_x = feature_x, feature_y = feature_y,\n",
    "                                                                                   marginal_hist = param_individual_scatter_marginal)\n",
    "        feature_x = transform_strings_to_save(feature_x) # replace bad characters to save name\n",
    "        feature_y = transform_strings_to_save(feature_y) # replace bad characters to save name\n",
    "        fig_individual_scatter_features_features.write_html(f\"output_eda/{id_report}/bivariate_analysis/scatter-features-features/scatter-{feature_x}-{feature_y}.html\")\n",
    "\n",
    "    \n",
    "    \n",
    "    ################### fig correlations features lagged vs target ###################\n",
    "    print(f'correlations features lagged vs target... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    df_corr_features_lag_target = bv.calculate_corr_features_lag_target(df = data, target = param_target, lags = param_lag_features)\n",
    "    fig_corr_lag = bv.plot_corr_features_lag_target(df_corr_lags = df_corr_features_lag_target)\n",
    "    fig_corr_lag.write_html(f\"output_eda/{id_report}/bivariate_analysis/plot_corr_features_lag_target.html\")\n",
    "\n",
    "\n",
    "\n",
    "    ################### fig parallel all continuous variables ###################\n",
    "    print(f'parallel continuous ... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_parallel_continuous = bv.plot_parallel_continuous(df = data, \n",
    "                                                         list_features_target = param_features_target_parallel, \n",
    "                                                         target = param_target)\n",
    "    fig_parallel_continuous.write_html(f\"output_eda/{id_report}/bivariate_analysis/parallel_continous_variables.html\")\n",
    "    \n",
    "    \n",
    "    # ################### end ###################\n",
    "    print(f'end... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b62d45c-f7d7-4a71-925b-850d1424d42a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0a450bf-ea0c-40f0-8009-e599ba2406c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "21df8876-38ca-4cfb-b56a-02090115361b",
   "metadata": {},
   "source": [
    "### 5. segmentation_analysis\n",
    "The segmentation analysis se puede hacer sobre múltiples segmentaciones de los datos independientes\n",
    "\n",
    "TODO: \n",
    "1. modify codes to multiple independient segmentations\n",
    "2. modify codes to select the kind of plot to do"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3390b762-1cc8-44ea-9b7c-a22f57b0407e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "freq segment... time:2024-01-22 01:00:19\n",
      "descriptive staticstics segmented segmented data... time:2024-01-22 01:00:19\n",
      "histograms - boxplots segmented data... time:2024-01-22 01:00:19\n",
      "trend - scatter segmentation... time:2024-01-22 01:00:19\n",
      "correlations segmented data... time:2024-01-22 01:00:21\n",
      "scatters plots segmented... time:2024-01-22 01:00:21\n",
      "parallel discrete target... time:2024-01-22 01:00:27\n",
      "end... time:2024-01-22 01:00:27\n"
     ]
    }
   ],
   "source": [
    "if show_segmentation_analysis:\n",
    "    \"\"\" PARAMS \"\"\"\n",
    "    # list of independient segmentations of the data\n",
    "    list_segments_data = config['segmentation_analysis']['segments']\n",
    "    config_segments_data = list_segments_data[0] # TODO: modify codes to multiple independient segmentations\n",
    "    \n",
    "    param_segments_data_var = config_segments_data['var_segment']\n",
    "    param_segments_data_intervals = config_segments_data['interval_segment']\n",
    "    param_segments_data_labels = config_segments_data['labels_segment']\n",
    "    \n",
    "    # params correlations\n",
    "    param_segmentation_corr_all_threshold = config['segmentation_analysis']['correlations']['threshold_corr_all_features']\n",
    "    param_segmentation_corr_target_threshold = config['segmentation_analysis']['correlations']['threshold_corr_target']\n",
    "\n",
    "    # read param scatter plot individual\n",
    "    param_segmentation_individual_scatter_marginal = config['segmentation_analysis']['scatter_plot']['individual_scatter']['marginal']\n",
    "\n",
    "    # read param parallel discrete target\n",
    "    param_parallel_discrete_target_show = config['segmentation_analysis']['parallel_target_discrete']['show']\n",
    "    param_features_parallel_discrete_target = config['segmentation_analysis']['parallel_target_discrete']['list_features']\n",
    "    param_features_target_parallel_discrete_target = param_features_parallel_discrete_target + [param_segments_data_var + '_segments'] # list features and target with the suffix \"_segment\" beacuase the target is segmented\n",
    "\n",
    "\n",
    "    \"\"\" GENERATE DATA SEGMENTED \"\"\" # segment and sort data by variable segment to do all the plots in order incremental of the segmentation\n",
    "    data_segmented = aux.custom_segmentation(df = data.copy(), \n",
    "                                            var_segment = param_segments_data_var, \n",
    "                                            intervals_segments = param_segments_data_intervals, \n",
    "                                            labels_segments = param_segments_data_labels\n",
    "                                           )\n",
    "\n",
    "    \"\"\" PLOTS \"\"\"\n",
    "    ################### freq each segment ###################\n",
    "    print(f'freq segment... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_freq_segmentation = se.plot_freq_segmentation(df = data_segmented, var_segment = param_segments_data_var)\n",
    "    fig_freq_segmentation.write_html(f\"output_eda/{id_report}/segmentation_analysis/freq_segmentation.html\")\n",
    "    \n",
    "    \n",
    "    ################### descriptive statistics ###################\n",
    "    print(f'descriptive staticstics segmented segmented data... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    dict_df_statistics_segment = se.calculate_descriptive_statistics_segment(df = data_segmented, var_segment = param_segments_data_var + '_segments')\n",
    "    df_statistics_segments = se.merge_segmentation_statistics(dict_df_statistics_segment)\n",
    "    fig_statistics_segmentation = se.plot_descriptive_statistics_segment(df_statistics_segments)\n",
    "    fig_statistics_segmentation.write_html(f\"output_eda/{id_report}/segmentation_analysis/statistics_segmentation.html\")\n",
    "    \n",
    "    \n",
    "    ################### histograms - boxplots ###################\n",
    "    print(f'histograms - boxplots segmented data... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    # hist\n",
    "    fig_hist_segment = se.plot_histograms_segments(df = data_segmented, var_segment = param_segments_data_var + '_segments', \n",
    "                                                 number_columns = param_number_columns)\n",
    "    fig_hist_segment.write_html(f\"output_eda/{id_report}/segmentation_analysis/histograms_segmentation.html\")\n",
    "    \n",
    "    # boxplot\n",
    "    fig_boxplots_segment = se.plot_boxplots_segments(df = data_segmented, var_segment = param_segments_data_var + '_segments', \n",
    "                                                 number_columns = param_number_columns)\n",
    "    fig_boxplots_segment.write_html(f\"output_eda/{id_report}/segmentation_analysis/boxplots_segmentation.html\")\n",
    "    \n",
    "    \n",
    "    ################### trend - scatter segmentation ###################\n",
    "    print(f'trend - scatter segmentation... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    data_segmented_sort_index = data_segmented.sort_index() # sort data by index\n",
    "    fig_trend_segment = se.plot_multiple_tendency_segmentation(df = data_segmented_sort_index, var_segment = param_segments_data_var + '_segments', \n",
    "                                                               number_columns = 1)\n",
    "    fig_trend_segment.write_html(f\"output_eda/{id_report}/segmentation_analysis/trend_segmentation.html\")\n",
    "    \n",
    "    ################### correlations ###################\n",
    "    print(f'correlations segmented data... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    \n",
    "    # corr all features\n",
    "    dict_df_corr_segment = se.calculate_correlations_triu_segmentation(df =  data_segmented, \n",
    "                                                                       var_segment = param_segments_data_var + '_segments')\n",
    "    dict_df_corr_segment = se.filter_correlations_segment_by_threshold(dict_df_corr_segment, threshold = param_segmentation_corr_all_threshold)\n",
    "    fig_corr_segmentation_heatmap = se.plot_corr_segmentation_subplots_heatmap(dict_df_corr_segment)\n",
    "    fig_corr_segmentation_heatmap.write_html(f\"output_eda/{id_report}/segmentation_analysis/corr_segmentation_heatmap.html\")\n",
    "    \n",
    "    # corr target\n",
    "    dict_df_corr_segment_target = se.calculate_correlations_target_segmentation(df =  data_segmented, \n",
    "                                                                                var_segment = param_segments_data_var + '_segments', \n",
    "                                                                                target = param_target)\n",
    "    dict_df_corr_segment_target = se.filter_correlations_segment_by_threshold(dict_df_corr_segment_target, threshold = param_segmentation_corr_target_threshold)\n",
    "    fig_corr_segmentation_target_barchat = se.plot_corr_segmentation_vertical_barchart(dict_df_corr_segment_target)\n",
    "    fig_corr_segmentation_target_barchat.write_html(f\"output_eda/{id_report}/segmentation_analysis/corr_segmentation_target_barchat.html\")\n",
    "    fig_corr_segmentation_target_barchat.write_image(f\"output_eda/{id_report}/segmentation_analysis/corr_segmentation_target_barchat.png\")\n",
    "    \n",
    "    \n",
    "    ################### fig scatter plots ###################\n",
    "    # # individual scatter\n",
    "    # if param_segmentation_individual_scatter_show == True:\n",
    "    #     fig_segmentation_individual_scatter = se.plot_individual_scatter_plot_x_y_segment(df = data_segmented, \n",
    "    #                                                              feature_x = param_segmentation_individual_scatter_feature_x, \n",
    "    #                                                              feature_y = param_segmentation_individual_scatter_feature_y, \n",
    "    #                                                              var_segment = param_segments_data_var + '_segments')\n",
    "    #     fig_segmentation_individual_scatter.write_html(f\"output_eda/{id_report}/segmentation_analysis/segmentation_individual_scatter.html\")\n",
    "        \n",
    "    # # scatter all features vs all features\n",
    "    # if param_segmentation_features_scatter_show == True:\n",
    "    #     #fig_segmentation_scatter_all_features = se.plot_all_features_scatter_plot_segment_mine(df = data_segmented, var_segment = param_segments_data_var + '_segments', number_columns = param_number_columns)\n",
    "    #     fig_segmentation_scatter_all_features = se.plot_all_features_scatter_plot_segment(df = data_segmented, var_segment = param_segments_data_var + '_segments')\n",
    "    #     fig_segmentation_scatter_all_features.write_html(f\"output_eda/{id_report}/segmentation_analysis/segmentation_scatter_matrix_all_features.html\")\n",
    "    #     #pio.write_image(fig_segmentation_scatter_all_features, f\"output_eda/{id_report}/segmentation_analysis/segmentation_scatter_matrix_all_features.png\") \n",
    "    \n",
    "\n",
    "\n",
    "    ################### fig scatter plots ###################\n",
    "    print(f'scatters plots segmented... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "\n",
    "\n",
    "    # --- plots features-target - subplots. save in scatter-features-target\n",
    "    # TODO\n",
    "    list_features_features_segmented = param_list_features + [param_segments_data_var + '_segments']\n",
    "    \n",
    "    # --- plots features-target - individual. save in scatter-features-target\n",
    "    for feature_ in param_list_features:\n",
    "        fig_individual_scatter_features_target = se.plot_individual_scatter_plot_segment(df = data_segmented, feature_x = feature_, feature_y = param_target,\n",
    "                                                                                         var_segment = param_segments_data_var + '_segments',\n",
    "                                                                                   marginal_hist = param_segmentation_individual_scatter_marginal)\n",
    "        feature_ = transform_strings_to_save(feature_) # replace bad characters to save name\n",
    "        fig_individual_scatter_features_target.write_html(f\"output_eda/{id_report}/segmentation_analysis/scatter-features-target/scatter-{feature_}.html\")\n",
    "\n",
    "    \n",
    "    # --- plots features-features - scatter matrix. save in scatter-features-features\n",
    "    #fig_scatter_all_features = se.plot_all_features_scatter_plot_mine(df = data_segmented, var_segment = param_segments_data_var + '_segments', number_columns = param_number_columns) ## mine old\n",
    "    fig_scatter_features_features = se.plot_all_features_scatter_plot_segment(df = data_segmented[list_features_features_segmented], \n",
    "                                                                      var_segment = param_segments_data_var + '_segments')\n",
    "    fig_scatter_features_features.write_html(f\"output_eda/{id_report}/segmentation_analysis/scatter-features-features/scatter_matrix_features_features.html\")\n",
    "\n",
    "    \n",
    "    # --- plots features-features - individual. save in scatter-features-features\n",
    "    list_features_features = se.list_map_features_features(df = data_segmented[param_list_features])\n",
    "    for feature_x, feature_y in list_features_features:\n",
    "        fig_individual_scatter_features_features = se.plot_individual_scatter_plot_segment(df = data_segmented, feature_x = feature_x, feature_y = feature_y,\n",
    "                                                                                           var_segment = param_segments_data_var + '_segments',\n",
    "                                                                                   marginal_hist = param_segmentation_individual_scatter_marginal)\n",
    "        feature_x = transform_strings_to_save(feature_x) # replace bad characters to save name\n",
    "        feature_y = transform_strings_to_save(feature_y) # replace bad characters to save name\n",
    "        fig_individual_scatter_features_features.write_html(f\"output_eda/{id_report}/segmentation_analysis/scatter-features-features/scatter-{feature_x}--{feature_y}.html\")\n",
    "\n",
    "\n",
    "    ################### fig parallel plots ###################\n",
    "    if param_parallel_discrete_target_show == True:\n",
    "        print(f'parallel discrete target... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "        fig_parallel_target_discrete = se.plot_parallel_continuous_discrete_target(df = data_segmented, \n",
    "                                                                        list_features_target = param_features_target_parallel_discrete_target, \n",
    "                                                                        var_segment_target_discrete = param_segments_data_var + '_segments')\n",
    "        fig_parallel_target_discrete.write_html(f\"output_eda/{id_report}/segmentation_analysis/parallel_target_discrete.html\")\n",
    "\n",
    "\n",
    "    # ################### end ###################\n",
    "    print(f'end... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9153bf08-68a4-4b32-bd3e-8ef7a6a099ef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b79b328-7ffd-4305-87e9-e2a46dda379c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "69daeba9-9980-470f-8354-7e1c230096f2",
   "metadata": {},
   "source": [
    "### 6. categorical_analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0dec9511-28ac-4580-b9c4-d3ad7b0420b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "table freq each catergory each feature.. time:2024-01-22 01:00:27\n",
      "hist2d freq each feature categortical... time:2024-01-22 01:00:27\n",
      "statistics target each category each feature... time:2024-01-22 01:00:30\n",
      "boxplot target each category each feature... time:2024-01-22 01:00:30\n",
      "statistics target categorical each feature categorical... time:2024-01-22 01:00:30\n",
      "barplot freq target each category each feature... time:2024-01-22 01:00:31\n",
      "heatmap continous target vs 2 categorical features... time:2024-01-22 01:00:31\n",
      "heatmap continous target vs 3 categorical features... time:2024-01-22 01:00:34\n",
      "barplot categorical target vs 2 categorical features... time:2024-01-22 01:00:42\n",
      "parellel categorical features and categorical target... time:2024-01-22 01:00:45\n",
      "end... time:2024-01-22 01:00:45\n"
     ]
    }
   ],
   "source": [
    "if show_categorical_analysis:\n",
    "    \"\"\" PARAMS \"\"\" # categorical analysis: ca\n",
    "    \n",
    "    # params with features and kind of percentile transformation\n",
    "    # list features\n",
    "    param_list_features_ca = config['categorical_analysis']['percentile_transform']['categories_features']['features']\n",
    "    param_list_cat_features_percentile_ca = config['categorical_analysis']['percentile_transform']['categories_features']['percentile']\n",
    "    \n",
    "    # list feature+target\n",
    "    param_list_target_ca = config['categorical_analysis']['percentile_transform']['categories_target']['target']\n",
    "    param_list_target_percentile_ca = config['categorical_analysis']['percentile_transform']['categories_target']['percentile']\n",
    "    param_list_features_target_ca = param_list_features_ca + param_list_target_ca\n",
    "    param_list_cat_features_target_percentile_ca = param_list_cat_features_percentile_ca + param_list_target_percentile_ca\n",
    "\n",
    "\n",
    "    ## GENERATE LIST OF NAMES OF FEATURES AND TARGET when the data is transformed into categorical change its name adding a suffix the percentile\n",
    "    param_target_name_percentile = param_list_target_percentile_ca[0] + '_' + param_list_target_ca[0]\n",
    "\n",
    "    \n",
    "    # params to calculate table/heatmap/hist2d of frequency between eaach pair of features in the data\n",
    "    param_ct_normalized_freq_pair_feature = config['categorical_analysis'][\"crosstab_freq_pair_features\"][\"freq_normalized\"]\n",
    "    \n",
    "    # params to calculate table/heatmap of frequency between each target categorical vs feature categorical\n",
    "    param_ct_normalized_freq_target_feature = config['categorical_analysis'][\"crosstab_freq_target_feature\"][\"freq_normalized\"]\n",
    "    \n",
    "    # param functions of aggregation target in heatmap feature1 & feature 2 vs target\n",
    "    param_list_agg_target_multiple_features = config['categorical_analysis'][\"heatmap_multiple_features_vs_target_continuous\"][\"aggregation_target\"]\n",
    "    \n",
    "    # param list of features and target to plot into a parellel plot - read features originals name and transform name according percetile transformtation\n",
    "    param_list_features_to_parallel_original = config['categorical_analysis'][\"parallel\"][\"list_features\"]\n",
    "    list_indexes_to_parallel_plot = []\n",
    "    for index in range(len(param_list_features_ca)):\n",
    "        if param_list_features_ca[index] in param_list_features_to_parallel_original:\n",
    "            list_indexes_to_parallel_plot.append(index)\n",
    "            \n",
    "    list_percentile_to_parallel_plot = [param_list_cat_features_percentile_ca[element_index] for element_index in list_indexes_to_parallel_plot]\n",
    "    param_list_features_to_parallel = [list_percentile_to_parallel_plot[index] + '_' + param_list_features_to_parallel_original[index] \\\n",
    "                                       for index in range(len(list_percentile_to_parallel_plot))]\n",
    "\n",
    "    param_list_features_target_to_parallel = param_list_features_to_parallel + [param_target_name_percentile]\n",
    "\n",
    "\n",
    "    \"\"\" GENERATE DATA CATEGORICAL \"\"\"\n",
    "    # categorize only features and conserve continuos target\n",
    "    data_percentile_feature = data.copy()\n",
    "    for index, variable in enumerate(param_list_features_ca):\n",
    "        data_percentile_feature = aux.percentile_segmentation(df = data_percentile_feature, \n",
    "                                                              var_segment = variable, \n",
    "                                                              type_percentile = param_list_cat_features_percentile_ca[index]\n",
    "                                                         )\n",
    "        data_percentile_feature.drop(columns = variable, inplace = True)\n",
    "    \n",
    "    \n",
    "    # categorize features+target and delete features continous variables\n",
    "    data_percentile_feature_target = data.copy()\n",
    "    for index, variable in enumerate(param_list_features_target_ca):\n",
    "        data_percentile_feature_target = aux.percentile_segmentation(df = data_percentile_feature_target, \n",
    "                                                                 var_segment = variable, \n",
    "                                                                 type_percentile = param_list_cat_features_target_percentile_ca[index]\n",
    "                                                                )\n",
    "        data_percentile_feature_target.drop(columns = variable, inplace = True)\n",
    "\n",
    "\n",
    "    \n",
    "    \"\"\" PLOTS \"\"\"\n",
    "    # --- table frequency for each categorie for each feature\n",
    "    print(f'table freq each catergory each feature.. time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    df_freq_categorical_variables, df_freq_categorical_variables_plotly = ca.calculate_freq_data(df = data_percentile_feature_target)\n",
    "    fig_table_freq_categorical_variables = ca.plot_df_table_plotly(df_to_plotly = df_freq_categorical_variables_plotly)\n",
    "    fig_table_freq_categorical_variables.write_html(f\"output_eda/{id_report}/categorical_analysis/table_freq_categorical_variables.html\")\n",
    "    df_freq_categorical_variables.to_excel(f\"output_eda/{id_report}/categorical_analysis/df_freq_categorical_variables.xlsx\")\n",
    "    \n",
    "    \n",
    "    \n",
    "    # --- plots/tables/heatmap-> hist2d frecuency between each pair of features\n",
    "    print(f'hist2d freq each feature categortical... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_freq_heatmap_all_features_percentile = ca.heatmap_hist2d_features_percentile(df = data_percentile_feature, \n",
    "                                                                                  target = param_target, \n",
    "                                                                                  ct_normalized = param_ct_normalized_freq_pair_feature)\n",
    "    fig_freq_heatmap_all_features_percentile.write_html(f\"output_eda/{id_report}/categorical_analysis/freq_heatmap_hist2d_all_features_percentile.html\")\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    # --- plots Individual analysis between \"feature x categorical\" and \"target y Continuous\" \n",
    "    \n",
    "    # table statistics of target for each category in each feature\n",
    "    print(f'statistics target each category each feature... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    df_statistics_target, df_statistics_target_to_plotly = ca.descriptive_statistics_target_for_each_feature(df = data_percentile_feature,\n",
    "                                                                                                     target = param_target)\n",
    "    fig_table_statistics_target_to_plotly = ca.plot_df_table_plotly(df_statistics_target_to_plotly)\n",
    "    fig_table_statistics_target_to_plotly.write_html(f\"output_eda/{id_report}/categorical_analysis/table_statistics_target.html\")\n",
    "    df_statistics_target.to_excel(f\"output_eda/{id_report}/categorical_analysis/statistics_target.xlsx\")\n",
    "    \n",
    "    \n",
    "    # boxplot of target for each category in each feature\n",
    "    print(f'boxplot target each category each feature... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_boxplot_target_colored_cartegorical_features = ca.plot_boxplots_target_categorical_features(df = data_percentile_feature, \n",
    "                                                                                                 var_continuous_hist = param_target, # target continous\n",
    "                                                                                                 number_columns = param_number_columns)\n",
    "    fig_boxplot_target_colored_cartegorical_features.write_html(f\"output_eda/{id_report}/categorical_analysis/boxplot_target_colored_cartegorical_features.html\")\n",
    "    \n",
    "    \n",
    "    \n",
    "    # --- plots Individual analysis between \"feature x categorical\" and \"target y categorical\"\n",
    "    \n",
    "    # table freq of target categorical for each category in each feature\n",
    "    print(f'statistics target categorical each feature categorical... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    df_freq_target_each_feature, df_freq_target_each_feature_plotly = ca.calculate_freq_target_each_features(df = data_percentile_feature_target, \n",
    "                                                                                     target = param_target_name_percentile,\n",
    "                                                                                     ct_normalized = param_ct_normalized_freq_target_feature)\n",
    "    fig_freq_target_each_feature = ca.plot_df_table_plotly(df_freq_target_each_feature_plotly)\n",
    "    fig_freq_target_each_feature.write_html(f\"output_eda/{id_report}/categorical_analysis/freq_target_each_feature.html\")\n",
    "    df_freq_target_each_feature.to_excel(f\"output_eda/{id_report}/categorical_analysis/statistics_target.xlsx\")\n",
    "    \n",
    "    \n",
    "    # barplot freq of target categorical for each category in each feature\n",
    "    print(f'barplot freq target each category each feature... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    barplot_freq_target_1_all_features = ca.barplot_crosstab_freq_target_1_features(df = data_percentile_feature_target,\n",
    "                                                                             target = param_target_name_percentile,\n",
    "                                                                             number_columns = 1)\n",
    "    barplot_freq_target_1_all_features.write_html(f\"output_eda/{id_report}/categorical_analysis/barplot_freq_target_all_features_invidually.html\")\n",
    "    \n",
    "    \n",
    "    # --- plots multiple analysis between \"feature x categorical\" and \"target y Continuous\" - in the heatmap of relation between feature_x, feature_y and target, the target must be aggregate as mean, std, etc\n",
    "    \n",
    "    # heatmap feature1 & feature2 vs target\n",
    "    print(f'heatmap continous target vs 2 categorical features... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    for param_agg_target in param_list_agg_target_multiple_features:\n",
    "        fig_crosstab_agg_target_2_features = ca.heatmap_crosstab_aggregation_target_2_features(df = data_percentile_feature, \n",
    "                                                                                               target = param_target, \n",
    "                                                                                               agg_target = param_agg_target, \n",
    "                                                                                               number_columns = 1)\n",
    "        fig_crosstab_agg_target_2_features.write_html(f\"output_eda/{id_report}/categorical_analysis/crosstab_{param_agg_target}_target_2_features.html\")\n",
    "    \n",
    "    \n",
    "    # heatmap feature1 & feature2 & feature3 vs target\n",
    "    print(f'heatmap continous target vs 3 categorical features... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    for param_agg_target in param_list_agg_target_multiple_features:\n",
    "        fig_crosstab_mean_target_3_features = ca.heatmap_crosstab_aggregation_target_3_features(df = data_percentile_feature, \n",
    "                                                                                                target = param_target,\n",
    "                                                                                                agg_target = param_agg_target, \n",
    "                                                                                                number_columns = 1)\n",
    "        fig_crosstab_mean_target_3_features.write_html(f\"output_eda/{id_report}/categorical_analysis/crosstab_{param_agg_target}_target_3_features.html\")\n",
    "    \n",
    "    \n",
    "    # --- plots multiple analysis between \"feature x categorical\" and \"target y Continuous\"\n",
    "    print(f'barplot categorical target vs 2 categorical features... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    barplot_freq_target_2_all_features = ca.barplot_crosstab_freq_target_2_features(df = data_percentile_feature_target,\n",
    "                                                                                    target = param_target_name_percentile,\n",
    "                                                                                    number_columns = 1)\n",
    "    barplot_freq_target_2_all_features.write_html(f\"output_eda/{id_report}/categorical_analysis/barplot_freq_target_2_all_features.html\")\n",
    "    \n",
    "    \n",
    "    # --- plots parallel features categorical vs target categorical\n",
    "    print(f'parellel categorical features and categorical target... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "    fig_parallel_discrete = ca.plot_parallel_discrete_variables(df_percentile = data_percentile_feature_target, \n",
    "                                                                list_features_target_to_plot = param_list_features_target_to_parallel, \n",
    "                                                                target = param_target_name_percentile)\n",
    "    fig_parallel_discrete.write_html(f\"output_eda/{id_report}/categorical_analysis/parallel_discrete.html\")\n",
    "    \n",
    "    \n",
    "    \n",
    "    # ################### end ###################\n",
    "    print(f'end... time:{datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6845e3d-2604-4e1f-b4c5-064e3fb279cb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95753ef0-e960-424f-8a7e-b2cbdeb50073",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a12e9ee-f71b-40fd-a853-325c02b15735",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da75488d-0c72-4dfe-bf98-0beedcb35399",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}