1. ML_baseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Untitled42.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iun8WNVuLVZJ",
        "outputId": "69369e69-1001-477b-87b4-0236688051d3"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 171,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZxVPnERLZJt"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import io\n",
        "import sklearn \n",
        "from datetime import datetime, timedelta\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 172,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QvbVQ5GVLZo9",
        "outputId": "b3362b0b-a0c0-45d0-a7c6-033bc419cf8e"
      },
      "source": [
        "df = pd.read_csv('/content/drive/MyDrive/DATA/4. 1+2+3_DFEDTAR_data.csv')\n",
        "#필요없는 열 제거 \n",
        "df = df.drop(\"Unnamed: 0\", axis=1)\n",
        "df = df.drop(\"DATE\", axis=1)\n",
        "#형태 확인\n",
        "df,df.shape"
      ],
      "execution_count": 173,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(      DFEDTAR  NASDAQCOM  BAMLH0A0HYM2  ...  USD/KRW  USD/EUR  USD/CNY\n",
              " 0        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              " 1        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              " 2        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              " 3        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              " 4        4.75    2251.27          5.57  ...  1164.50   0.8501   8.2795\n",
              " ...       ...        ...           ...  ...      ...      ...      ...\n",
              " 8186     0.25   13748.74          3.34  ...  1106.58   0.8177   6.3688\n",
              " 8187     0.25   13736.48          3.33  ...  1108.38   0.8187   6.3796\n",
              " 8188     0.25   13756.33          3.31  ...  1110.92   0.8188   6.3805\n",
              " 8189     0.25   13614.51          3.28  ...  1116.46   0.8245   6.4036\n",
              " 8190     0.25   13614.51          3.28  ...  1110.52   0.8218   6.3945\n",
              " \n",
              " [8191 rows x 15 columns], (8191, 15))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 173
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQMaRK7vM1K9"
      },
      "source": [
        "#df = df.set_index('DATE')"
      ],
      "execution_count": 174,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y60c2mLFpJRv",
        "outputId": "9257ae5d-cd64-411c-d014-10d3ffe68a66"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 175,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 8191 entries, 0 to 8190\n",
            "Data columns (total 15 columns):\n",
            " #   Column        Non-Null Count  Dtype  \n",
            "---  ------        --------------  -----  \n",
            " 0   DFEDTAR       8191 non-null   float64\n",
            " 1   NASDAQCOM     8191 non-null   float64\n",
            " 2   BAMLH0A0HYM2  8191 non-null   float64\n",
            " 3   HSN1F         8191 non-null   float64\n",
            " 4   UMCSENT       8191 non-null   float64\n",
            " 5   UNRATE        8191 non-null   float64\n",
            " 6   M2            8191 non-null   float64\n",
            " 7   ICSA          8191 non-null   float64\n",
            " 8   NG            8191 non-null   float64\n",
            " 9   ZG            8191 non-null   float64\n",
            " 10  ZI            8191 non-null   float64\n",
            " 11  HG            8191 non-null   float64\n",
            " 12  USD/KRW       8191 non-null   float64\n",
            " 13  USD/EUR       8191 non-null   float64\n",
            " 14  USD/CNY       8191 non-null   float64\n",
            "dtypes: float64(15)\n",
            "memory usage: 960.0 KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DK-NmDTpfR2H",
        "outputId": "faf152a5-d35d-450f-bfdc-2c462ebc9c9a"
      },
      "source": [
        "# Numerical Features\n",
        "\n",
        "num_columns = [c for c, t in zip(df.dtypes.index, df.dtypes) if t == 'float64']\n",
        "num_columns"
      ],
      "execution_count": 176,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['DFEDTAR',\n",
              " 'NASDAQCOM',\n",
              " 'BAMLH0A0HYM2',\n",
              " 'HSN1F',\n",
              " 'UMCSENT',\n",
              " 'UNRATE',\n",
              " 'M2',\n",
              " 'ICSA',\n",
              " 'NG',\n",
              " 'ZG',\n",
              " 'ZI',\n",
              " 'HG',\n",
              " 'USD/KRW',\n",
              " 'USD/EUR',\n",
              " 'USD/CNY']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 176
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OEUdX4FJrLRn"
      },
      "source": [
        "#**정규화 및 모델링**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JoDZgm6F1XO0"
      },
      "source": [
        "정규화"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "LvF75u13LcDl",
        "outputId": "d085aec5-d668-4fb0-84b4-7b66b869111d"
      },
      "source": [
        "# from sklearn.preprocessing import StandardScaler\n",
        "# scaler = StandardScaler()\n",
        "# df[num_columns] = scaler.fit_transform(df[num_columns])\n",
        "# df[num_columns]"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DFEDTAR</th>\n",
              "      <th>NASDAQCOM</th>\n",
              "      <th>BAMLH0A0HYM2</th>\n",
              "      <th>HSN1F</th>\n",
              "      <th>UMCSENT</th>\n",
              "      <th>UNRATE</th>\n",
              "      <th>M2</th>\n",
              "      <th>ICSA</th>\n",
              "      <th>NG</th>\n",
              "      <th>ZG</th>\n",
              "      <th>ZI</th>\n",
              "      <th>HG</th>\n",
              "      <th>USD/KRW</th>\n",
              "      <th>USD/EUR</th>\n",
              "      <th>USD/CNY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.476892</td>\n",
              "      <td>-0.646109</td>\n",
              "      <td>-0.025829</td>\n",
              "      <td>0.560273</td>\n",
              "      <td>1.395258</td>\n",
              "      <td>-0.826073</td>\n",
              "      <td>-1.326946</td>\n",
              "      <td>-0.174485</td>\n",
              "      <td>-1.030318</td>\n",
              "      <td>-1.328004</td>\n",
              "      <td>-1.190292</td>\n",
              "      <td>-1.585160</td>\n",
              "      <td>0.559719</td>\n",
              "      <td>-0.031200</td>\n",
              "      <td>1.297171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.476892</td>\n",
              "      <td>-0.646109</td>\n",
              "      <td>-0.025829</td>\n",
              "      <td>0.560273</td>\n",
              "      <td>1.395258</td>\n",
              "      <td>-0.826073</td>\n",
              "      <td>-1.326946</td>\n",
              "      <td>-0.174485</td>\n",
              "      <td>-1.030318</td>\n",
              "      <td>-1.328004</td>\n",
              "      <td>-1.190292</td>\n",
              "      <td>-1.585160</td>\n",
              "      <td>0.559719</td>\n",
              "      <td>-0.031200</td>\n",
              "      <td>1.297171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.476892</td>\n",
              "      <td>-0.646109</td>\n",
              "      <td>-0.025829</td>\n",
              "      <td>0.560273</td>\n",
              "      <td>1.395258</td>\n",
              "      <td>-0.826073</td>\n",
              "      <td>-1.326946</td>\n",
              "      <td>-0.174485</td>\n",
              "      <td>-1.030318</td>\n",
              "      <td>-1.328004</td>\n",
              "      <td>-1.190292</td>\n",
              "      <td>-1.585160</td>\n",
              "      <td>0.559719</td>\n",
              "      <td>-0.031200</td>\n",
              "      <td>1.297171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.476892</td>\n",
              "      <td>-0.646109</td>\n",
              "      <td>-0.025829</td>\n",
              "      <td>0.560273</td>\n",
              "      <td>1.395258</td>\n",
              "      <td>-0.826073</td>\n",
              "      <td>-1.326946</td>\n",
              "      <td>-0.174485</td>\n",
              "      <td>-1.030318</td>\n",
              "      <td>-1.328004</td>\n",
              "      <td>-1.190292</td>\n",
              "      <td>-1.585160</td>\n",
              "      <td>0.559719</td>\n",
              "      <td>-0.031200</td>\n",
              "      <td>1.297171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.476892</td>\n",
              "      <td>-0.629844</td>\n",
              "      <td>-0.048387</td>\n",
              "      <td>0.560273</td>\n",
              "      <td>1.395258</td>\n",
              "      <td>-0.826073</td>\n",
              "      <td>-1.326946</td>\n",
              "      <td>-0.174485</td>\n",
              "      <td>-1.073149</td>\n",
              "      <td>-1.328004</td>\n",
              "      <td>-1.190292</td>\n",
              "      <td>-1.589642</td>\n",
              "      <td>0.340210</td>\n",
              "      <td>0.006617</td>\n",
              "      <td>1.297416</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8186</th>\n",
              "      <td>-0.860585</td>\n",
              "      <td>3.697024</td>\n",
              "      <td>-0.886785</td>\n",
              "      <td>0.519330</td>\n",
              "      <td>0.148406</td>\n",
              "      <td>-0.056445</td>\n",
              "      <td>2.596282</td>\n",
              "      <td>-0.027357</td>\n",
              "      <td>-0.622084</td>\n",
              "      <td>1.773580</td>\n",
              "      <td>1.477220</td>\n",
              "      <td>2.030039</td>\n",
              "      <td>-0.244340</td>\n",
              "      <td>-0.259746</td>\n",
              "      <td>-1.044672</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8187</th>\n",
              "      <td>-0.860585</td>\n",
              "      <td>3.692410</td>\n",
              "      <td>-0.890544</td>\n",
              "      <td>0.519330</td>\n",
              "      <td>0.148406</td>\n",
              "      <td>-0.056445</td>\n",
              "      <td>2.596282</td>\n",
              "      <td>-0.027357</td>\n",
              "      <td>-0.569438</td>\n",
              "      <td>1.773005</td>\n",
              "      <td>1.487377</td>\n",
              "      <td>2.008528</td>\n",
              "      <td>-0.226174</td>\n",
              "      <td>-0.251525</td>\n",
              "      <td>-1.031433</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8188</th>\n",
              "      <td>-0.860585</td>\n",
              "      <td>3.699880</td>\n",
              "      <td>-0.898064</td>\n",
              "      <td>0.519330</td>\n",
              "      <td>0.148406</td>\n",
              "      <td>-0.056445</td>\n",
              "      <td>2.596282</td>\n",
              "      <td>-0.027357</td>\n",
              "      <td>-0.582377</td>\n",
              "      <td>1.782392</td>\n",
              "      <td>1.499149</td>\n",
              "      <td>1.952958</td>\n",
              "      <td>-0.200539</td>\n",
              "      <td>-0.250703</td>\n",
              "      <td>-1.030330</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8189</th>\n",
              "      <td>-0.860585</td>\n",
              "      <td>3.646509</td>\n",
              "      <td>-0.909343</td>\n",
              "      <td>0.519330</td>\n",
              "      <td>0.148406</td>\n",
              "      <td>-0.056445</td>\n",
              "      <td>2.596282</td>\n",
              "      <td>-0.027357</td>\n",
              "      <td>-0.597546</td>\n",
              "      <td>1.712272</td>\n",
              "      <td>1.415241</td>\n",
              "      <td>1.837784</td>\n",
              "      <td>-0.144627</td>\n",
              "      <td>-0.203843</td>\n",
              "      <td>-1.002015</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8190</th>\n",
              "      <td>-0.860585</td>\n",
              "      <td>3.646509</td>\n",
              "      <td>-0.909343</td>\n",
              "      <td>0.519330</td>\n",
              "      <td>0.148406</td>\n",
              "      <td>-0.056445</td>\n",
              "      <td>2.596282</td>\n",
              "      <td>-0.027357</td>\n",
              "      <td>-0.597546</td>\n",
              "      <td>1.712272</td>\n",
              "      <td>1.415241</td>\n",
              "      <td>1.837784</td>\n",
              "      <td>-0.204576</td>\n",
              "      <td>-0.226040</td>\n",
              "      <td>-1.013169</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8191 rows × 15 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       DFEDTAR  NASDAQCOM  BAMLH0A0HYM2  ...   USD/KRW   USD/EUR   USD/CNY\n",
              "0     1.476892  -0.646109     -0.025829  ...  0.559719 -0.031200  1.297171\n",
              "1     1.476892  -0.646109     -0.025829  ...  0.559719 -0.031200  1.297171\n",
              "2     1.476892  -0.646109     -0.025829  ...  0.559719 -0.031200  1.297171\n",
              "3     1.476892  -0.646109     -0.025829  ...  0.559719 -0.031200  1.297171\n",
              "4     1.476892  -0.629844     -0.048387  ...  0.340210  0.006617  1.297416\n",
              "...        ...        ...           ...  ...       ...       ...       ...\n",
              "8186 -0.860585   3.697024     -0.886785  ... -0.244340 -0.259746 -1.044672\n",
              "8187 -0.860585   3.692410     -0.890544  ... -0.226174 -0.251525 -1.031433\n",
              "8188 -0.860585   3.699880     -0.898064  ... -0.200539 -0.250703 -1.030330\n",
              "8189 -0.860585   3.646509     -0.909343  ... -0.144627 -0.203843 -1.002015\n",
              "8190 -0.860585   3.646509     -0.909343  ... -0.204576 -0.226040 -1.013169\n",
              "\n",
              "[8191 rows x 15 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sEcjm89rQRQo"
      },
      "source": [
        "!pip install category_encoders"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3DshmgNUbiM"
      },
      "source": [
        "from sklearn import linear_model\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn import svm\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from xgboost import XGBRegressor\n",
        "from lightgbm import LGBMRegressor\n",
        "from sklearn.neighbors import KNeighborsRegressor\n",
        "\n",
        "\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from category_encoders import OrdinalEncoder\n",
        "\n",
        "from sklearn.metrics import mean_squared_error, r2_score, mean_squared_log_error"
      ],
      "execution_count": 177,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNmW94UkZ3l2"
      },
      "source": [
        "# df = df.drop(\"DATE\", axis=1)"
      ],
      "execution_count": 178,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlHGYa3FNbxC"
      },
      "source": [
        "x = df.drop('DFEDTAR', axis=1)\n",
        "y = df['DFEDTAR']"
      ],
      "execution_count": 179,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sNryvSLlV9ng",
        "outputId": "14e5b5f1-bb40-4290-b6ab-b2e153f07c19"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.4, random_state=20, shuffle=True)\n",
        "x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=2020,shuffle=True)\n",
        "\n",
        "x_train.shape, x_test.shape, y_train.shape, x_val.shape"
      ],
      "execution_count": 180,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((4914, 14), (1639, 14), (4914,), (1638, 14))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 180
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I76juDt60hjU"
      },
      "source": [
        "선형회귀"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Ul2NcAibmO-",
        "outputId": "c651e592-15d1-4add-af09-9bae67012d55"
      },
      "source": [
        "lg = linear_model.LinearRegression()\n",
        "\n",
        "lg.fit(x_train, y_train)      \n",
        "print('coefficient = ', lg.coef_[0]) # 독립변수의 계수를 출력\n",
        "print('intercept = ', lg.intercept_)\n",
        "print('mean_squared_error', mean_squared_error(y_train, lg.predict(x_train))) # 절편을 출력                  "
      ],
      "execution_count": 181,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "coefficient =  0.0005517301416845362\n",
            "intercept =  -4.986236568654231\n",
            "mean_squared_error 0.4484365194263439\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eB_VOaG-LYoZ"
      },
      "source": [
        "SGD"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6gZDB8QDK-5o",
        "outputId": "15d6cc02-b03b-4422-e76e-3199e5bb1774"
      },
      "source": [
        "sgd = linear_model.SGDRegressor()\n",
        "\n",
        "sgd.fit(x_train, y_train)      \n",
        "print('coefficient = ', sgd.coef_[0]) # 독립변수의 계수를 출력\n",
        "print('intercept = ', sgd.intercept_)\n",
        "print('mean_squared_error', mean_squared_error(y_train, sgd.predict(x_train))) # 절편을 출력                  "
      ],
      "execution_count": 182,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "coefficient =  293286468640203.2\n",
            "intercept =  [-1.46699896e+11]\n",
            "mean_squared_error 4.048122866753416e+39\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fRXpBeK_LaJ3"
      },
      "source": [
        "BayesianRidge"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3_yfK82bLeb0",
        "outputId": "453776bb-8b3f-4373-95b5-0cce893c4623"
      },
      "source": [
        "reg = linear_model.BayesianRidge()\n",
        "\n",
        "reg.fit(x_train, y_train)\n",
        "print('coefficient = ', reg.coef_[0]) # 독립변수의 계수를 출력\n",
        "print('intercept = ', reg.intercept_)\n",
        "print('mean_squared_error', mean_squared_error(y_train, reg.predict(x_train))) # 절편을 출력 "
      ],
      "execution_count": 183,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "coefficient =  0.0005535250563370219\n",
            "intercept =  -4.919033396420122\n",
            "mean_squared_error 0.4484639702535901\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5EIVHmZo3QIl"
      },
      "source": [
        "비교"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jciU_7RnaQPg",
        "outputId": "178c8af4-e47c-46e2-a6b4-c12a86a7f7f5"
      },
      "source": [
        "#정확도 지표로 어느것을 사용할 수 있나 \n",
        "sklearn.metrics.SCORERS.keys()"
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['explained_variance', 'r2', 'max_error', 'neg_median_absolute_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_root_mean_squared_error', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted', 'balanced_accuracy', 'average_precision', 'neg_log_loss', 'neg_brier_score', 'adjusted_rand_score', 'homogeneity_score', 'completeness_score', 'v_measure_score', 'mutual_info_score', 'adjusted_mutual_info_score', 'normalized_mutual_info_score', 'fowlkes_mallows_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'jaccard', 'jaccard_macro', 'jaccard_micro', 'jaccard_samples', 'jaccard_weighted'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 144
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "01eC5WLMWbdu",
        "outputId": "ee01e3f7-8f20-4283-eeae-bf65b83f49ba"
      },
      "source": [
        "from sklearn.model_selection import KFold #for K-fold cross validation\n",
        "from sklearn.model_selection import cross_val_score #score evaluation\n",
        "from sklearn.model_selection import cross_val_predict #prediction\n",
        "\n",
        "kfold = KFold(n_splits=5, random_state=20, shuffle=True) # k=10, split the data into 10 equal parts\n",
        "\n",
        "xyz=[]\n",
        "accuracy=[]\n",
        "std=[]\n",
        "\n",
        "Regressors=['Linear Regression',\n",
        "             'Random Forest',\n",
        "             'XGB',\n",
        "             'LGBM',\n",
        "             'KNeighbors']\n",
        "\n",
        "models=[LinearRegression(),\n",
        "        RandomForestRegressor(),\n",
        "        XGBRegressor(),\n",
        "        LGBMRegressor(),\n",
        "        KNeighborsRegressor()]\n",
        "\n",
        "for i in models:\n",
        "    model = i\n",
        "    cv_result = cross_val_score(model , x , y, cv = kfold, scoring = \"neg_mean_squared_error\")\n",
        "    cv_result=cv_result\n",
        "\n",
        "    xyz.append(cv_result.mean())\n",
        "    std.append(cv_result.std())\n",
        "    accuracy.append(cv_result)\n",
        "\n",
        "new_models_dataframe2 = pd.DataFrame(\n",
        "    {'mean_squared_error_CV Mean':xyz, 'Std':std},\n",
        "     index=Regressors\n",
        "     )      \n",
        " \n",
        "new_models_dataframe2.round(4)"
      ],
      "execution_count": 184,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[16:12:50] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[16:12:50] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[16:12:51] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[16:12:51] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[16:12:52] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean_squared_error_CV Mean</th>\n",
              "      <th>Std</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Linear Regression</th>\n",
              "      <td>-0.4533</td>\n",
              "      <td>0.0158</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest</th>\n",
              "      <td>-0.0019</td>\n",
              "      <td>0.0004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>XGB</th>\n",
              "      <td>-0.0107</td>\n",
              "      <td>0.0008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LGBM</th>\n",
              "      <td>-0.0026</td>\n",
              "      <td>0.0005</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>KNeighbors</th>\n",
              "      <td>-0.0200</td>\n",
              "      <td>0.0035</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   mean_squared_error_CV Mean     Std\n",
              "Linear Regression                     -0.4533  0.0158\n",
              "Random Forest                         -0.0019  0.0004\n",
              "XGB                                   -0.0107  0.0008\n",
              "LGBM                                  -0.0026  0.0005\n",
              "KNeighbors                            -0.0200  0.0035"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 184
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tfjo7GtmNpVz"
      },
      "source": [
        "#**RandomForestRegressor ML**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X6uWRpy0RPiF"
      },
      "source": [
        "먼저 xgb부스팅모델"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ubI9mxvaNRtF",
        "outputId": "3a07b90c-0fa5-4d1d-deae-93c9f89d1b33"
      },
      "source": [
        "from xgboost.sklearn import XGBModel\n",
        "from xgboost import XGBRegressor\n",
        "pipe = make_pipeline(\n",
        "    OrdinalEncoder(), \n",
        "    XGBRegressor()\n",
        ")\n",
        "\n",
        "pipe.fit(x_train, y_train)\n",
        "print('훈련 R^2: ', pipe.score(x_train, y_train))\n",
        "print('검증 R^2: ', pipe.score(x_val, y_val))\n",
        "print('TEST R^2: ', pipe.score(x_test, y_test))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe.predict(x_train), y_train))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe.predict(x_val), y_val))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe.predict(x_test), y_test))"
      ],
      "execution_count": 185,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[16:13:05] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "훈련 R^2:  0.9975120517235978\n",
            "검증 R^2:  0.9970168414894269\n",
            "TEST R^2:  0.9969588130282647\n",
            "\n",
            "훈련 MAE:  0.06075673339009819\n",
            "검증 MAE:  0.06396850506901304\n",
            "TEST MAE:  0.06534298454355074\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L3oBbiyURS7S"
      },
      "source": [
        "랜덤 포레스트 .,,,\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ejwapYYXQCyT",
        "outputId": "6c55d759-1329-406b-8ac4-8c3bcc5bcee1"
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from sklearn.pipeline import Pipeline\n",
        "# encoder, imputer를 preprocessing으로 묶었습니다. 후에 eli5 permutation 계산에 사용합니다\n",
        "pipe = Pipeline([\n",
        "    ('preprocessing', make_pipeline(OrdinalEncoder())),\n",
        "    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) \n",
        "])\n",
        "\n",
        "pipe.fit(x_train, y_train)\n",
        "print('훈련 R^2: ', pipe.score(x_train, y_train))\n",
        "print('검증 R^2: ', pipe.score(x_val, y_val))\n",
        "print('TEST R^2: ', pipe.score(x_test, y_test))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe.predict(x_train), y_train))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe.predict(x_val), y_val))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe.predict(x_test), y_test))"
      ],
      "execution_count": 186,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 R^2:  0.9999400757166952\n",
            "검증 R^2:  0.9996794620287783\n",
            "TEST R^2:  0.9992072039293762\n",
            "\n",
            "훈련 MAE:  0.002900386650386649\n",
            "검증 MAE:  0.006759768009768006\n",
            "TEST MAE:  0.010488102501525317\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kDLz8CdyTIlT"
      },
      "source": [
        "#**Feature importance**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q8U2iuSRX_Ha",
        "outputId": "19cdec8b-29bf-4b28-f55c-b6cf49e24077"
      },
      "source": [
        "pipe.named_steps"
      ],
      "execution_count": 187,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'preprocessing': Pipeline(memory=None,\n",
              "          steps=[('ordinalencoder',\n",
              "                  OrdinalEncoder(cols=[], drop_invariant=False,\n",
              "                                 handle_missing='value', handle_unknown='value',\n",
              "                                 mapping=[], return_df=True, verbose=0))],\n",
              "          verbose=False),\n",
              " 'rf': RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',\n",
              "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
              "                       max_samples=None, min_impurity_decrease=0.0,\n",
              "                       min_impurity_split=None, min_samples_leaf=1,\n",
              "                       min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
              "                       n_estimators=100, n_jobs=-1, oob_score=False,\n",
              "                       random_state=20, verbose=0, warm_start=False)}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 187
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRKjKqOWYH9m"
      },
      "source": [
        "!pip install eli5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AffANE3VWaVp"
      },
      "source": [
        "import warnings\n",
        "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
        "\n",
        "import eli5\n",
        "from eli5.sklearn import PermutationImportance\n"
      ],
      "execution_count": 188,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ghUbwtMnYGtP"
      },
      "source": [
        "# permuter 정의\n",
        "permuter = PermutationImportance(\n",
        "    pipe.named_steps['rf'], # model\n",
        "    scoring='r2', # metric\n",
        "    n_iter=5, # 다른 random seed를 사용하여 5번 반복\n",
        "    random_state=20\n",
        ")\n",
        "\n",
        "# permuter 계산은 preprocessing 된 X_val을 사용합니다.\n",
        "x_val_transformed = pipe.named_steps['preprocessing'].transform(x_val)\n",
        "\n",
        "# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다\n",
        "permuter.fit(x_val_transformed, y_val);"
      ],
      "execution_count": 189,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TGACf1xDb3cS",
        "outputId": "64c7bfa9-e0aa-41b4-ed55-31132ff3ca91"
      },
      "source": [
        "feature_names = x_val.columns.tolist()\n",
        "pd.Series(permuter.feature_importances_, feature_names).sort_values()"
      ],
      "execution_count": 190,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "UMCSENT         0.000344\n",
              "NASDAQCOM       0.001007\n",
              "ZI              0.001136\n",
              "ICSA            0.001174\n",
              "BAMLH0A0HYM2    0.001484\n",
              "USD/KRW         0.002292\n",
              "USD/EUR         0.003090\n",
              "USD/CNY         0.003146\n",
              "HSN1F           0.003453\n",
              "ZG              0.003704\n",
              "HG              0.003848\n",
              "NG              0.004443\n",
              "UNRATE          0.691620\n",
              "M2              1.090124\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 190
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "id": "alg0_NUxeUmo",
        "outputId": "3e54f875-e351-4e9f-c46e-1fa8051f94d3"
      },
      "source": [
        "# 특성별 score 확인\n",
        "eli5.show_weights(\n",
        "    permuter, \n",
        "    top=None, # top n 지정 가능, None 일 경우 모든 특성 \n",
        "    feature_names=feature_names # list 형식으로 넣어야 합니다\n",
        ")"
      ],
      "execution_count": 191,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <style>\n",
              "    table.eli5-weights tr:hover {\n",
              "        filter: brightness(85%);\n",
              "    }\n",
              "</style>\n",
              "\n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
              "    <thead>\n",
              "    <tr style=\"border: none;\">\n",
              "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
              "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
              "    </tr>\n",
              "    </thead>\n",
              "    <tbody>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                1.0901\n",
              "                \n",
              "                    &plusmn; 0.0982\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                M2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 85.46%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.6916\n",
              "                \n",
              "                    &plusmn; 0.0167\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UNRATE\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.58%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0044\n",
              "                \n",
              "                    &plusmn; 0.0012\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.62%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0038\n",
              "                \n",
              "                    &plusmn; 0.0004\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.63%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0037\n",
              "                \n",
              "                    &plusmn; 0.0002\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.64%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0035\n",
              "                \n",
              "                    &plusmn; 0.0005\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HSN1F\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.67%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0031\n",
              "                \n",
              "                    &plusmn; 0.0003\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/CNY\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.67%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0031\n",
              "                \n",
              "                    &plusmn; 0.0002\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/EUR\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.73%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0023\n",
              "                \n",
              "                    &plusmn; 0.0003\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/KRW\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.80%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0015\n",
              "                \n",
              "                    &plusmn; 0.0005\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                BAMLH0A0HYM2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.83%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0012\n",
              "                \n",
              "                    &plusmn; 0.0001\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ICSA\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.84%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0011\n",
              "                \n",
              "                    &plusmn; 0.0000\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZI\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.85%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0010\n",
              "                \n",
              "                    &plusmn; 0.0002\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NASDAQCOM\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0003\n",
              "                \n",
              "                    &plusmn; 0.0000\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UMCSENT\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "    \n",
              "    </tbody>\n",
              "</table>\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 191
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zycxYnCfeZJ-"
      },
      "source": [
        "통화량을 알면 금리를 맞출수 있는 확률이 너무나도 커진다는 것을,,,, 알게 되었습니다. \n",
        "(사실 원래 아는거 였지만 이렇게 크게 작용할 줄은 몰랐습니다.)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xbo30BfdeqM7"
      },
      "source": [
        "#**다시측정**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EnuG7KtAevJT"
      },
      "source": [
        " M2 통화량을 빼고 측정해 보도록 하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "dFAGQq10euo3",
        "outputId": "ec6a95a9-8b66-4f95-803b-967378bcee08"
      },
      "source": [
        "df_x_m2 = df.drop(\"M2\", axis=1)\n",
        "df_x_m2"
      ],
      "execution_count": 205,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DFEDTAR</th>\n",
              "      <th>NASDAQCOM</th>\n",
              "      <th>BAMLH0A0HYM2</th>\n",
              "      <th>HSN1F</th>\n",
              "      <th>UMCSENT</th>\n",
              "      <th>UNRATE</th>\n",
              "      <th>ICSA</th>\n",
              "      <th>NG</th>\n",
              "      <th>ZG</th>\n",
              "      <th>ZI</th>\n",
              "      <th>HG</th>\n",
              "      <th>USD/KRW</th>\n",
              "      <th>USD/EUR</th>\n",
              "      <th>USD/CNY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>4.3</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>4.3</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>4.3</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>4.3</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2251.27</td>\n",
              "      <td>5.57</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>4.3</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>1.975</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6390</td>\n",
              "      <td>1164.50</td>\n",
              "      <td>0.8501</td>\n",
              "      <td>8.2795</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8186</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13748.74</td>\n",
              "      <td>3.34</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>5.8</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>2.986</td>\n",
              "      <td>1905.3</td>\n",
              "      <td>28.014</td>\n",
              "      <td>4.6775</td>\n",
              "      <td>1106.58</td>\n",
              "      <td>0.8177</td>\n",
              "      <td>6.3688</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8187</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13736.48</td>\n",
              "      <td>3.33</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>5.8</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.104</td>\n",
              "      <td>1905.0</td>\n",
              "      <td>28.102</td>\n",
              "      <td>4.6535</td>\n",
              "      <td>1108.38</td>\n",
              "      <td>0.8187</td>\n",
              "      <td>6.3796</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8188</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13756.33</td>\n",
              "      <td>3.31</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>5.8</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.075</td>\n",
              "      <td>1909.9</td>\n",
              "      <td>28.204</td>\n",
              "      <td>4.5915</td>\n",
              "      <td>1110.92</td>\n",
              "      <td>0.8188</td>\n",
              "      <td>6.3805</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8189</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13614.51</td>\n",
              "      <td>3.28</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>5.8</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.041</td>\n",
              "      <td>1873.3</td>\n",
              "      <td>27.477</td>\n",
              "      <td>4.4630</td>\n",
              "      <td>1116.46</td>\n",
              "      <td>0.8245</td>\n",
              "      <td>6.4036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8190</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13614.51</td>\n",
              "      <td>3.28</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>5.8</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.041</td>\n",
              "      <td>1873.3</td>\n",
              "      <td>27.477</td>\n",
              "      <td>4.4630</td>\n",
              "      <td>1110.52</td>\n",
              "      <td>0.8218</td>\n",
              "      <td>6.3945</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8191 rows × 14 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      DFEDTAR  NASDAQCOM  BAMLH0A0HYM2  ...  USD/KRW  USD/EUR  USD/CNY\n",
              "0        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "1        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "2        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "3        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "4        4.75    2251.27          5.57  ...  1164.50   0.8501   8.2795\n",
              "...       ...        ...           ...  ...      ...      ...      ...\n",
              "8186     0.25   13748.74          3.34  ...  1106.58   0.8177   6.3688\n",
              "8187     0.25   13736.48          3.33  ...  1108.38   0.8187   6.3796\n",
              "8188     0.25   13756.33          3.31  ...  1110.92   0.8188   6.3805\n",
              "8189     0.25   13614.51          3.28  ...  1116.46   0.8245   6.4036\n",
              "8190     0.25   13614.51          3.28  ...  1110.52   0.8218   6.3945\n",
              "\n",
              "[8191 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 205
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QALy7Z2rhjHo",
        "outputId": "7df060a5-9279-4e0e-a037-6db7cc85f338"
      },
      "source": [
        "x_ = df_x_m2.drop('DFEDTAR', axis=1)\n",
        "y_ = df_x_m2['DFEDTAR']\n",
        "x_, y_"
      ],
      "execution_count": 193,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(      NASDAQCOM  BAMLH0A0HYM2  HSN1F  ...  USD/KRW  USD/EUR  USD/CNY\n",
              " 0       2208.05          5.63  875.0  ...  1186.25   0.8455   8.2793\n",
              " 1       2208.05          5.63  875.0  ...  1186.25   0.8455   8.2793\n",
              " 2       2208.05          5.63  875.0  ...  1186.25   0.8455   8.2793\n",
              " 3       2208.05          5.63  875.0  ...  1186.25   0.8455   8.2793\n",
              " 4       2251.27          5.57  875.0  ...  1164.50   0.8501   8.2795\n",
              " ...         ...           ...    ...  ...      ...      ...      ...\n",
              " 8186   13748.74          3.34  863.0  ...  1106.58   0.8177   6.3688\n",
              " 8187   13736.48          3.33  863.0  ...  1108.38   0.8187   6.3796\n",
              " 8188   13756.33          3.31  863.0  ...  1110.92   0.8188   6.3805\n",
              " 8189   13614.51          3.28  863.0  ...  1116.46   0.8245   6.4036\n",
              " 8190   13614.51          3.28  863.0  ...  1110.52   0.8218   6.3945\n",
              " \n",
              " [8191 rows x 13 columns], 0       4.75\n",
              " 1       4.75\n",
              " 2       4.75\n",
              " 3       4.75\n",
              " 4       4.75\n",
              "         ... \n",
              " 8186    0.25\n",
              " 8187    0.25\n",
              " 8188    0.25\n",
              " 8189    0.25\n",
              " 8190    0.25\n",
              " Name: DFEDTAR, Length: 8191, dtype: float64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 193
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3JlLQgCJfEyj",
        "outputId": "91ed05e4-c98e-4851-83f3-6a88c1892055"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)\n",
        "x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)\n",
        "\n",
        "x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape"
      ],
      "execution_count": 194,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((4914, 13), (1639, 13), (1638, 13), (4914,), (1639,), (1638,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 194
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j6mivNJ2iUI5",
        "outputId": "f6d5e310-b000-4b8d-f5c4-cc08d3753b5a"
      },
      "source": [
        "x_train_.head(),x_train_.shape"
      ],
      "execution_count": 195,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(      NASDAQCOM  BAMLH0A0HYM2  HSN1F  UMCSENT  ...     HG  USD/KRW  USD/EUR  USD/CNY\n",
              " 5766    4215.32          5.08  476.0     86.9  ...  3.005  1060.97   0.7790   6.1259\n",
              " 443     4798.13          5.41  900.0    107.1  ...  0.805  1117.25   1.0284   8.2785\n",
              " 5109    2985.91          5.28  399.0     72.9  ...  3.590  1072.51   0.7553   6.2360\n",
              " 5194    3245.00          4.78  444.0     78.6  ...  3.454  1119.35   0.7700   6.2122\n",
              " 4339    2514.40          5.80  287.0     71.6  ...  3.830  1130.90   0.7329   6.6334\n",
              " \n",
              " [5 rows x 13 columns], (4914, 13))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 195
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1sthGSITfNXw",
        "outputId": "08f38195-dd08-4b2d-fde9-b5f2c4d366ec"
      },
      "source": [
        "pipe_ = Pipeline([\n",
        "    ('preprocessing', make_pipeline(OrdinalEncoder())),\n",
        "    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) \n",
        "])\n",
        "\n",
        "pipe_.fit(x_train_, y_train_)\n",
        "print('훈련 R^2: ', pipe_.score(x_train_, y_train_))\n",
        "print('검증 R^2: ', pipe_.score(x_val_, y_val_))\n",
        "print('TEST R^2: ', pipe_.score(x_test_, y_test_))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))"
      ],
      "execution_count": 196,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 R^2:  0.9999034834500641\n",
            "검증 R^2:  0.9991185714437719\n",
            "TEST R^2:  0.9987776192958645\n",
            "\n",
            "훈련 MAE:  0.004187525437525434\n",
            "검증 MAE:  0.01199481074481074\n",
            "TEST MAE:  0.013145210494203781\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qr_NloaqjyOF"
      },
      "source": [
        "거의 변화없음"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0OGGWlYYmO_5"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lpTRUiffgsAC"
      },
      "source": [
        "# permuter 정의\n",
        "permuter = PermutationImportance(\n",
        "    pipe_.named_steps['rf'], # model\n",
        "    scoring='r2', # metric\n",
        "    n_iter=5, # 다른 random seed를 사용하여 5번 반복\n",
        "    random_state=20\n",
        ")\n",
        "\n",
        "# permuter 계산은 preprocessing 된 X_val을 사용합니다.\n",
        "x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)\n",
        "\n",
        "# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다\n",
        "permuter.fit(x_val_transformed, y_val_);"
      ],
      "execution_count": 198,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "FFvsNiDEkve4",
        "outputId": "a9c310e5-397a-4540-95f9-cf50c34785b0"
      },
      "source": [
        "feature_names = x_val_.columns.tolist()\n",
        "pd.Series(permuter.feature_importances_, feature_names).sort_values()\n",
        "\n",
        "# 특성별 score 확인\n",
        "eli5.show_weights(\n",
        "    permuter, \n",
        "    top=None, # top n 지정 가능, None 일 경우 모든 특성 \n",
        "    feature_names=feature_names # list 형식으로 넣어야 합니다\n",
        ")"
      ],
      "execution_count": 200,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <style>\n",
              "    table.eli5-weights tr:hover {\n",
              "        filter: brightness(85%);\n",
              "    }\n",
              "</style>\n",
              "\n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
              "    <thead>\n",
              "    <tr style=\"border: none;\">\n",
              "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
              "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
              "    </tr>\n",
              "    </thead>\n",
              "    <tbody>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.7491\n",
              "                \n",
              "                    &plusmn; 0.0604\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UNRATE\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 81.07%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.6926\n",
              "                \n",
              "                    &plusmn; 0.0314\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/CNY\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 95.24%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0965\n",
              "                \n",
              "                    &plusmn; 0.0091\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.07%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0094\n",
              "                \n",
              "                    &plusmn; 0.0006\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZI\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.13%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0085\n",
              "                \n",
              "                    &plusmn; 0.0013\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/EUR\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.20%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0075\n",
              "                \n",
              "                    &plusmn; 0.0006\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.38%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0052\n",
              "                \n",
              "                    &plusmn; 0.0006\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.48%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0041\n",
              "                \n",
              "                    &plusmn; 0.0004\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HSN1F\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.58%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0030\n",
              "                \n",
              "                    &plusmn; 0.0007\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                BAMLH0A0HYM2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.66%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0023\n",
              "                \n",
              "                    &plusmn; 0.0002\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UMCSENT\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.69%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0019\n",
              "                \n",
              "                    &plusmn; 0.0002\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/KRW\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.77%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0013\n",
              "                \n",
              "                    &plusmn; 0.0002\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ICSA\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.78%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0012\n",
              "                \n",
              "                    &plusmn; 0.0004\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NASDAQCOM\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "    \n",
              "    </tbody>\n",
              "</table>\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 200
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s_ZkKRZalBXa"
      },
      "source": [
        "실업률이랑 달러/위완화 만으로도 금리 예측이 어느정도 가능하다 ?\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "jsnIx_41k_xh",
        "outputId": "78f39e56-bf82-40ec-d409-8d86e165cceb"
      },
      "source": [
        "df_x_m2 = df_x_m2.drop(\"UNRATE\", axis=1)\n",
        "df_x_m2"
      ],
      "execution_count": 206,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DFEDTAR</th>\n",
              "      <th>NASDAQCOM</th>\n",
              "      <th>BAMLH0A0HYM2</th>\n",
              "      <th>HSN1F</th>\n",
              "      <th>UMCSENT</th>\n",
              "      <th>ICSA</th>\n",
              "      <th>NG</th>\n",
              "      <th>ZG</th>\n",
              "      <th>ZI</th>\n",
              "      <th>HG</th>\n",
              "      <th>USD/KRW</th>\n",
              "      <th>USD/EUR</th>\n",
              "      <th>USD/CNY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "      <td>8.2793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2251.27</td>\n",
              "      <td>5.57</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>1.975</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6390</td>\n",
              "      <td>1164.50</td>\n",
              "      <td>0.8501</td>\n",
              "      <td>8.2795</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8186</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13748.74</td>\n",
              "      <td>3.34</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>2.986</td>\n",
              "      <td>1905.3</td>\n",
              "      <td>28.014</td>\n",
              "      <td>4.6775</td>\n",
              "      <td>1106.58</td>\n",
              "      <td>0.8177</td>\n",
              "      <td>6.3688</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8187</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13736.48</td>\n",
              "      <td>3.33</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.104</td>\n",
              "      <td>1905.0</td>\n",
              "      <td>28.102</td>\n",
              "      <td>4.6535</td>\n",
              "      <td>1108.38</td>\n",
              "      <td>0.8187</td>\n",
              "      <td>6.3796</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8188</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13756.33</td>\n",
              "      <td>3.31</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.075</td>\n",
              "      <td>1909.9</td>\n",
              "      <td>28.204</td>\n",
              "      <td>4.5915</td>\n",
              "      <td>1110.92</td>\n",
              "      <td>0.8188</td>\n",
              "      <td>6.3805</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8189</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13614.51</td>\n",
              "      <td>3.28</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.041</td>\n",
              "      <td>1873.3</td>\n",
              "      <td>27.477</td>\n",
              "      <td>4.4630</td>\n",
              "      <td>1116.46</td>\n",
              "      <td>0.8245</td>\n",
              "      <td>6.4036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8190</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13614.51</td>\n",
              "      <td>3.28</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.041</td>\n",
              "      <td>1873.3</td>\n",
              "      <td>27.477</td>\n",
              "      <td>4.4630</td>\n",
              "      <td>1110.52</td>\n",
              "      <td>0.8218</td>\n",
              "      <td>6.3945</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8191 rows × 13 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      DFEDTAR  NASDAQCOM  BAMLH0A0HYM2  ...  USD/KRW  USD/EUR  USD/CNY\n",
              "0        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "1        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "2        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "3        4.75    2208.05          5.63  ...  1186.25   0.8455   8.2793\n",
              "4        4.75    2251.27          5.57  ...  1164.50   0.8501   8.2795\n",
              "...       ...        ...           ...  ...      ...      ...      ...\n",
              "8186     0.25   13748.74          3.34  ...  1106.58   0.8177   6.3688\n",
              "8187     0.25   13736.48          3.33  ...  1108.38   0.8187   6.3796\n",
              "8188     0.25   13756.33          3.31  ...  1110.92   0.8188   6.3805\n",
              "8189     0.25   13614.51          3.28  ...  1116.46   0.8245   6.4036\n",
              "8190     0.25   13614.51          3.28  ...  1110.52   0.8218   6.3945\n",
              "\n",
              "[8191 rows x 13 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 206
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QXs3o5YIlfZo",
        "outputId": "87ca69b6-3206-4f12-cd4e-f36758b023dd"
      },
      "source": [
        "x_ = df_x_m2.drop('DFEDTAR', axis=1)\n",
        "y_ = df_x_m2['DFEDTAR']\n",
        "x_, y_\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)\n",
        "x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)\n",
        "\n",
        "x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape\n",
        "\n",
        "pipe_ = Pipeline([\n",
        "    ('preprocessing', make_pipeline(OrdinalEncoder())),\n",
        "    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) \n",
        "])\n",
        "\n",
        "pipe_.fit(x_train_, y_train_)\n",
        "print('훈련 R^2: ', pipe_.score(x_train_, y_train_))\n",
        "print('검증 R^2: ', pipe_.score(x_val_, y_val_))\n",
        "print('TEST R^2: ', pipe_.score(x_test_, y_test_))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))"
      ],
      "execution_count": 207,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 R^2:  0.9998319627222674\n",
            "검증 R^2:  0.9984024542466661\n",
            "TEST R^2:  0.9969649733533579\n",
            "\n",
            "훈련 MAE:  0.005388685388685385\n",
            "검증 MAE:  0.015695970695970698\n",
            "TEST MAE:  0.020707748627211715\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "wrvWJ0s9lxb9",
        "outputId": "b014ef32-5925-430d-d964-7c176ba8d8de"
      },
      "source": [
        "# permuter 정의\n",
        "permuter = PermutationImportance(\n",
        "    pipe_.named_steps['rf'], # model\n",
        "    scoring='r2', # metric\n",
        "    n_iter=5, # 다른 random seed를 사용하여 5번 반복\n",
        "    random_state=20\n",
        ")\n",
        "\n",
        "# permuter 계산은 preprocessing 된 X_val을 사용합니다.\n",
        "x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)\n",
        "\n",
        "# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다\n",
        "permuter.fit(x_val_transformed, y_val_);\n",
        "\n",
        "feature_names = x_val_.columns.tolist()\n",
        "pd.Series(permuter.feature_importances_, feature_names).sort_values()\n",
        "\n",
        "# 특성별 score 확인\n",
        "eli5.show_weights(\n",
        "    permuter, \n",
        "    top=None, # top n 지정 가능, None 일 경우 모든 특성 \n",
        "    feature_names=feature_names # list 형식으로 넣어야 합니다\n",
        ")"
      ],
      "execution_count": 208,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <style>\n",
              "    table.eli5-weights tr:hover {\n",
              "        filter: brightness(85%);\n",
              "    }\n",
              "</style>\n",
              "\n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
              "    <thead>\n",
              "    <tr style=\"border: none;\">\n",
              "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
              "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
              "    </tr>\n",
              "    </thead>\n",
              "    <tbody>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                1.5751\n",
              "                \n",
              "                    &plusmn; 0.0649\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/CNY\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 93.20%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.3375\n",
              "                \n",
              "                    &plusmn; 0.0215\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NASDAQCOM\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 97.12%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0987\n",
              "                \n",
              "                    &plusmn; 0.0070\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 97.93%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0615\n",
              "                \n",
              "                    &plusmn; 0.0042\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ICSA\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 98.20%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0504\n",
              "                \n",
              "                    &plusmn; 0.0085\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 98.36%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0443\n",
              "                \n",
              "                    &plusmn; 0.0040\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HSN1F\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 98.71%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0312\n",
              "                \n",
              "                    &plusmn; 0.0058\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/KRW\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 98.73%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0308\n",
              "                \n",
              "                    &plusmn; 0.0023\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.11%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0184\n",
              "                \n",
              "                    &plusmn; 0.0033\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/EUR\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.28%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0136\n",
              "                \n",
              "                    &plusmn; 0.0006\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZI\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.47%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0087\n",
              "                \n",
              "                    &plusmn; 0.0005\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UMCSENT\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.68%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0044\n",
              "                \n",
              "                    &plusmn; 0.0004\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                BAMLH0A0HYM2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "    \n",
              "    </tbody>\n",
              "</table>\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 208
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ogf1qt9qm3qr"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "vO0s5Dibm3QD",
        "outputId": "1741e0bf-6135-4820-d3cf-ffabbef48224"
      },
      "source": [
        "df_x_m2 = df_x_m2.drop(\"USD/CNY\", axis=1)\n",
        "df_x_m2"
      ],
      "execution_count": 209,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DFEDTAR</th>\n",
              "      <th>NASDAQCOM</th>\n",
              "      <th>BAMLH0A0HYM2</th>\n",
              "      <th>HSN1F</th>\n",
              "      <th>UMCSENT</th>\n",
              "      <th>ICSA</th>\n",
              "      <th>NG</th>\n",
              "      <th>ZG</th>\n",
              "      <th>ZI</th>\n",
              "      <th>HG</th>\n",
              "      <th>USD/KRW</th>\n",
              "      <th>USD/EUR</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2208.05</td>\n",
              "      <td>5.63</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>2.071</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6440</td>\n",
              "      <td>1186.25</td>\n",
              "      <td>0.8455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4.75</td>\n",
              "      <td>2251.27</td>\n",
              "      <td>5.57</td>\n",
              "      <td>875.0</td>\n",
              "      <td>103.9</td>\n",
              "      <td>331000.0</td>\n",
              "      <td>1.975</td>\n",
              "      <td>286.4</td>\n",
              "      <td>4.902</td>\n",
              "      <td>0.6390</td>\n",
              "      <td>1164.50</td>\n",
              "      <td>0.8501</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8186</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13748.74</td>\n",
              "      <td>3.34</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>2.986</td>\n",
              "      <td>1905.3</td>\n",
              "      <td>28.014</td>\n",
              "      <td>4.6775</td>\n",
              "      <td>1106.58</td>\n",
              "      <td>0.8177</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8187</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13736.48</td>\n",
              "      <td>3.33</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.104</td>\n",
              "      <td>1905.0</td>\n",
              "      <td>28.102</td>\n",
              "      <td>4.6535</td>\n",
              "      <td>1108.38</td>\n",
              "      <td>0.8187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8188</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13756.33</td>\n",
              "      <td>3.31</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.075</td>\n",
              "      <td>1909.9</td>\n",
              "      <td>28.204</td>\n",
              "      <td>4.5915</td>\n",
              "      <td>1110.92</td>\n",
              "      <td>0.8188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8189</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13614.51</td>\n",
              "      <td>3.28</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.041</td>\n",
              "      <td>1873.3</td>\n",
              "      <td>27.477</td>\n",
              "      <td>4.4630</td>\n",
              "      <td>1116.46</td>\n",
              "      <td>0.8245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8190</th>\n",
              "      <td>0.25</td>\n",
              "      <td>13614.51</td>\n",
              "      <td>3.28</td>\n",
              "      <td>863.0</td>\n",
              "      <td>88.3</td>\n",
              "      <td>385000.0</td>\n",
              "      <td>3.041</td>\n",
              "      <td>1873.3</td>\n",
              "      <td>27.477</td>\n",
              "      <td>4.4630</td>\n",
              "      <td>1110.52</td>\n",
              "      <td>0.8218</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8191 rows × 12 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      DFEDTAR  NASDAQCOM  BAMLH0A0HYM2  HSN1F  ...      ZI      HG  USD/KRW  USD/EUR\n",
              "0        4.75    2208.05          5.63  875.0  ...   4.902  0.6440  1186.25   0.8455\n",
              "1        4.75    2208.05          5.63  875.0  ...   4.902  0.6440  1186.25   0.8455\n",
              "2        4.75    2208.05          5.63  875.0  ...   4.902  0.6440  1186.25   0.8455\n",
              "3        4.75    2208.05          5.63  875.0  ...   4.902  0.6440  1186.25   0.8455\n",
              "4        4.75    2251.27          5.57  875.0  ...   4.902  0.6390  1164.50   0.8501\n",
              "...       ...        ...           ...    ...  ...     ...     ...      ...      ...\n",
              "8186     0.25   13748.74          3.34  863.0  ...  28.014  4.6775  1106.58   0.8177\n",
              "8187     0.25   13736.48          3.33  863.0  ...  28.102  4.6535  1108.38   0.8187\n",
              "8188     0.25   13756.33          3.31  863.0  ...  28.204  4.5915  1110.92   0.8188\n",
              "8189     0.25   13614.51          3.28  863.0  ...  27.477  4.4630  1116.46   0.8245\n",
              "8190     0.25   13614.51          3.28  863.0  ...  27.477  4.4630  1110.52   0.8218\n",
              "\n",
              "[8191 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 209
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxqohGOhmzno",
        "outputId": "99f81db1-ba64-4029-a072-a388c60a0559"
      },
      "source": [
        "#타겟설정\n",
        "x_ = df_x_m2.drop('DFEDTAR', axis=1)\n",
        "y_ = df_x_m2['DFEDTAR']\n",
        "x_, y_\n",
        "#데이터 분할 \n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)\n",
        "x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)\n",
        "\n",
        "x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape\n",
        "#파이프라인구축\n",
        "pipe_ = Pipeline([\n",
        "    ('preprocessing', make_pipeline(OrdinalEncoder())),\n",
        "    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) \n",
        "])\n",
        "#훈련\n",
        "pipe_.fit(x_train_, y_train_)\n",
        "#검증\n",
        "print('훈련 R^2: ', pipe_.score(x_train_, y_train_))\n",
        "print('검증 R^2: ', pipe_.score(x_val_, y_val_))\n",
        "print('TEST R^2: ', pipe_.score(x_test_, y_test_))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))"
      ],
      "execution_count": 210,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 R^2:  0.9998492260093473\n",
            "검증 R^2:  0.997869207415114\n",
            "TEST R^2:  0.9976963867029616\n",
            "\n",
            "훈련 MAE:  0.005511803011803011\n",
            "검증 MAE:  0.016782661782661783\n",
            "TEST MAE:  0.019238865161683952\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "id": "3-Zf0uagnL7N",
        "outputId": "54f9f132-08ef-49f5-8afd-4afb6726e5b5"
      },
      "source": [
        "# permuter 정의\n",
        "permuter = PermutationImportance(\n",
        "    pipe_.named_steps['rf'], # model\n",
        "    scoring='r2', # metric\n",
        "    n_iter=5, # 다른 random seed를 사용하여 5번 반복\n",
        "    random_state=20\n",
        ")\n",
        "\n",
        "# permuter 계산은 preprocessing 된 X_val을 사용합니다.\n",
        "x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)\n",
        "\n",
        "# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다\n",
        "permuter.fit(x_val_transformed, y_val_);\n",
        "\n",
        "feature_names = x_val_.columns.tolist()\n",
        "pd.Series(permuter.feature_importances_, feature_names).sort_values()\n",
        "\n",
        "# 특성별 score 확인\n",
        "eli5.show_weights(\n",
        "    permuter, \n",
        "    top=None, # top n 지정 가능, None 일 경우 모든 특성 \n",
        "    feature_names=feature_names # list 형식으로 넣어야 합니다\n",
        ")"
      ],
      "execution_count": 211,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <style>\n",
              "    table.eli5-weights tr:hover {\n",
              "        filter: brightness(85%);\n",
              "    }\n",
              "</style>\n",
              "\n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
              "    <thead>\n",
              "    <tr style=\"border: none;\">\n",
              "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
              "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
              "    </tr>\n",
              "    </thead>\n",
              "    <tbody>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                1.7566\n",
              "                \n",
              "                    &plusmn; 0.1594\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 91.92%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.4815\n",
              "                \n",
              "                    &plusmn; 0.0176\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NASDAQCOM\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 96.19%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.1645\n",
              "                \n",
              "                    &plusmn; 0.0213\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 97.60%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0852\n",
              "                \n",
              "                    &plusmn; 0.0070\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ICSA\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 97.94%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0684\n",
              "                \n",
              "                    &plusmn; 0.0116\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/KRW\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 98.34%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0504\n",
              "                \n",
              "                    &plusmn; 0.0056\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HSN1F\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.35%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0133\n",
              "                \n",
              "                    &plusmn; 0.0013\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UMCSENT\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.39%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0120\n",
              "                \n",
              "                    &plusmn; 0.0016\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/EUR\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.54%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0081\n",
              "                \n",
              "                    &plusmn; 0.0009\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                BAMLH0A0HYM2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.61%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0064\n",
              "                \n",
              "                    &plusmn; 0.0013\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.67%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0050\n",
              "                \n",
              "                    &plusmn; 0.0008\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ZI\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "    \n",
              "    </tbody>\n",
              "</table>\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 211
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H-dj3d4XomtJ",
        "outputId": "9a70c238-f456-486c-8fd4-3b16aa96e0ab"
      },
      "source": [
        "pd.Series(permuter.feature_importances_, x_test_.columns).sort_values()"
      ],
      "execution_count": 215,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ZI              0.004952\n",
              "HG              0.006401\n",
              "BAMLH0A0HYM2    0.008089\n",
              "USD/EUR         0.012024\n",
              "UMCSENT         0.013266\n",
              "HSN1F           0.050365\n",
              "USD/KRW         0.068404\n",
              "ICSA            0.085187\n",
              "NG              0.164527\n",
              "NASDAQCOM       0.481545\n",
              "ZG              1.756619\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 215
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 692
        },
        "id": "gxl5ZEv7oaCf",
        "outputId": "19532efe-a010-4c49-8219-8f9b78591b7c"
      },
      "source": [
        "import seaborn as sns\n",
        "zipp  = []\n",
        "for zipper in zip(x_train_.columns, permuter.feature_importances_):\n",
        "  zipp.append(zipper)\n",
        "zipp = pd.DataFrame(zipp,columns=['feature','importance']).sort_values('importance',ascending=False)\n",
        "plt.figure(figsize=(5,5))\n",
        "sns.barplot(y = zipp.feature[:10], x= zipp.importance[:10], palette='Blues_r')\n",
        "plt.title(\"Permutation Importance\",fontsize=20);"
      ],
      "execution_count": 219,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxsAAAKjCAYAAAB1MBJlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd7gkVbWw8XfBACI5KEkERAmKgo6gIBkVI8Gs1yuDCCpmvebAgOmCoqig90OUwYQoCmYUkCCCoINkxMQgGSTnMKzvj13t9NR0PNM1J8z7e55++nTV3rV39+lQq3aKzESSJEmSRm2J8a6AJEmSpKnJYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOStFiJiGy7zRrv+kjSVGawIWkBEbF+7YSs2+2uiLg6Is6IiIMjYuvxrrukhdfhO2DOeNdJ0uRksCFpYSwPPA7YHvgAcHZEnBcRTxvfamlhTPQr/xFxuifB0nA6BJAzx7tOWjwYbEgatS2B8yLipeNdEUmSNL6mjXcFJE0K1wLbdti+IrAR8Arglcy7gLEMcGxEbJWZly2aKkqSpInGYEPSIB7OzDld9l0EHB8RXwd+AixXbV8O+DSwZ/PVkwaXmTHedZCkxYXdqCSNRGaeCryztnmPiFhrPOojSZLGn8GGpFE6htLlqt3O41ERSZI0/uxGJWlkMnNuRJwBvK5t88aD5I2INYCtgTWBVYE7gOuB32XmzaOqY0QsWZWzPtBqdTkvM88YMP8qlPEr6wIrAzcCZ2XmFX3yrUCZtWsjYFngJuCPmXnhGJ7GpBERywGbUd4Hq1Oe+x3AzcCfMvMf41i9MYmIFYHtgHWA1YC7KO+DczPzXyMuq/39tiLwb+ACYHZm5ijLalJEPAGYTpm9bmlgDnBaZt7UJ9/jmPf8g/KdcFpmXjPCuj0VeEpVt4eAa4DTM/OWERx7XeDZwBrACsCtwHWU77XbF/b4tbI2BLagfK+tSPmOOSYzHxplOW3lrUz5bG9E+c5eGrideZ+Fkf2P2sp8JrAJ5bN3H+Xi1qj+V48GtqG8Dx4DLEl5Pn8D/pyZt47xuCsDzwHWpnwH3gPcAJyTmVcvbL0nhcz05s2bt/lulBPxbLvNGSLvwbW8X+2RNoBXA38EHqnla93mAmcC242x7jOr7Y8CPkM5WamXcWJb/hm1fTu2Hfe7wP1d6vkb4Akd6rMq8FXKD2OnfBcAzxrwuZ0+lv9Jlbe9zFkd9s/qUr+ety5lrQd8EDgLeLDPMeYA7waW7VP/+v9l0NuOw74WPerwTOBXfZ7TRZRgOwY85sxa/vWr7WsB3+7xvrkaeMNE+Q6ov3/atu8AnEbnz/eD1Wdj+Q7HezLwyy75HgF+BKyzMJ8bynfP+V1e3weBHwKPG+Pr9yrKZ7vb++Qh4GTg2UMcs9vzeCnwhy7lrFzLN+it4/8b2BQ4CPgT5bu51zEuBfYGpi3s9xTw38BlXcqZCxwLrDvG/9VOlO/vB3o8l7nAucDbO71fuxz3+dVr/3CP4/4Z2K2pz/FEuY17Bbx58zbxbiyCYIPSgnH2kD+AX6TPSVyHus+knPxe2OO4PYMNypXV2wao383AZm3H2hi4aoB89wPPG+C1PX0s/5Mqb8cf8bb9s4b8XySQHY6zJN0Dx163C4D1etS//n8Z9LbjsK9Fh/QBHDLk8zodWGWAY8+s5VufcoJ+44DlHDYRvgPq759q2zsHfM3+SNsJHPAy4N4B8s1hgBNMOnxugCMGfH3vZIDPZltZKwKnDPkePQxYYtjnUb0vD+tz7JEFG8ATxnCcBE4FVh3L9xSlteRbA5ZzHfDkIf5XKwE/HcPz2aPPcVegTJYyzDF/ACzTxGd5ItzsRiVp1NapPf53PUHV3H8qJQhodz3lpPM2yo/2dOZ1dYJyBXwF4E1D1OdRwAlAa6HB+ylXqK6vjvXkPvmfBHyO8sMEcCVwMeWK84ZVHVuzG60O/KjqlrEK5Tm2Xo8bgNmUk5fHUbpytb6DW1MFb5wj6A4wzoJ5rweUH9IrKV0Rbq8er07p7rF6W7rNgZMj4hmZefciquugvg7sU9v2IOVq8nWUE7pnMv/z2QE4MyK2z8zbhijryZSrtCtWj6+hfCbupHwWtqa8p1veFRF/yszvDFFG4yLiVcCXqodJeQ7/oLw3tqB8dlqeCXwF2DsidgKOY95n4y/V7UHKVfWntuVbj3IiutOQdfsosH/bpkuBK6q6bVKV07ICcGJEPC8zz+5z3BWAM6rn1+424Lzqfi1Kt6pl2va/i/Leef0wz4OykOq7as/jb5RWk8dTXtdRqo/znVuV909K18glKd3FtmDe9yWUcXs/iYgdM3PukGUeQWnVgPK8/kj5TCxFeS88sS3tWsAPI+Lpmflgr4NGxJqU7+f69/9cSmvXtZTv+NUoXcXWHqSyEbE6pcWq/h64pTruvykzNT4V2KBt/yuBlSLihZn5yCBlTSrjHe148+Zt4t0YY8sG5cfm6lre19fSLMOC3Qt+BzynyzF3o/y4tKd/3RB1v6u6fwD4CLBchzzrtf09o5a/1aJxOZ2vkm9OOYlqz/MWSheQrOq+J7UWGcoPzexavkP6vL6nD/s/acs73xXDDvtXr167+ut3fNv2BW4djjONclLwQ0pXkpW61GcJYFdK4NZe3hFd0i/fVm57l5FretUPeNSwr0Ut7V619I9Qriav3OF570M56WpPf2yf48+spf93dX8RsFOH9KtSTsbb81wLLDme3wEs2DLW+tz8HNiwQ/pXMX8XsUcogdR11eOzgS065NuRcuLWXtaLhvjc3M28bnCzgad3SL9V9fq3l/F3+nf1O7qW507gzcBStXQrAYeyYKvPm4d4Hvcxr3vOr4BNOqRfp3pfrln9P7etlXcY3T83C3Qfo5zY3wd8E3hJt9eDEgi8igVbdd8/5PdU67Mwl9IFduUO6V/Mgq3Ob+1TxpKUoLD+v/pIpzKqPBsCH6b8vnVs2aAEq7+qHfdi4EV0aJGnjOG7tJb+I6P8HE+U27hXwJs3bxPvxtiDjfqJWQJr19J8trb/aPqcKFU/mu0Bx7XUfsB71L31Y9XzhKQt/4wO+S+lRzcAylWq9n65rR+/64DH98i3FvOCoVb6rt0paDDYGGvaDnmj13PukH454Jy28u7t9Vov7OswzPOjBDi319K/p8+xn0UZANqep2s3HBYMNrJ6PVbokWcJyniYgU+4x/Aa1T9HPV9nOnfD+26f9/Nba+lbn5tT6NGlhHKi257vuCHeL63buXS48NCWZyUW7Hp5QI/029XS3kuXCyhted5ey3M3PbredXkePV/jPv/TmUO+Jx4NrD5E+sdSWj5a5V1Dn/EbHZ7fI8Cr+uTZtZbnvD7p/6eW/npg8wGf01Ld/keUwLL9uL+mf4Baf5/dN8xrPFluTn0raSSq7g+H1zb/PDOva0uzIvN3X7gY2Df7NK1n5rWUL/KWtSnNzoM6IjN/OUT6ujdkj5lIMvNiStN5y8rV/Vuzx+xEmXk95WShZS0GnL1rospi4BmZMvMe5v/fLktpzZoI9mL+7iAnZeYXe2XIzHOBT9Q2v6tT2i4eAF6bmXf1KOMRyiDddjsMUcaicB3lSn2vLiHfoLQEtaxMOUn/78x8oFumzPw5pdtTy45D1u3Bqox7epRxB/AGyoWKlv0iolv38/oaQwdm5u97VSIzD6e0/LQsx4Ld9Xq5DnhLn9d4ZDLz3sxcoFtsj/Q3Ae9t27QOpXVlGF/PzB/0KefXlAC9ZXo1C94CImIZ4H3t2YH/ygFnBczMh7JDt8hqlsMPtG26EXhlZt7X53h3ULrPZbXpUcB+g9RlMjHYkDSIaRGxfofbZhHx8oj4HuVke/m2PPcBH60d5/XM64sO5Qf54UEqkJm/oPQNbnnJgHVP4AsDpu3k9MycPUC639Ye/4My+HDYfJsPVKspJDMvoozraHnWeNWl5r9qjw8YMN+XKZMFtLwoIlYdMO8PMnPOAOl+SwlMWiba++bI7DP2Jku/+voJ+ferILyf9s/NY6s++IP6QWb+tV+i6gS0/TO8Nh3Gh1Qntnu0bbqN0kVpEB+rPR5m3MaRvYLSCeIk5n+fDvvZPmTAdO0Xk5Zg/rE97V5K6VbWcmJm1r+Dx2JXygD6ls9l5p2DZKwuVp3WtmnQ37ZJwwHikgaxDvOfDPbzEOXK4UW17e0/1PcDvxiyHr9j3hf6cwbMc/GAJ2/d/GbAdPX1Ik7Jqp28j7/XHj9mwPImnYh4FCXYXJb5B5FD6YffGjC5yaKsVyfVFdDpbZv+mZnnDZI3Mx+KiB8yrxUvKIOCB2ldO2nAMh6OiL9T1oeAife+Gevn5uSOqRbU6XNzw4B5e14przmOMuaqZWsWrOOWzH8+dUKvlpl2mXlhRFzGvIHKT42IFQYMIga5mLFIVGtUrEC5Ml//bN/GvBP8YT7bf8vB1+H5S+1xt89DPVg8aoj69FI/7vFD5v8d8xbAnR4Rywz6HpoMDDYkjdqFwD5dWgPaA4SrgDUj6r9LPbU3Sa8bEUsM0IXggmEK6KD+I9ZN/SpWz0X+euRbsWOqSSginkRZc2InypXGQa/ur9JYpQa3GWXazZZzh8z/B+bvMjidwYKNy4coo70L0kR730zkz80fFyLt9A5p6tvG8l5pBRtLUGYy+l2fPHOBS4YsZ2QiYnPKZ3tbymd7hQGzDvPZHutnAbq/H9pbVpIFW9bGqv237W4gImL9IfK3L7y4NKUVbZgLfBOawYakhXEv5Uv+n5Qf2J9k5pmdElZ9Wtubrzdm4b5Mg/LD1W+q2IVdfbz+I9ZNvTvYWPMtNWC+CataMffzwBtZ8CrnICbCiXP9yujfhsxfP2ketOVh0PcNzH+CMtHeNxP1c3NvZg7aAgLlO+oR5nU7f2yHNOPxXrk9G1oZvJdqRfevMH+3sWEM89ke62cBur8f2n+DrqvGTIzC49r+Xp6FDxRWHcExJgyDDUmDuCoz11/IY6zC2E48e1me/sHGwq7ZMNbBl1NvrvQBRERrfZGnL8RhJsJ4wpVrjwfqf92mfhIz6BXdKfG+WYhBy00//6H+j5mZEXEn894P9fdFp22L4r2yyNeiqa7Un86C6yMNY5jPdhPvhdXa/r59hMcdtNV2UMv3TzJ5TIQvdEmLhyauvI46eNHC+wLzBxr3UxZe++9q+xqUWXeWzMxo3Sjz3kuauL7J/IHGncDXKDMDPo2yVs+jKVPxtn+2r1rkNR3MIGPqBjXq37cp9dtmy4akRaU+dex5mTlRZh2a0iJikVxYioh1KdPFtlwH7JyZg/TDH7TP96JSv+o5bNeulWqPh1lFXM0Z6v8YZVBZe55OV8On/HslIrZl/kHQlwDPH3DmsIn02b6VeauBd2qlWpjjtrpo3ZSZa4zw2JOeLRuSFolqZo327gWrj1ddJrH2furDXCwa5Y9qLy9i/ityHxgw0ID5+1JPBPWxPk8cMv9GfY6n8fHoIafJ3YD5z5Vu6pBmcXivvLj2+C2DBBrVDHSL6vtnEO3jddau1n4ahfb3xaqL6gLPZOGLIWlR+kPb3xtExESbrnOiaw/W6ldDe3ly/yQjUT/J+vUgmaoWkbX7JpxnlN0furmEsvhby1ZD5n927fEga7Vo0dhyIdJ2+j/Wtw3bYtv+XnmEhZ9Br5uF+dy0f7bv7rdgYZstmVjnmu2L/y3B4FOo99P+2zYNeMaIjjslTKQ3gKSp75S2v4PhVgHX/Fc8l69mhhnErkOW0z6/+9JdUy2oHgANOlC2vnheP2Ot38Cqlrj2k8gnRkSnaU8XUK0y3f7eToafDlXNedUQaV9de3xOhzR/ZP5Wxz0jYqD3ZUQ8jXlrpQBc0uBCffV1G8b62R6mfsN+tpt2Wu3xm0Z03FNqj4d5j015BhuSFqVvMf9aGR8eYmVllTVM2r2gX4Zqdqg3D1lO++w4w3Q5qfddr3cPWUDVuvWeIcqA+eu3ejWtchO+V3v8iQHzvZ35p0j9VWb2mzVNi86rImKQ9+bmwG5tm65jwZNVMvMe4Cdtm1YF3jZgXQ6qPf7OgPnGoj7r1Vg/24+tprfuKSI2Bt4wRBmLws+A9u5fe0TE9g0cd/+I2KBb4sWNwYakRSYzbwT+r23T44AThg04ImL7QU4WpqD6ic4Hqj7RHVVXV49h+NWl28dZbBkRg07DeHHt8Xt7Ja5WHf4+ndcu6KW9fktRFhZrwjHMf4K2W0Ts3y0xQERsCXyytvnLo66YFsrSwLcjYrluCSJiJcrFkfZA9sjMrK/v0VL/H3+yei90FRFvBXZv23QP8I1eeRZGZt4HXN22afshAvX2z/aSwDt7Ja4uIvwQWHaoSjYsMx8EDm3btATw/YjYbJD8EbFUdQGnftz7gf9t27Qc8POIePww9YuILSLimcPkmQwMNiQtah9j/j7J2wMXRMS+1clnRxGxUUS8PyJmU6ZJfULD9ZxwMvNy5u8b/CTgZxGxTj1tRDwD+C3wUoafT759YcblgV9ExJ4RsWlErN9+q+X7FWWhx5a9I+ILEbHAbDQRsR1l9d6dKd2MhrnyX1848piI2Kf6od6gVseuwVg/VXeWesD0lYj4XHUy+h8RMS0i9gZOZv458n+QmQONXdEicQ9lAbitgDMiYoH1YCJiK8p77Gltm/8JHNLtoNVipt9u27QccHL1vpxvMoeIWDEiDgEOrx3m/ZlZn7Vv1No/OxsCx0fEiyJi49rnpt5F88fMP+bjgIj4QEQs054oIpaIiN0o31NPpUx9vcjXBOnjMOZ/HdYCfh8RH6x/rlsi4gkR8WHgH8AOXY57OOU7sOXJwJ8j4n2dApS2Yz8+It4WEWcAfwamXLDh1LeSFqnMvDci9qD0cW0NOlwXOBI4IiIuAK6l/ECtQLkq/xSGGxA9lb2f8kPZmvXpucCVEfEH4BrKPPdPpgQiUAaczgBOHKKMo4D3Aa0T9e2rWyf/mX0qM/8dEYcCH2/b/x5g36p+N1KmBd0caL/idyhlIGm3H/G6U4HLmDfwfb2qzp3sRFmIbEwy85tVN4vWlL5LAP8DvCMizqF0nViZUv/6DGuXAW8da9lqxL8prQcHAdOB8yPiEkprWQCbVrd29wL/XbUM9PJ2ynu7FaSsRHlfHhwR51GC/jWBrZn32Wr5fmZ+bUzPaDhHAK9j3ud2DzqvBn4VsH7rQWZeGhHfoayXA+VzcDDwoeqzfStlMcLplLV0Wv6H8p01YRapy8y5EfEayvdI63+9IqVl4lMRcT7lu/QBSpe4zYAFLuh0OO4jEfE6ysQYrQklVgU+DxwSERdTXtc7KcHoapTvsCk/M6PBhqRFLjOvqroYHMP8faKXopy09Zst5mEm3tWyRSIzz4qId1OuzrVOGJYCtuuQ/CHgTZn5k7JcwMBlzImIGcDRDN8N4kDKD/gr2rYtTwmKOjkS+CClFWbQ+mVEvBr4BfMHLU3Zm3Iy9W7mvebLADv2yHMWsPsiuFKt4X2K0oVzv+rxZtWtk7uAl2fm2f0Ompl3RsQOlFaA9jUpVgNe2CPrEfTpljQqmXlORPwPpZVm2LFOb6G0KLfP4LQK3Z/bxzPziIh4//A1bVZmXh8R2wDHMv/Yt2mUQGHY2edax729ujhxOLAP874vlqAEopv3OwQLjq2Z9OxGJWlcZObtmbk75Yr5z5l/4HgnD1KuUL8fWDczz2q2hhNXZn6ZMu/9pV2SPAL8EnhWZn5rjGUcRwkaDqS87tfT/39EZs6lzMTyLuaf077uHMpJ3Jsz85Ex1O8SSjeNt1HeP3MoAejIp8XN4r2UKUp/w/wzD9VdShkUu72BxsRU/T/fDLweuKhLsoeA44GnZObJQxz7dmAXSutBt2MDzKVcWX9OZr59LJ+BscrML1BaXz4HnE2Z5a4+U1WnfPdSgqiD6H5CPJfSlXCnzPzUSCrckOo36IWU9YHOpNS9m7mUbp/7Ub4Deh33gczcF9iCEsz0m5VvLmW2ugOADTPz2MGeweQRmYtiunJJ6q3q+/tsStP96pRuBndTfgj/Alw+QDeGxU5EPIVyFe6xlBOGq4E/ZOa141oxymBKSt2eRrkCeiclaDk/M68cz7otjKpf9/aUrhWrUt6nNwLnZuaccayaOoiI05nXRe+qzFy/tn9zSlfNdShBxjXAaaOYQawaIPxsSteiFSgtZNcBv8vMCbdS+KAiYllgG8oFiZWY97zOzcxeFxkmrGqGrW0pa/6sRgkCbgf+Bvy5CiTHctxplHEYT6qOuxxl7NCtlO57lzU43fGEYLAhSZKmrH7BhqRm2Y1KkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiNcZ0OSJElSI2zZkCRJktSIaeNdAWlxFBFXAisCc8a5KpIkSf2sD9yZmRsMm9FgQxofKy677LKrbrrppquOd0UkSZJ6ufzyy7nvvvvGlNdgQxofczbddNNVZ8+ePd71kCRJ6mn69Omcf/75c8aS1zEbkiRJkhphsCFJkiSpEQYbkiRJkhrhmA1pnFz09+tZ+0UHjHc1JEnSJHbdLw8c7yr0ZMuGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbGhKiog5EZED3mbV8kZE7BERx0XElRFxT0TcHxHXRMQvI+KdEbHaOD01SZKkSWPaeFdAashhwMo99j8aeC+wJHBJa2NErA0cB2wL3AucBpwA3A+sCWwDfAn4dEQ8MTNvbKT2kiRJU4DBhqakzDys276ICOAHlEDjeODQavvywEnAU4EfAm/NzFs65H828Flg2dHXXJIkaeow2NDi6CDgFcCfgb0yM6vt76UEGr8DXpOZj3TKnJl/iIidsRuiJElSTwYbWqxExGuBjwE3ALtl5r1tu99U3X+yW6DRUgUoc5uppSRJ0tRgsKHFRkRsBXyTMv5ij8y8pm3f44F1gYeBM0dY5uwuuzYZVRmSJEkTlcGGFgsR8TjgROBRwOsz89xakjWr+1sy84EO+fcAtqhtPj0zTx91XSVJkqYKgw1NeRHxaOAnwFrAZzPzu2M4zB7AXh22n94rU2ZO71Kn2cAzxlAPSZKkScMBrprSqpmnjqGc2J8IfLRL0huq+9UiYpn6zsyckZmRmQHs20hlJUmSphiDDU11B1JmnrqI0n0qOyXKzH8BV1Na+7ZfdNWTJEmaugw2NGVFxGuAjwM3UWaeuqdPlm9U9x+NCD8bkiRJC8kTKk1J1cxTRwMPAntm5lUDZDuUspr4DsB3I2LVLul6rUwuSZKkigPENeVExArMm3nqj8DzI+L5PbLMycxZmXl3ROxKWT38NcBuEfFb4ApK0LIGZezHFsBdlEUBJUmS1IXBhqai1SgzTwFsWd16OQOYBZCZ10XEdsDuwOuArYDnAgHcAlwMvBv4TmbeMvKaS5IkTSEGG5pyMnMOJTgYa/5HgBOqmyRJksbIMRuSJEmSGmGwIUmSJKkRBhuSJEmSGmGwIUmSJKkRBhuSJEmSGmGwIUmSJKkRBhuSJEmSGmGwIUmSJKkRBhuSJEmSGmGwIUmSJKkRBhuSJEmSGmGwIUmSJKkRBhuSJEmSGjFtvCsgLa6e9sS1mP3LA8e7GpIkSY2xZUOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDVi2nhXQFpcXTrnZjae8dXxroYmuStm7T/eVZAkqStbNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBjIUREVrerIuJRXdLMqdJM63Gcj7Yda+M+Za4dEV+MiMsi4t6IuC8i/hURZ0TEpyNiw1r6WW3HzoiYGxF3RMQ/IuLEiHh7RKw2wHNdt8qbEfGZAdJPi4h9IuI3EXFTRDxY3f86ImZExJJ98i8XEe+OiN+25b89Is6rnucTuuRbJSI+UaW7LSIeiIirI+IHEfG8HuW1/k8ZETv3SHd0W7qZ/V4HSZKkxZnBxmg8Hnj3WDJGRABvArLatG+PtJsBF1dlPQwcA3wB+DWwMvARYKcu2X8CHAh8EjgKmA08C/gK8M+ImNGnqm+ivF8S2LtP8PQ44LyqnKcBvwA+D/y8enw08PuIWKNL/mcDVwBfBJ4E/LLKfwxwP/BB4C8R8Yxavu2Bv1bPc3ngu5TX51zgxcBvIuLbEbFMj+f5cPVcO9VrReBVVRpJkiT10fWEUQO7jXIC/qGIOCoz/z1k/ucD6wOzgBcAe0XERzLzwQ5pDwNWBWZm5oH1ndXV/qW7lHNiZs6qpZ8GvBH4EnB0RDyQmcd2OO6SVbo7ge8A+wO7AT/ukPbRwK+AzSjBwf6ZeW9t/1eBvYCfR8Rz2p9rRGxCCZ6WBz4EHJqZ853cR8QGwMHAim3bnkwJSpYF3gEckZnZtn9d4ETg9cCDwD5dXqefAy+LiNUy85bavv8CHg2cAOzZJb8kSZIqtmwsvHsprQUrAQeMIX+rJePrlCvxq9P9RHab6v5LnXZm5j8z8y+DFpyZD2fmkZTgAeALEbFsh6QvBB4HHAd8rVbvuvdSAo2zgTe2BxpVmfdSApezgWe2ld3yFUoQcXBmHlwPNKpjXJmZrwLOadv8ZWA54JDMPLw90KjyXA28hBIcvjEitqGzrwPLAP/dYd++wNXASV3ySpIkqY3BxmgcAfwDeHNEPGnQTFU3ot2Av2bm2ZTWDYD9umRpXWnfaIz17OYY4CpgTaDTeIVWfWZl5iWULljPj4j1OqRtBSGfysxHOhVWbf909fAtrSY7OLcAACAASURBVO1Vi8VzKV2lDulX6cx8oC3fLsADvfJl5vWUrl0Ab+6S7GRgDrWuVBExHXg68E2g4/OSJEnS/Aw2RiAzH6J0+VmK0r1nUHtXeWZVx2mdyO8UEU/skP646v6nEXFARGxfjSNYKNXJ/++qh1u174uIdYAXMS8goqrvEix4Qr4uZfzKw8DpfYo9rUq3cdvYjW2r+9mZefsQT6E932190p5c3T+ny/4EvgE8JSK2btu+LyXI+OYQ9SIiZne6AZsMcxxJkqTJyGBjRDLzeEq3nj0jYtt+6dsGhj8CfKtt1ywg6NxN6aOUbj6rATOBM4DbI+IvEXFYtxmaBnRtdf+Y2vY3Aksyr9UF4HuUcQ9vrM0qtVZ1f0tm3tersGp/q6VmnVr+awav9nz5rh4gbSvN2j3SHA3MpfofRMRywOuAX2fmv4asmyRJ0mLLYGO03lfdf36AtDsDGwInZ+a1bdtbJ/IzImKp9gyZ+UBm7kcZPzGDMn7iPOCJwLuASyLiJWOse7SK+c+GiCUoA6nnC4gy81bgZ5QT9hePsbx2HacNHi/V/+OXwKsiYgXgNcAKlEBv2GNN73QDBh5bI0mSNFkZbIxQZp4DHA88KyJe3Sf5f8ZB1I7ROpF/LLB7l3JuzMxjMnP/zHx2lfYoykxM34yIbjNS9dK60n9z27ZdgfVYMCBqr3f7+JIbqvvVugw0/49qf2t9j1aZ11f36yyYo6dWuesOkLaV5ro+6b5OGXD+OkoLxw2U/4skSZIGZLAxeh8GHgI+2+2kPyIeA+xRPTy2tuheAi+v9nUbKD6fKkB5M/AvSjeozYapcNWCsX318Ny2Xa3yd+1Qx9aJ9wuqsRpUXYyupkypvGOfYnes0t1OGVwPcFZ1/8yIWGmIp9DKNz0iVu6T9rnV/e/7pPslpWvZxyjrkRzdaWYsSZIkdec6GyOWmX+PiK9SujW9o0uyvSjrYcwGLuiSZjfguRGxQWZeOUC5j0TEPdXD6Jl4QTMoA7uvpwzcJiLWpEwVeyfwwy75NqEMtH4jZSE9gCMpUwF/JCJOqk9BWx17CcoChADHtWatyswrI+IUSkDwfsqJflcRsUzVteyfEXEaZUHD91PGtnRKvwbzxsIc2evYmTk3Ir4JfJzSteyoXuklSZK0IIONZhxECSg+SudpUlsnvPtn5nmdDhARn6ScbL+pOg4RcQBwTGbO6ZD+FZST/9uASwapZLWo396UNSoSeE9m3l/tfiPl/fHdzKyvhdHK/0TKit37RMQnq6Dhi5QxDtsCR0XE29sHi1fdpw6v9t8IfKp22HcCfwA+HBG3AV/qsKjf4ylT3P4f82a9eleV74MRcU1mfq2WZx3Kon6rUFop+rVsUL0u5wP3ZOY/B0gvSZKkNgYbDcjMWyPiM3RY8yEidqSsk3Fxt0Cj8g1KkLF3RBxQnXC/B5gZEX8G/kQZ67AS8Axga8pUsm9prT9Rs0dErF/9vRylJWM7ykxOdwD7ZeZxVR1bM2VBjyv6VSvOGZQuUS8EfpGZ90TECyjdrN4IvCgifkkZ87AGZRrdtSgtJrtn5jW1Y14eEbtSxr58HnhXRJxKGWOxHLA5pTUlaZtmODMvrgbHHw98NSLeRmmluYsygP7FlNW/vwu8tdtzqtXl35QARZIkSWNgsNGcL1NWx16/tr3VqtGzW05mzqm6FD0PeClwAqVb0wuBHYAXUE7eH6ZMFXsU8OXMvLjLIXevbo8A91AClfOAU4DvVeM+Wp4LbAD8OTPP7/M8v04JNvYDflHV/ZqI2IrSuvMaSpewVZk3RuhnlODmhgWOVvL/ISI2obxWu1EChVUoq7X/HTgUOLLevSwzT4uIjSjd114CvIEy09XNlDEYX8/M3/R5PpIkSRqR6NClXmpEROwCnAT8DdghM2/uk2XKiojZy6y67jPWe+mHxrsqmuSumNWxl6MkSSMzffp0zj///POr6fuH4mxUWmQy81RKF6ZNgZMjYpVxrpIkSZIaZDcqLVKZeVREPETpprUtrl0hSZI0ZRlsaJHLzGPGuw6SJElqnt2oJEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCFcSlcfKU9R/D7Fn7j3c1JEmSGmPLhiRJkqRGGGxIkiRJaoTBhiRJkqRGGGxIkiRJaoTBhiRJkqRGGGxIkiRJaoTBhiRJkqRGGGxIkiRJaoTBhiRJkqRGGGxIkiRJasS08a6AtLi64tpb2eYDx453NRba2Ye8dryrIEmSJihbNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDkiRJUiMMNiRJkiQ1wmBDU05EZHW7KiIe1SXNnCrNtC77N4qIL0TE+RFxa0Q8VN2fGxGfj4jpzT4LSZKkyc9gQ1PZ44F3D5MhigOAy4H3AAkcBxwCfAe4D3gH8KeIeNtoqytJkjS1dLyqK00Bt1EChQ9FxFGZ+e8B830CmAlcDbw2M39fTxARj6UEMSuNqK6SJElTki0bmqruBT5JCQgOGCRDRDwB+BjwIPDCToEGQGbelJkfobR2SJIkqQuDDU1lRwD/AN4cEU8aIP3elNa+4zPz0n6JM/PhhayfJEnSlGY3Kk1ZmflQRHwI+CFwMPCyPlmeU93/dlR1iIjZXXZtMqoyJEmSJiqDDU1pmXl8RJwD7BkR22bmWT2Sr1ndX1vfERHrAzNqm2/PzMNGUU9JkqSpyGBDi4P3AWcDnweePcZjrM+CYz+uAnoGG5nZcYrcqsXjGWOsiyRJ0qTgmA1NeZl5DnA88KyIeHWPpDdU92t3OMbpmRmZGcBSDVRTkiRpyjHY0OLiw8BDwGcjYukuaVqzT+2yaKokSZI0tRlsaLGQmX8HvgpsQFmUr5NZwMPAKyJi00VUNUmSpCnLYEOLk4OA24GPAsvXd2bmP4BPAUsDv4qIbbocZ+XGaihJkjSFOEBci43MvDUiPkPvxfgOAgL4OPD7aiD3ecCtlCBjfeC5Vdozm6utJEnS5GfLhhY3XwbmdNuZxUzgyZSZpqYBrwM+WN2vAXwNmJ6Zb2i6spIkSZOZLRuacqoZo7rte4AybqPfMa4A3jPKekmSJC1ubNmQJEmS1AiDDUmSJEmNMNiQJEmS1AiDDUmSJEmNMNiQJEmS1AiDDUmSJEmNMNiQJEmS1AiDDUmSJEmNMNiQJEmS1AiDDUmSJEmNMNiQJEmS1AiDDUmSJEmNMNiQJEmS1Ihp410BaXG18TqrcvYhrx3vakiSJDXGlg1JkiRJjTDYkCRJktQIgw1JkiRJjTDYkCRJktQIgw1JkiRJjTDYkCRJktQIgw1JkiRJjTDYkCRJktQIgw1JkiRJjTDYkCRJktSIaeNdAWlx9Y8b72DPQ3/ReDknvO/FjZchSZLUiS0bkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbmnAiIiMie+zfJCK+EhGXRMQdEfFgRFwXEb+IiH0iYpla+iUjYt+IOCMibo2IhyLipoi4KCKOiojd+tTnv1p1iojnj+p5SpIkTXXTxrsC0jAi4hPAAZRA+RzgGOBuYA1gR+Ao4K3AM6v0SwI/B14A3A78ArgGWBp4CvA6YBPgpz2K3Q9IIKq/fzPaZyVJkjQ1GWxo0oiIjwAHAlcDr8zMczukeQnwvrZNr6UEGhcCO2TmHbX0jwae1aPMjYHtgVOAVYDdImKNzLxxIZ+OJEnSlGc3Kk0KEbE+MBN4CHhRp0ADIDNbrRgt21T3s+qBRpX+3sw8rUfR+1b3RwOzgKWAGYPXXJIkafFlsKHJYm/Kif6PMvOSXgkz84G2h7dU9xsNW2BELA3sBdwBnAB8D3gQeFNExLDHkyRJWtzYjUqTxbbV/alD5vsx8EHgLRGxAiVomJ2ZVw2Q92XA6sCRmXkfcF9E/Ax4ObDzIHWJiNlddm0ySOUlSZImM1s2NFmsVd1fM0ymzPwz8Hrgxur+R8CciLglIk6IiJf2yN7qQjWrbVvr7/2GqYckSdLiyJYNTXmZ+YOIOAHYidJC8vTqfg9gj4j4FjAjM/8z3W5EPLFKf0VmntN2uJOAG6p8q2fmv/uUPb3T9qrF4xkL8bQkSZImPFs2NFlcX92vM5bMmflQZv4mMz+RmS+ldI96NXAP8AZg91qWfSlT3c6qHedh4LuUqXNnjKUukiRJiwuDDU0WZ1X3u4ziYJk5NzN/AHyx2rRza19EtM849dm2Bf1aiw22ptbdF0mSJHVlNypNFkcDHwZeHhFPzszLuiWMiGVqM1L1clcrW9u23YHHAlcwL8ip2wnYKCJ2yMwzBixLkiRpsWKwoUkhM+dExEzg08AvIuKVmfmnerqIeAHwAaqWioh4LfBv4NTMfKSWdk3mtU6c2barNfj7E1XrxwIiYh/KauX7AQYbkiRJHRhsaNLIzM9ExDTgAOCPEXE28CfgbmANykrfT6q2tTwLeBdwQ0ScBVxZbd8AeDGwLPAT4HiAiNgAeC4lQDmxR3WOAw6jtLS8IzNvHcmTlCRJmkIMNjSpZOZBEfFDYH9KV6a9gUdRFu+7ADgY+E5blkOBv1ECiKcBu7alP52yUN/32maiehOlS9W3M/PBHvW4OyKOpbSM7MW8sR+SJEmqGGxowsnMnqtzZ+blwDsGPNbVwBHVbZD0HwU+OmDa/XC9DUmSpK6cjUqSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSIww2JEmSJDXCYEOSJElSI6aNdwWkxdWGa6zECe978XhXQ5IkqTG2bEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEYYbEiSJElqhMGGJEmSpEZMG+8KSIurq2+5m7cf87uRHvPwvbYb6fEkSZIWhi0bkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhphsCFJkiSpEQYbkiRJkhox8mAjIh4TEW+JiC9FxFG17VtFxLKjLlPzRMSOEZERcXqPNOtXaebUtq8dEV+MiMsi4t6IuC8i/hURZ0TEpyNiw1r6WdVxWre5EXFHRPwjIk6MiLdHxGoD1PmVVf6XV49nVo9ndkl/YLX/yojYqNo2o1aXjIgHIuKqiPhuRGxeO8b7qjQHdynj/3V6jdr271Tt/22/5ydJkrS4mjbKg0XEPsCXgUcBASTwpmr3GsA5wH7AN0ZZrhZeRGwGnAGsClwMHAPcCjwW2Ar4CHAl8I8O2X8CXFD9vQKwLrAdsDvw6Yh4V2bO6lH8nsB9wK/61HFJ4KuU99CFwAsz8/pasguBE6u/VwSeA7wOeHlE7JKZv6/2nVrd79yluF0o79/1ImLDzKw/712q+1N61VmSJGlxNrJgIyKeBxwJXAQcAOwKvKW1PzMviYhLgT0w2JiIDqMEGjMz88D6zoh4ArB0l7wn1oOJiJgGvBH4EnB0RDyQmcd2OO7SwIuB32Tmvd0qFxGPAo6lvH9OB/bIzDs6JL0gM2fW8v4f8GbgU8BO1eYLgVuAp0fESu3HiojHAxsCPwReSQlI6sFGK0g5FUmSJHU0ym5UHwSuB3bIzJ8CN3VIcxHw5BGWqdHZprr/UqedmfnPzPzLoAfLzIcz80hg/2rTF7p0oduZ0gJxQrdjRcTKwG8ogcbxwAu6BBrdtILbLdvql8BpwJLAjh3qBKWV7hZqrR8RsUJ1rDuAPw1RD0mSpMXKKIONZwI/z8w7e6S5BlhzhGVqdG6p7jca8XGPAa6i/N87dVl6GfAw8LNOmSNiHeB3lG5ZXwVenZkPjLEuD9Uet8Zb7FLbvgtwD3AupRWlXu/tKa2CZ2Tm3DHWRZIkacobZbCxNOUErZeVAU/OJqbjqvufRsQBEbF9RKy4sAfNzEcowQKUsR//ERFLUMZ1nJGZt3bIvjFwNrAZ8InMfFt1vGHtV92fVdvebdzGTsBZmfkQpfXjsdWYFmrp+3ahiojZnW7AJsM9BUmSpMlnlAPE5wDT+6R5FnDFCMvU6HyU0p1pb2BmdcuI+CtwEvDlzPznGI99bXX/mNr2bSgD0Lt1oXpNdf+NzPzkgGVt0TaL1YqUFpFnAtcB72tPmJl/jYhrgKdExGMz86aI2BhYh9KFCkqwASXAuKTtb3C8hiRJUk+jbNn4CbBdRLyy086I2Bt4GvCjEZapEcnMBzJzP+BxwAzga8B5wBOBdwGXRMRLxnj4aBVT275nte1EOjsTuB+YERGvH7CszSkTFBwAvIcSaPwL2Doz/9ohfasr1c61+9MAMvMy4MbW9moq382BGzLz0n6VyczpnW7AwONfJEmSJqtRBhuHUE7qjo2I44CtAaq1Fo6jzFT1N+ArIyxTC2p1M+r1v23tW6BLUmbemJnHZOb+mflsSsvDUcCywDer2aOGtXZ1f3Nt+57AHzPzWjo7DXgJJeA4JiLe1CVdu2MyMyjPcU1Ki826wM8i4tEd0rdaJ3Zpu78DOL8tzRnADtXUuztRgidbNSRJkvoYWbCRmbdRZvU5izJd6PMpJ2Vfrh6fDeySmf3GdWjhtGZp6rWY3urV/e39DlaNpXgzJZB8DGX8xMCqcRnbVw/Pbdu+BbAB8OM+5Z8KvAC4GzgyIt4+SLlZ3JiZnwEOpbSqfapD0v+0bEREUN7DZ9YGfp9GGW/0DOxCJUmSNLCRriCemVdl5o7AFsBbgY8B7wC2zMwdelzB1uhcATwAbNRj9e6tq/sLBzlgNSi7FSRGr7QdzAAeT5kW+bS27XtW912nvG0r/yzguZTg6CsR8YEh63AQpVXl7RGxQe3Y1wB/BZ4A7EYJ0uqrgreP2zDYkCRJGtDIgo2I+G1EfBIgMy/KzP+XmZ/JzCMyc/aoylFvmXk/8H3K4P/PVVfr/yMiHge8v3o4q237ARGxfqdjRsQrKLMn3ca8QdI9RcS0iNgXOIIyLuM9Vd1a9gQu6zKOYgGZ+UdKF6abgYMj4oBB8lV57wIOBpaiDHyvawUXrUHo7UERmXkFJVh6PWWGrL9n5r8GLV+SJGlxNcrZqJ4N/GGEx9PYvY+y6NzewNYRcTJwJ7AeZarZFYCDM/OMtjzvAWZGxJ8pC9XdDKxE6Tq0NWUtjLd0WeNij7ZAZTlKS8Z2wFqUbl37ZWZral0i4onAU4FPD/OkMvPCiNiB0qowMyKWzcwPDZj9q8D/AK+PiP/NzMvb9p1KWe3+qZT1Ri7qkP804HVt6SVJktTHKIONv1EG4mqcZeYtEfEs4J2UFoQZlAHet1AGO38tM39Zy/YS4IXADpQxEmtQAoxrKAPEv5yZF3cpcvfq1upudTNlJqtTgO91WEOj1YWq53iNLs/t8ojYnnLC/8FqVfJ3D5Dvvoj4DGUM0aeAl7ftPo3S+hLA6dXq4nUGG5IkSUOKzudVYzhQxDuAA4Et7GKiXiLibGDtzFx/vOsyXiJi9mPW2+gZrzrwqJEe9/C9thvp8SRJkqZPn875559/fjV9/1BG2bLxM+B5wO8j4mDgj8ANLLi2AgYji6+IWIvS5e7L/dJKkiRpchtlsPFP5nVF+VKPdDnicjWJZOb1jHgWNEmSJE1Mozzp/xYdWjEkSZIkLZ5GFmxk5oxRHUuSJEnS5Gd3FkmSJEmNMNiQJEmS1IiRdaOKiG8OmDQzc59RlStJkiRpYhrlAPEZffa3ZqpKwGBDkiRJmuJGGWxs0GX7ysCWwMeBs4EPjbBMSZIkSRPUKGejuqrLrquACyPi18BFwCnAN0ZVriRJkqSJaZENEM/MqymrjL9rUZUpSZIkafws6tmobgSetIjLlCRJkjQORjlmo6eIWBLYGbhjUZUpTWTrrrY8h++13XhXQ5IkqTGjnPp2+x5lrAvsDWwBHDWqMiVJkiRNXKNs2TidMq1tNwGcCbx/hGVKkiRJmqBGGWwcROdg4xHgNuC8zDxvhOVJkiRJmsBGOfXtzFEdS5IkSdLkN7LZqCJi+4h4fJ806/YY2yFJkiRpChnl1LenATP6pHlDlU6SJEnSFDfKYCMGTNNrELkkSZKkKWJRL+q3HnDXIi5TkiRJ0jhYqAHiEfGJ2qYdIzo2cCwJPB54DXDWwpQpSZIkaXJY2NmoZrb9ncCO1a2ba4EPLWSZkiRJkiaBhQ02dqruA/gtMAs4pkO6ucAtwBWZ+chClilNCTfecS+H/PLPY87/gRc9fYS1kSRJGr2FCjYy84zW3xFxDHBi+zZJkiRJi69RLuq396iOJUmSJGnyW9SzUUmSJElaTIw02IiItSLiiIj4e0TcFxFzO9weHmWZkiRJkiamkXWjioh1gPOANYBLgWWAq4AHgCdUZV0A3DGqMiVJkiRNXKNs2fgEsCbwgszcvNp2dGZuQgk2fg0sC7xshGVKkiRJmqBGGWzsCpyUmafUd2TmNcArKcHGgSMsU5IkSdIENcpgY01K96mWuZTgAoDMvBs4Gdh9hGVKkiRJmqBGGWzcCSzd9vg2YJ1amjuAx4ywTEmSJEkT1CiDjauAddseXwjsHBGPBoiIJYDnA9eMsExJkiRJE9Qog41TgZ0iYqnq8THA2sDZEfE54PfAU4DjRlimJEmSpAlqZFPfAt+gdJ1aHbg+M78TEdOBdwBPq9J8H/j0CMuUJEmSNEGNLNjIzL8BB9e2vSciPkOZ+nZOZt44qvIkSZIkTWyjbNnoKDNvBm5uuhxJkiRJE0sjwUZEbAJsCiyfmd9uogxJkiRJE9soB4gTEVtExJ8o620cD8xq27dDRNwbES8dZZkaXxGREZF90syp0q3fYd8rI+KkiLgpIh6KiFsi4rKI+E5E7FVLu36rvIg4t0+dFpj1LCL2iYj/FxHnVu/FjIhP9TjOjLbyOt56PW9JkqTF3chaNiJiI+B0YEngS8BGwAvbkpwJ3Aq8AvjZqMrV5BURRwL7AvcBvwCuBALYBHgpsCNlVrNOtoqI12Tm94co8lBgJcpEBtcBGw6Y70LgxCHKkSRJEqPtRnUAZVG/Z2bmZRFxAG3BRmZmRJwDbDnCMjVJRcS2lEDjGmDrzLymtn8pSrDRyb+AtYDPRMSPM/PBAYt9DXB5Zl4VETOAowfMd0FmzhwwrSRJkiqj7Ea1C/DjzLysR5qrKWtvSNtU9z+qBxoAmflQZp7cJe/VwFeBDShTKw8kM0/KzKuGrqkkSZLGZJTBxir0Xx08KK0f0i3V/ZPGmP8g4HbgoxGx6miqJEmSpFEaZTeqG4En9knzFMpVaU0xETGzx+6VO2w7CbgDeFFE/JSy4OMfgb9nZt+B15l5a0R8Gvgc8DHgvUNXenBbdHl+J2bmBb0yRsTsLrs2WehaSZIkTXCjDDZ+C7w2IjbOzCvqOyNiS0pXqyNGWKYmjgOGSZyZ10bEnsDXKYPBW7OU3RURZwPfAY7NzLk9DvMV4G3A2yLi8Mz85xjqPYjNq1vdHKBnsCFJkrQ4G2U3qs8CDwNnRsRbqcZmRMRTqsc/A+4CPj/CMjVBZGZ0uwEdx0lk5mmUWcu2Bz4O/Bi4F9gV+DZwUkQs06PMB4CPULrm/e9on9F8juny3Gb1y5iZ0zvdgL80WF9JkqQJYWTBRtWa8XLKid/hwJsoYzQuorRmLA28LDP/NaoyNfll/v/27jzekqK+///rzSZrQHFBITq4AGpcIiiKG4gLLhGQ4IYGUBaNuyTqzyQ6aPyp0UTUJC4QHDfEHcUVtxk0mCggRqKgUQcVVGTfkeXz/aP6wuFwzl1mTs+9d+b1fDz60XOqq7uq6/a905/TVdV1Y1V9u6r+sar2o80y9QTgd8BjgRfOcIip7lf7J3lov7WVJEnSXKxysJHkpUkeMphWVV+hzRD0SuATwNdp31b/LXDPqvrmatRV64BqTqKNwwB4zEz5gb/pPvrUTJIkaQFZnTEbRwFLge8BJLkBWFpVb6S91O+dq107rcsu79aZKWNVnZzkc8DeSfbrt1qSJEmardXpRnUNMNifPszixlACSLJXkqd1L+8b3rY58PLu48mzPOSraWOG+hy7IUmSpDlYnScbvwSekORdVfX7Lm3GKUulzk7AO4CLk3wb+BktWNgOeDJtutz/po3/mVFVnZ3k/cBfj8uT5BDgEd3HqWma/yLJdt2/z6oqgxVJkqQJWZ1g4320rlTnJTc90Fg6w/sWoHWzn+SUu1qcPgJcBjyONq3so4DNaS/qOwP4JHBMVf1xDsdcCjwH+JMx2x8BHDiUdv9uAViBT0YkSZImZpVv+qvqXUnOp30LfRdgD+BXtHcPaB3RTW07U54lI9IuAI7tltmWtZJpuupV1R+ALafZfhBw0BzKWwYsm21+SZIk3dJqPWGoquNpU4+S5EbgA1X1hklUTJIkSdLiNsmX+h0JLJ/g8SRJkiQtYhMbO1FVR07qWJIkSZIWv0k+2ZAkSZKkmxhsSJIkSeqFwYYkSZKkXhhsSJIkSeqFwYYkSZKkXhhsSJIkSeqFwYYkSZKkXhhsSJIkSeqFwYYkSZKkXhhsSJIkSerFBvNdAWlddactN+VVT/rz+a6GJElSb3yyIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSLmccPQAAIABJREFUemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSerHBfFdAWlddfOW1fOJ7P7vp89Mfcq95rI0kSdLk+WRDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBjkUuye5JKsnyaPEu6PCsH0g7q0irJyTPse+NU3mny7ZTk3UnOTHJpkj8mOS/JF5M8P8lthvKvn+TQJCuSXJTkuiTnJ/mfJMckeeqY85x2Gdpn2cC2J42p99Ju+yEj9pnNMrbdJUmS1nUbzHcFNO+uBx6ZZMeqOnvE9kOAdPlGXi9JXge8nha8fhf4IHAFcCdgd+AY4IXALl3+9YEvAHsBlwBfBH4DbATcF3g2sBPw+RHFnQMsm/NZwj8l+WpV3TBDvhOAlUNpuwOPBlYAy4e2DeeVJElSx2BDXwD2oQUVfzu4oQsKDga+D9wF2HZ45ySvBY4Efg3sX1X/PSLPU4AjBpKeRQs0fgg8uqouHcq/KbDrmPqurKqlszmxAf9HC2KeBxw9XcaqOoEWcAzWZykt2Fi+CmVLkiSts+xGpf+lPY04MMmGQ9ueTAsyRt6gJ1kCLAWuA540KtAAqKqppxhTduvWy4YDjS7/VVX1rdmfwozeCFwFvCHJZhM8riRJkqZhsCFowcQdgL2H0g+ldYf62Jj9DgY2BD5dVWdOV0BVXTvw8cJuvcPcq7pKzgP+GdgGeNUaKlOSJGmdZzcqAXwCOIoWXHwKIMm2wBOBD1TVFUlG7feIbv2NOZb3GeDVwAuSbAF8Fjitqs6Zxb5Lum5No5xVVceP2fZPwGHAEUneW1W/nWOdV0mS08Zs2mlNlC9JkjSfDDZEVV2Z5Djg8CRLqmolbXzD+kw/xuHO3fo3cyzvB0meA7wTeE63kOQi4GTg2Ko6cczud6MNRh/lc8DIYKMLmF4PvJfWreqQudRZkiRJc2c3Kk05mjbr1POTrAc8H/ifqvpeH4VV1SeAuwJPoN38f4F2Pe4DfD7JBzP6ccqKqsqYZZ8Zij0G+DFwUJI/m+DpjFVVO49agLPWRPmSJEnzyWBj8buxW0/3s5zaduO4DFV1OnA6bRzGE2lPEKaduQmY6op0q1mqZqOqrquqk6rqdVX1F8DtgWcAVwJ/xa3HkKyWbtrbV9Ge2LxtkseWJEnSrRlsLH5TszltPU2e23frS2Y41vtpgcN7gauBj8yQ/zvdes8Z8s1KVd3QPfF4R5f0mEkcd6iMLwLfAvZK8thJH1+SJEk3M9hY/M4GrgV2SDIu4HhYt/7hDMc6jvZUYTvgk1U1U3DyAdq0t/sluc90GYffID6Dy6d2m8M+c3EEUMDb8XdAkiSpN95oLXJVdQ1tUPQGwNuGxzkk2Y6bX9a3bIZjXU57H8a+wN/PouyVtPdsbAR8Mckuo/Il2Qv48sDnZyV5XDc2ZDjvNrRZsaANFp+4qvoB7anNA2gvGJQkSVIPnI1q7XAE8GDaeIuHJfkacBlt3MXewBbAW6tqxUwHqqrvzJRnKP//n2QD2gxR309yCnAq7f0cdwIeBdyrS5uyK/Ay4HdJvgP8skvfnvYiwU1oM0t9akSR0019C3DULJ7IAPwdsD9wz1nklSRJ0iow2FgLVNWFSXYFXkp7KnEQ7Yb9QmAF8J6q+lKP5b8hySeBvwb2oAU9G3flnwG8lVuO//hn4GfAY4H702akmsq/nNad67iqqhHFTTf1LbSnNzMGG1X16yRHAa+ZKa8kSZJWTUbfz0nqU5LTtt/xvg96y4c+e1Pa0x9yr3mskSRJ0mg777wzp59++und9P1z4pgNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb3YYL4rIK2rbrvZbXj6Q+4139WQJEnqjU82JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLzaY7wpI66orr7mOU87+3U2fd9txm3msjSRJ0uT5ZEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww21nJJdk9SSZZPk2dJl2flUPpdkrwjyY+TXJXk6iS/SrIiyZuS3GMo/7LuOFPLDUkuTfLzJCckeXGSrWdR5/27/ffrPi8dOu6oZfnA/iPPZ0Q5laTGtNfgcl2S85J8JsmjZqq/JEmSmg3muwJamJL8GbACuB3wI+CDwEXAHYGHAK8Ffgn8fMTunwPO6P69BfCnwCOBvYE3JXlZVS2bpvh9gauBLw+lrwCWj9ln5XTnswrOAZZ1/94U2Lmr1z5JnlFVn5xweZIkSWsdgw2NcxQt0FhaVUcOb0xyd2CjMfueMBxMJNkAeB7wTuADSa6tqo+NOO5GwJOBk6rqqqHNy6tq6VxPZBWtHC4ryWuANwP/BBhsSJIkzcBuVBpnt279zlEbq+oXVXXWbA9WVddX1fuBv+6S/iXJJiOyPgb4E+Czc6nsGvIf3XpJktvPa00kSZIWAYMNjXNht95hwsf9IK2L0ja0wGLY04DrgRMnXO6kXTffFZAkSVro7EalcT4OHAF8Psl7gG8BZ1TVZatz0Kq6Mcm3gbvRxn58cWpbkvVo4zpWVNVFI3bfPcnSMYf+SlX91+rUbRYO79ZnVtWls9khyWljNu00mSpJkiQtXAYbGufvaN2ZDgaWdksl+SnwFeBdVfWLVTz2ud36DkPpu9EGoI/rQvXobhnlEmCSwcaSgcBmU2AXYA/gMm4OOiRJkjQNgw2NVFXXAocl+QdgL2BX4EG0m+6XddueXlVfWIXDZ6qYofR9u7QTxux35BocIH434PVDaRcDj6mqM0bkH6mqdh6V3j3xeNCqV0+SJGnhc8zG2u/Gbj3dz3pq243DG6rq91X1war666p6KO3JwzHAJsCx3exRc3WXbv2HofR9ge9X1bmsnhnPueuyBbcOeKasqKpUVYCtgcOAzYATk2yzmvWTJElaJxhsrP2mxhZM9zK9qZmVLpnpYN1YisOBX9G6Qf3ZXCrT3eRPvRjvvwfSHwhsD3xmLscbY+qcb5ckY/LM6Zyr6mjglcB2wL+vfhUlSZLWfgYba7+zgWuBHaZ5e/fDuvUPZ3PAqroRuLL7OO5mfpyDgLsCv6UNOp+yb7de7Slvu8Hb59CeRNxvTLY5nXPnvcD/Avsmefiq11CSJGndYLCxlquqa4DjaeNz3jb8TX+S7YC/7T4uG0h/fZIlo46Z5C9psyldDJw5m3ok2SDJocC/0bouvaKr25R9gR9X1U9nc7xZWNat/ynJbYbqshVw5FC+GVXVDdw8juNNq1k/SZKktZ4DxNcNRwAPps0s9bAkX6PNqnQ32lSzWwBvraoVA/u8Alia5AfAqbTxFVvSBjU/jPYujBd0A8mH7TMQqGxGe5LxSODOtC5Oh1XVx6cyJ7kn7QnETDfw0019e0lVHTXw+c202aOeAPw0yZdo7w7Zpjvn2wMfAz40Q5nDPgOcATw6yROq6qtz3F+SJGmdYbCxDqiqC5PsCryU9gThINoA7wuBFcB7qupLQ7s9BXgibarZvYA70QKM39AGiL+rqn40psi9u2Wqu9UfgO8BXweOG/EOjakuVDON15hu6ttzgJuCjaq6NsljaeNLntktm9OCnTOAD3R1GTdAfKSqqiSvAz4P/CNgsCFJkjRG5nivJU1cklOAu1TVkvmuy5qS5LQd73O/Bx37mZNuStttRye5kiRJC8/OO+/M6aeffvq4Kf2n45gNzaskdwYeyvh3a0iSJGmRshuV5lVV/RaDXkmSpLWSN3mSJEmSemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXG8x3BaR11WYbb8huO24z39WQJEnqjU82JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2JEmSJPXCYEOSJElSLww2pHlyzXU3cNa5l3DWuZfMd1UkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqReLKthIUiOWa5OsTPLBJPeexTG+1u336yTrT5Nv2UAZb5gm34ED+ZYPbdt9VPqY4yzp8q6cIV8lqTHbtktybJLzBtrlqCS3nUX5R3fHvirJVrPIf2CS7yW5IsmlSZYnecqYvFNtedA0x1va5Vnafd45yR+TnDNdfZKc0O33ku7zVJtXkl8myZj9Nk9y2UDeJQPbNktyQJLjkpyV5Moklyc5NckRSTaaqX0kSZK0yIKNAUcOLP8OnAv8FfD9JA8ct1OSuwN7AgVsBzxxFmVdDxw8TWByaJdnXiW5B3AacDDwPeAdwC+AlwHfTbL1NPtuATyT1i6bAM+Zoay3A8uAOwNHAx8B7gecmOTFq3suAFV1GrAUuCvtZzyqHocAewNfBf51aPP1wBLgcWOKeCawBaN/do+kndMTgDOBdwPHAdsCbwe+lWTjWZ+MJEnSOmpRBhtVtXRgeUVVPZx2s7kZ8PJpdj0UCPDW7vNhsyjuC7TAZK/hDd2TlIcDJ86l/j35d+COwEurap+qek1VPYYWdOwIvGmafZ8NbN7l/SOtnUZKshtwBPBz4P5d+78I2Bm4CHj74FOC1fQW4DvAs5I8e6ge9+jqeyFwcFUNP+35OnAt48/lUOC3tABt2O9oAdedq+ovu7Y8HNgBOB3YDXjRqp2SJEnSumNRBhtjnNSt7zBqY5INgIOAy4A30G4yn5Rk2xmO+1HgakbftE6lHTPXyk5Sd+P9eGAl8G9Dm18PXAk8N8lmYw5xKHAjcBQtcLp/kl3H5H1Bt35TVV08lVhVU2XfhvZ0ZbVV1Y3Ac2k/s39L8qcA3VOmD9MCpEOr6rcjdr8Q+Aywd5JbXBNJ7g88BPgAI55sVNUZVfXRqvrjUPrlwD93H3dfjVOTJElaJ6xNwcZju/WpY7Y/FdgG+HhVXU3rBrQ+8LwZjnsJ8EngyUm2mUpMchta160VwE9XvdoTsUe3Pqm7Qb9Jd4P8n8CmwEOHd0zy57SnEt+oql/T2gXGP/V5TLf+yohtXx7Ks9q6IOYlwFbAh5KsB7wWeBhwbFV9dprdjwY2BA4cSj+U1mXsP1ahStd163nvOidJkrTQbTDfFVgVU4OIO38CPJjWnekLtD71o0zdPH+gWx9H+5b6+UneNHyTPuRoWmBxMPDmLm1fYOtu2yRtNXR+s7Fjtx4X9PyM9uRjB+AbQ9sO79ZT7fIVWjeiZyR5RVVdNpWxezKyLXDFmKcJP+vWO4ypxz7TdLHafUw6VfWhbvD5/rQA4QBaN66Xjdunsxz4P+AQuusiydSYlG9U1S/GjB+fzlRwOirYupUko7ppAew014IlSZIWm0UZbNC6Bg37MfCx7pv8W0hyN9pA4bOr6rsAVXVRkhOB/WgDgb88vN+UqvpOkrOAQ5K8pRsfcChwMfBp2piOSdmS0ec30z4Al47ZPpV+i1mduuDh2d32zwJU1fVJPkobl3EA8J7VLWfA3t2yKg6njZU4CLgBeE5VXTHdDlVVSY4B3pLkUVV1MvCXXf3mHCR2g9/3As4Ajp3r/pIkSeuaRdmNqqoytdD67e8K/B74aJJRA6EPoZ3rsqH0qc9jB0QPOBq4O/CYJPekdV36cFVdM/czmNY5g+c3vEy4rKkZmY4fOo9l3Xo27TIXB09zXkdOt2M3PmTqZ/vpqvqvWZa5jNb1aepcDgMuAE6YS8WTPI02puV3wH5Vdd0MuwBQVTuPWoCz5lK+JEnSYrQog41BVXVlVX0PeBptIPSrpgYSw02DiZ9HGwD94aHdp7oM/cXgeIwxPkSb3eiQbgmT70K1qqaeKGw5ZvtU+iVD6VNdy5YNJlbVmbQB9H+eZJcJlDMpVw+tZ1RVv6cNet8vycOARwAfHB78PZ0k+wDHA+cDu1fVL2ZfZUmSpHXXog82plTVJcDZtK5hDxrY9BTgLrRz/c3AS9yK9o33Nt0+0w4Ur6oLaF2N9u3yfre7KV8Izu7W48ZK3Ktb3zSmY2BGJmjv4aihttm523bTQPGqupL2TpPNk9x5NuUsEO+nvT/kE93nWQeJSfanTRDwe+DRVXX2DLtIkiSps1jHbIwz9abswSBqqvvMF2g3jMPWp40DeH6SN494X8Ogo2ldj+4AvHr1qjpR3+rWj0+y3uBg9+6FfQ8HrgIGux5NBRHLaYOtR3k27R0XrxwYH/FN2nS0e3HzoPIpTxzIs5B8DTgHuBtw8mwDhiQHAB+kBVh7+ERDkiRpbtaaYKPr6rI97WnFKV3an9Juii8G9h83vqIbg/EI2vS5X5ummG/RBjivR3tr9YJQVT9PchJtxqkX0d54PeVI2ssO39c9mZiakekA2kDrA6rqvFHH7ab3fQ7wLG5+GvBeWrDxd0lOmHrXRjfL1ItoXc2Gg5B5VVU3dmMu7gr8ZDb7JDmQNgj8HFqgcU6PVZQkSVorLcpgY2hq2M2A+3Dzt+qv7frpAzyf9uTiIzMM5D6GFmwcxjTBRvfU4/OrUOWdkiwbs+1XVfW6VTjmsL+mBVnvSrIn7aZ6V9pA9p8CfzeQ9xm0GZlOHBdodI6hBRuH0QUbVXVKkn8BXgn8T5JPARt1x7wd8JLu3RgLSlWdTnv794yS7EELNNajBZgHj5gi95KqOmqilZQkSVrLLMpgg1tODXsD8AfaIOB/raqvAXQvf5sahzHTG74/CbyT9rbpO1bV+ROu75249YvlpvwQWO1go3u6sQvt7eh7AU8Cfks7ryMH3/bNLN98XlUrkvwU2CXJA6vqjC79iCQ/oj3JOIw2+P504G1V9YXVPZcF4G7c3BVv3Fiec2izU0mSJGmMTD9EQVIfkpx2n/s94EGf/vJyAHbadtyrSSRJkubXzjvvzOmnn356N33/nKw1s1FJkiRJWlgMNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi8MNiRJkiT1wmBDkiRJUi82mO8KSOuqjTdcn5223Wq+qyFJktQbn2xIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxI8+S6G4rfXXLNfFdDkiSpNwYbkiRJknphsCFJkiSpFwYbkiRJknphsCFJkiSpFwYbkiRJknphsCFJkiSpFwYbkiRJknphsCFJkiSpFwYbkiRJknphsCFJkiSpFwYbkiRJknphsCFJkiSpFwYbkiRJknphsCFJkiSpFwYbWuskqSQ1Q56VXb4lY7Zvm+SNSb6b5IIk1yW5NMkPkvx7kt17qLokSdJaZYP5roC00CQ5DHgnsDFwFvBZ4HxgM2BH4K+AFyZ5e1X97bxVVJIkaYEz2JAGJPkr4H3AxcAzq+pzI/L8CXA4cNc1XD1JkqRFxWBD6iTZEjiq+/iMqvraqHxVdRnwtiT+/kiSJE3DMRvSzf4SuC1wyrhAY1BVXd9/lSRJkhYvv5nVWivJ0mk2bzUi7eHd+psTrMNpYzbtNKkyJEmSFiqDDa3NXj/H/Nt063OHNyTZCnj5cHpVLZ17tSRJktYNBhtaa1VVxm1LshK42xwOtxWjg5elM9Rh5zHlnwY8aA7lS5IkLTqO2ZBu9rtufZfhDVW1sqoytTDi6YckSZJuyWBDutl/dus957UWkiRJawmDDelmnwIuAXZLYsAhSZK0mgw2pE5VXcrNg8A/keQpo/Il2RTYcI1VTJIkaZFygLg0oKo+mOQ2wLuAE5P8hNa96nxgc2A74PHdv0+et4pKkiQtAgYb0pCqen+SLwEvBB4L7AdsAVwFrAQ+CnysqlbMWyUlSZIWAYMNrXWmm/J2IM+SGbb/Bvi7bpEkSdIqcMyGJEmSpF4YbEiSJEnqhcGGJEmSpF4YbEiSJEnqhcGGJEmSpF4YbEiSJEnqhcGGJEmSpF4YbEiSJEnqhcGGJEmSpF4YbEiSJEnqhcGGJEmSpF4YbEiSJEnqhcGGJEmSpF4YbEjzZMP1wzZbbTzf1ZAkSeqNwYYkSZKkXhhsSJIkSeqFwYYkSZKkXhhsSJIkSepFqmq+6yCtc5JcuMkmm9zu3ve+93xXRZIkaVo/+clPuPrqqy+qqq3nuq/BhjQPklwLrA/8cL7rskjs1K3PmtdaLB621+zZVnNje82ebTU3ttfcrOn2WgJcVlXbz3XHDSZfF0mzcCZAVe083xVZDJKcBrbXbNles2dbzY3tNXu21dzYXnOzmNrLMRuSJEmSemGwIUmSJKkXBhuSJEmSemGwIUmSJKkXBhuSJEmSeuHUt5IkSZJ64ZMNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JAkSZLUC4MNSZIkSb0w2JDmIMl2SY5Ncl6Sa5OsTHJUktvO8Ti36/Zb2R3nvO642/Vd9pq0unVOslmSA5Icl+SsJFcmuTzJqUmOSLLRmP1qmuW/JnuWkzOJn3GS5TOc/8Zj9rtPkk8kOT/JNUnOTnJkkk0md4aTM4Fra/cZ2mlq+dOh/RbdtZXkL5O8O8m3k1zW1fUjq3isObf7Yrq2JtFWSbZOckiSzyb5vyRXJ7k0yXeSPD/Jre69kiyZ4do6fnJnOTmTura662jcuf9umv12S/KlJBd17fw/SV6eZP3VO7N+TOj6OmgWf7duGNpnXq+vDfo8uLQ2SXIP4BTgjsDngLOAhwAvA/ZK8vCqunAWx9m6O84OwDeB44GdgIOBJyd5WFX9oo+y16QJ1fmRwEeAi4BvAScAtwWeCrwdeFqSPavqmhH7ngMsG5H+m7mfTf96+BkfOSb9+hFl70q7FjcEPgX8GngM8Dpgz66Nr51D2b2aUFutZHwb3Q94GnBmVf16xPZFdW0Bfw88ALiCVsedVuUgq9Lui+3aYjJttT/wHuC3tL9bvwLuRLumjgGemGT/Gv1W5R/S/s4NO3MV6rEmTOTa6lwKHDUi/YpRmZPsDXwauAb4OO3/ib8A3gE8nPZzWGgm0V5nMP5v1yNpv19fHrN9fq6vqnJxcZnFAnwVKOAlQ+n/0qW/d5bHeV+X/5+H0l/apX+lr7IXW3sBDwQOADYaSt8COK07zhEj9itg+Xy3wTxdX8vbn/ZZl7s+8OOujKcOpK9Huzks4DXz3T59tNU0x/9Yd5yXriXX1h7AvYAAu3fn8JG+232RXlur3Va0m72/ANYbSt+GFngUsN/QtiVd+rL5boN5urZWAivnkP9PgPOBa4FdBtI3pgXEBTxzvtunr/aa5vjfHf59WwjX17w3vIvLYliAe3S/qL8c8R/IFrRvKa4ENpvhOJsDV3X5txjatl73B7eAu0+67MXYXjOU8eyujBNHbFtUN4STbC/mHmw8pit7xYhtd++2rQQy3+20Jq4t4Pa0b0qvArZa7NfWiPqv0g3OqrT7Yru2JtVWMxzztd0x3z2UvoRFGGxMqr2Ye7DxvK6sD47YNva6W0jLpK8v2hPZoj0xWX9o27xeX47ZkGZnj259UlXdOLihqi4H/hPYFHjoDMd5KLAJ8J/dfoPHuZH2zeFgeZMse01aE3W+rlvfqltQZ6skz0vy2iQvSrKQ2mfYxNsryTOSvCbJK5M8McltxmR9TLf+yvCGat35fgrcjXZzuBD0fW0dCNwG+GRVXTImz2K6tiZlVdp9sV1ba8JMf7fukuTw7to6PMn911TFFoDbJHlOd+4vS7LHNGMvxl5bwMm0Lwt2m+bv3trosG79H1V1w5g883J9OWZDmp0du/VPx2z/GfB42jiMb6zmceiOM+my16Q1UefndetR/9lA6xf7H4MJSX4IPLeqfrSKZfalj/YaHvB3fpIXVdWnVqHsHbrl57Msu099X1uHduv3TZNnMV1bk7Iq7b7Yrq1eJdkA+Kvu47i/W4/rlsH9lgMHVtWv+qvdgrAN8OGhtF8mObiqVgylj722qur6JL8E7ksLZH8y8ZouMN1kC88BbqCNCxpnXq4vn2xIs7Nlt750zPap9K16OM6kyl6Teq1zkhcDe9EGyh07Isu/0AYI3oHWxePBtD7iDwC+mWTbVSm3R5Nsr8/R+otvR3uKthPw5m7fjyfZq8ey14Te6pvk0bSbmDOr6pQx2RbbtTUp68rfrj69Bfgz4EtV9dWhbVcBbwR2pk2CcVvg0bQB5rsD30iy2Zqr6hr3AWBPWsCxGa1L0Pto3X++nOQBQ/m9tm7p6bRz/UqNntRiXq8vgw1Ji0qSp9FmLPkdbZDldcN5quqIqjqlqi6oqiuq6tSq2p82c8ntgb9Zs7Vec6rqHVX1hao6t6quqaqzq+q1wBG0v/lvnucqLmRT3RDePy7DunxtadUleSntd/As4LnD26vq/Kp6XVWdXlWXdMvJtKdF/w3cEzhkjVZ6DaqqI6vqm1X1+6q6qqrOrKoX0IL7TYCdejeFAAALxElEQVSl81vDBW/qb9fIJ7LzfX0ZbEizM/UtyZZjtk+lj+vjvTrHmVTZa1IvdU6yD6170PnA7jU0RfAsvLdbP2qO+/VtTfyMj6H1E39gki3WcNmT1Ne1dTtgP+Bqbt2VYzYW6rU1KevK366J657EvpM2M9ceVXXRbPetquu5uVvM2nptTWfc75XXVifJfYHdaAPDvzSXfdfU9WWwIc3O2d16hzHb79Wtx/VNXp3jTKrsNWnidU6yP/BJ4PfAo6vq7Bl2GeUP3XqhdUfo/Wdc7V0kU5MSDJ7/Yru++qrv1MDwT0wzMHw6C/XampR15W/XRCV5OfBu2nsM9qiqsS+om8bafm1NZ9y5j722urEx29O+XJnrF1KL0WwGhk+n9+vLYEOanW9168dn6O2v3bfED6f1iZzpDcL/Rfvm9OFD3y7THffxQ+VNsuw1aaJ1TnIA7d0H59ECjZ/NsMs4UzPlLLT/gHr/GSfZkdZP93LggoFN3+zWw2M5SHJ32n/m57Bw2qyvtpoaGD62C9UMFuq1NSmr0u6L7dqaqCSvpr1g7gxaoHH+Kh5qbb+2pjPu3MdeW7Rv6DcFTqmF9cLIiUuyMa1b3g0MTVoxB71fXwYb0ixU1c+Bk2iD1V40tPlI2jcCH66qK6cSk+yU5BZvB62qK2hdNDbj1n1QX9wd/6uD3YNWpez5Nqn26tIPBD5EexnWo2bqOpXk/kk2HJUOvKn7+JHZn03/JtVeSbbvugMxlH4H2gBMgOO7R+dTVtBma3lUkqcO7LMe8Nbu43urm6x9vk3y2hrY/kjg3kw/MHxRXltzlWTDrr3uMZi+in+HFtW1NVfj2qrb9g+0AeGnAXtW1QW3OsAt8z9oOIjr0vcEXtF9XCuvrST3HjU4OckS4F+7j8Pn/inalybPTLLLwD4bA//YfXzPhKo+L6a7vgbsT/sS6ctjBoZPHWter68s0t9xaY3rfuFPAe5Im/HnJ8CutPnnfwrsVlUXDuRvbwCrytBxtu6OswPt25nv0W509qaNRdit+499lcteCCbRXkn2AL5O+2LkWGDUH9NLquqogX2W0WZj+naX/1rajEx70d5ofDRw+EK7wZlQex1E6+P8Hdq3VBcBdwWeROvDfCrwuOFuQkl2pV2LG9L+E/8VbWaYXWjvT9hzIX1DOKnfxYHtH6ZNG/nSqnr3NOUuY3FeW/sA+3QftwGeQLs+vt2lXVBVf9PlXUJ7cd85VbVk6Dhz/ju0CK+t1W6r7guSZbRvm9/N6BmTVlbVsoF9ltO6lZ1C63sPcH9ufp/EP1TVP7LATKi9ltIGz59Me9J1Oe0lkk+mvRH8S8C+VfXHEWV/ivYSzuNpf++eSptR7lPA09fW38WB430beATtjeEnTlPucubz+qoF8BZFF5fFsgB/SvuG+LfAH2l/GI8CbjsibzHmTc7A7WgDBs/pjvNb2s30dpMoe6Esq9tewEFT6dMsK4f22Qf4DPB/wGUD7Xsi7Q/yvLdLj+11P9pNzo+AC2kvELuI9h/ZS4CNpin7PrQxMRfQbqJ/SvvGepP5bpc+2mpg221pXRtHvjF8bbi2aE9RZ/U7xM1vGl455lhz/ju0mK6tSbTVLI5RDL2FHng+8AXam7Sv6NrpV8DHgUfOd7v03F6PpnWTPYs2oPs62jiCr9HeSzL2DfO07ntfAi7ufo9/RPumfv0+znchtNfA9nt323890/nO9/Xlkw1JkiRJvXDMhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIkiRJ6oXBhiRJkqReGGxIktY5SZYkqSTL5rsukrQ2M9iQJGktlWRZF1Qtme+6SFo3bTDfFZAkaR6cC9wbuHS+KyJJazODDUnSOqeqrgPOmu96SNLazm5UkqR1zqgxGwNdjrZP8uIkP05yTZKVSV6bJF2+/ZN8L8mVSc5P8q9JNhlRRiVZnuQuST7c5b06yWlJnj2mXusleUGS7ye5oivj+0lemORW/2cPlLFNkmOSnJvkhiQHJSngwC7rL7u8lWTlwP47J3lnkh8muag7358l+ecktx1R3kHdMQ5KskdX9uVJLkvyxST3HnNemyZ5dZJTu/xXJPlJkncludOIvP9fkjO6878iyXeTPGvUsSUtbD7ZkCTplt4O7A6cCJwEPBV4E7BRkouAtwAnAN8GHge8CFgfeOGIY90WOAW4BPgAsBXwdOCjSbatqrcN5f8w8Gzg18AxQAH7Av8OPAI4YEQZtwP+C7gC+AxwI/B74EhgH+ABwDu7OjCwBji0O/4K4Ou0LyF3Bl4JPDHJrlV1+YgynwLsDXwZeC9wH+BJwIOT3KeqLpjK2AUt3+rqcTZwLPBH4B7AwV2df9/l3Qr4JvDnwOld3vWAJwDHJblvVf39iPpIWqiqysXFxcXFZZ1agCW0G/llA2nLurSVwLYD6VsBFwBXAn8A7j2w7TbAj4FrgTsOlVHd8glgvYH07YGLaDfcdx9If1aX/3Rg84H0zYBTu23PHlPGh4ANRpzn1DktGdMOdwPWH5H+/G6/Vw+lH9SlXw/sObTtzd22Vw2lH9elv2ewHbptmwNbjqjv8DE2Br5CC6QeON/Xj4uLy+wXu1FJknRLb6yqc6c+VNUlwOeBTYH3VNVPBrZdC3wc2Ig24HzYDbQb9hsH9vkl8C5gQ+C5A3mf161fU1VXDOS/Enh19/GQEWX8Efibqrp+1md487HPqaobRmw6FriM9kRhlOOr6htDae/v1g+ZSkhyR+AZwG+7Ot44uENVXVFVl3Z5twaeA5xaVf80lO8aWhuE9uRH0iJhNypJkm7p1BFp53Xr00ZsmwpMthux7VddcDFsOfB6WnehKQ+ifXO/fET+FbTA5c9HbFtZVeePSJ9Rkg2Bw4Fn0rpCbcktx3NuO2bXUW306249ONbjwd3xTu6Cpuk8mNYdrZIsHbF9w249clyIpIXJYEOSpFsaNR3u9bPYtuGIbb8fU8bvuvWWA2lbAhdV1R+HM1fV9UkuAO44zbFWxcdpYzZ+AXyuO9a13baX07qJjXLJcEJXR2gBw5StuvW5w/lH2LpbP7hbxtl8FseStEAYbEiS1J87jUnfplsPBi+XArdLsmG1qXlvkmQD4Pa0rk3DalUqlmQXWqDxdeCJg92wupmvXrUqxx0yFZSMe0IyaKot3lFVr5xA2ZIWAMdsSJLUn7uOeXv37t36BwNpP6D9v/yoEfkfRXticPocy58aj7H+iG337NafHzHe4yHArabzXQXfo3UNe1SSzWaZ95ETKFfSAmGwIUlSf9YH3jr4jowk2wMvpXW/+shA3mO79ZuTbDqQf1PadLsA/zHH8i/s1ncdsW1lt959MLEb1P1vcyxnpKr6A3A8cGfg7cPvCkmyeZItu7znAx8FdknyD0luFSAluUfXfpIWCbtRSZLUn/8BdgVOS3ISN79nYyva9K4/n8pYVccl2bvb/r9JTqB1kdqHNl3ux6vqo3Ms/xvA3wJHJ/k0cDlwSVX9K/B94D+BpyU5BfgOrdvXE2nvwzhv9CHn7MXAnwEvAHZP8lXaDFrb02a7eio3D4p/MXAv4A3Ac5N8hzbu5S60geEPpk0RPGrQvaQFyCcbkiT152JgN+B/aS+wO5B2o3xA3fqFftBupF9EeyJxOO0G/WLaTficp3ytqq8CRwDX0QZ8vxH4m27bDbQb/ffQbuZfSntx4DG0IOC6EYecs6qaaoO/7455GO0FiPelPc358UDey4BHAy+hvdtkP9oLBvegBUqvAL42iXpJWjNStUrjyiRJ0jSSFLCiqnaf77pI0nzxyYYkSZKkXhhsSJIkSeqFwYYkSZKkXjhmQ5IkSVIvfLIhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRcGG5IkSZJ6YbAhSZIkqRf/D2jpdy3xBUJYAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1_ZeGsM3nqIX"
      },
      "source": [
        "zg는 금 가격인데 왜 ?  금리 예측에 도움이 되는걸까 ,,,"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qfrcVNcypQLV"
      },
      "source": [
        "ZG를 빼도 결과가 너무 좋다"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mh0MLZgZo35D",
        "outputId": "6a992820-3807-433d-b1b4-ccb98fb12552"
      },
      "source": [
        "df_x_m2 = df_x_m2.drop(\"ZI\", axis=1)\n",
        "\n",
        "x_ = df_x_m2.drop('DFEDTAR', axis=1)\n",
        "y_ = df_x_m2['DFEDTAR']\n",
        "x_, y_\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)\n",
        "x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)\n",
        "\n",
        "x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape\n",
        "\n",
        "pipe_ = Pipeline([\n",
        "    ('preprocessing', make_pipeline(OrdinalEncoder())),\n",
        "    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) \n",
        "])\n",
        "\n",
        "pipe_.fit(x_train_, y_train_)\n",
        "print('훈련 R^2: ', pipe_.score(x_train_, y_train_))\n",
        "print('검증 R^2: ', pipe_.score(x_val_, y_val_))\n",
        "print('TEST R^2: ', pipe_.score(x_test_, y_test_))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))"
      ],
      "execution_count": 222,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 R^2:  0.9998356901989786\n",
            "검증 R^2:  0.9956481227418897\n",
            "TEST R^2:  0.9973068731784086\n",
            "\n",
            "훈련 MAE:  0.005934574684574683\n",
            "검증 MAE:  0.018995726495726496\n",
            "TEST MAE:  0.019023794996949357\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "id": "jWV-xWJbpHlZ",
        "outputId": "e2140479-9adc-4cfe-a8cd-732c6bd66088"
      },
      "source": [
        "# permuter 정의\n",
        "permuter = PermutationImportance(\n",
        "    pipe_.named_steps['rf'], # model\n",
        "    scoring='r2', # metric\n",
        "    n_iter=5, # 다른 random seed를 사용하여 5번 반복\n",
        "    random_state=20\n",
        ")\n",
        "\n",
        "# permuter 계산은 preprocessing 된 X_val을 사용합니다.\n",
        "x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)\n",
        "\n",
        "# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다\n",
        "permuter.fit(x_val_transformed, y_val_);\n",
        "\n",
        "feature_names = x_val_.columns.tolist()\n",
        "pd.Series(permuter.feature_importances_, feature_names).sort_values()\n",
        "\n",
        "# 특성별 score 확인\n",
        "eli5.show_weights(\n",
        "    permuter, \n",
        "    top=None, # top n 지정 가능, None 일 경우 모든 특성 \n",
        "    feature_names=feature_names # list 형식으로 넣어야 합니다\n",
        ")"
      ],
      "execution_count": 223,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <style>\n",
              "    table.eli5-weights tr:hover {\n",
              "        filter: brightness(85%);\n",
              "    }\n",
              "</style>\n",
              "\n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
              "    <thead>\n",
              "    <tr style=\"border: none;\">\n",
              "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
              "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
              "    </tr>\n",
              "    </thead>\n",
              "    <tbody>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.5904\n",
              "                \n",
              "                    &plusmn; 0.0446\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HSN1F\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 85.42%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.3759\n",
              "                \n",
              "                    &plusmn; 0.0539\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/KRW\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 86.43%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.3393\n",
              "                \n",
              "                    &plusmn; 0.0281\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UMCSENT\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 90.90%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.1916\n",
              "                \n",
              "                    &plusmn; 0.0237\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/EUR\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 91.33%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.1790\n",
              "                \n",
              "                    &plusmn; 0.0219\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 94.96%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0824\n",
              "                \n",
              "                    &plusmn; 0.0064\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ICSA\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 98.82%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0103\n",
              "                \n",
              "                    &plusmn; 0.0007\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.17%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0063\n",
              "                \n",
              "                    &plusmn; 0.0005\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NASDAQCOM\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 99.39%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0040\n",
              "                \n",
              "                    &plusmn; 0.0004\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                BAMLH0A0HYM2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "    \n",
              "    </tbody>\n",
              "</table>\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 223
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJcMKuoHplv4"
      },
      "source": [
        "df_x_m2 = df_x_m2.drop(\"HSN1F\", axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pSbgLOf0paca",
        "outputId": "dbe56188-a95e-414c-818d-f945ca3d2939"
      },
      "source": [
        "\n",
        "\n",
        "x_ = df_x_m2.drop('DFEDTAR', axis=1)\n",
        "y_ = df_x_m2['DFEDTAR']\n",
        "x_, y_\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)\n",
        "x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)\n",
        "\n",
        "x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape\n",
        "\n",
        "pipe_ = Pipeline([\n",
        "    ('preprocessing', make_pipeline(OrdinalEncoder())),\n",
        "    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) \n",
        "])\n",
        "\n",
        "pipe_.fit(x_train_, y_train_)\n",
        "print('훈련 R^2: ', pipe_.score(x_train_, y_train_))\n",
        "print('검증 R^2: ', pipe_.score(x_val_, y_val_))\n",
        "print('TEST R^2: ', pipe_.score(x_test_, y_test_))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))\n",
        "print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))\n",
        "print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))\n",
        "\n",
        "print('\\n훈련 MAE: ', mean_squared_error(pipe_.predict(x_train_), y_train_))\n",
        "print('검증 MAE: ', mean_squared_error(pipe_.predict(x_val_), y_val_))\n",
        "print('TEST MAE: ', mean_squared_error(pipe_.predict(x_test_), y_test_))"
      ],
      "execution_count": 226,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 R^2:  0.9997811585453342\n",
            "검증 R^2:  0.9958233168920836\n",
            "TEST R^2:  0.9964944273965559\n",
            "\n",
            "훈련 MAE:  0.007042124542124543\n",
            "검증 MAE:  0.02043956043956044\n",
            "TEST MAE:  0.025167785234899327\n",
            "\n",
            "훈련 MAE:  0.0008262489824989828\n",
            "검증 MAE:  0.014813049450549452\n",
            "TEST MAE:  0.012791496339231235\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "id": "wMlWRKjFpfAx",
        "outputId": "5d24a7bb-e86d-471f-e0bc-eed1be2f4bd8"
      },
      "source": [
        "# permuter 정의\n",
        "permuter = PermutationImportance(\n",
        "    pipe_.named_steps['rf'], # model\n",
        "    scoring='neg_mean_absolute_error', # metric\n",
        "    n_iter=5, # 다른 random seed를 사용하여 5번 반복\n",
        "    random_state=20\n",
        ")\n",
        "\n",
        "# permuter 계산은 preprocessing 된 X_val을 사용합니다.\n",
        "x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)\n",
        "\n",
        "# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다\n",
        "permuter.fit(x_val_transformed, y_val_);\n",
        "\n",
        "feature_names = x_val_.columns.tolist()\n",
        "pd.Series(permuter.feature_importances_, feature_names).sort_values()\n",
        "\n",
        "# 특성별 score 확인\n",
        "eli5.show_weights(\n",
        "    permuter, \n",
        "    top=None, # top n 지정 가능, None 일 경우 모든 특성 \n",
        "    feature_names=feature_names # list 형식으로 넣어야 합니다\n",
        ")"
      ],
      "execution_count": 227,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <style>\n",
              "    table.eli5-weights tr:hover {\n",
              "        filter: brightness(85%);\n",
              "    }\n",
              "</style>\n",
              "\n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
              "    <thead>\n",
              "    <tr style=\"border: none;\">\n",
              "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
              "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
              "    </tr>\n",
              "    </thead>\n",
              "    <tbody>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.6821\n",
              "                \n",
              "                    &plusmn; 0.0437\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                HG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 81.43%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.6135\n",
              "                \n",
              "                    &plusmn; 0.0249\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/KRW\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 84.43%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.4769\n",
              "                \n",
              "                    &plusmn; 0.0252\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                USD/EUR\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 85.61%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.4263\n",
              "                \n",
              "                    &plusmn; 0.0176\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                UMCSENT\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 86.44%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.3917\n",
              "                \n",
              "                    &plusmn; 0.0139\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NG\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 86.55%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.3871\n",
              "                \n",
              "                    &plusmn; 0.0146\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                NASDAQCOM\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 91.86%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.1888\n",
              "                \n",
              "                    &plusmn; 0.0089\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                ICSA\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "        <tr style=\"background-color: hsl(120, 100.00%, 95.39%); border: none;\">\n",
              "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
              "                0.0837\n",
              "                \n",
              "                    &plusmn; 0.0110\n",
              "                \n",
              "            </td>\n",
              "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
              "                BAMLH0A0HYM2\n",
              "            </td>\n",
              "        </tr>\n",
              "    \n",
              "    \n",
              "    </tbody>\n",
              "</table>\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "    \n",
              "\n",
              "\n",
              "\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 227
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_uqrHG4qLJv"
      },
      "source": [
        "원자재 가격이 이렇게 큰 영향을 미친다고 ??"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9SQDKSWDgrQN"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oOu_py2kVMW0"
      },
      "source": [
        "# dpi(dots per inch) 수치를 조정해 이미지 화질을 조정 \n",
        "import matplotlib.pyplot as plt\n",
        "plt.rcParams['figure.dpi'] = 144"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wxc0XDxMVI39"
      },
      "source": [
        "!pip install pdpbox\n",
        "from pdpbox.pdp import pdp_isolate, pdp_plot"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-kkLgAnVqDg"
      },
      "source": [
        "feature = 'DFEDTAR'"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lNEzH91OQyIX",
        "outputId": "4bd49709-a537-459f-b207-7bad272df2c4"
      },
      "source": [
        "!pip install shap"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting shap\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b9/f4/c5b95cddae15be80f8e58b25edceca105aa83c0b8c86a1edad24a6af80d3/shap-0.39.0.tar.gz (356kB)\n",
            "\r\u001b[K     |█                               | 10kB 24.8MB/s eta 0:00:01\r\u001b[K     |█▉                              | 20kB 15.8MB/s eta 0:00:01\r\u001b[K     |██▊                             | 30kB 14.1MB/s eta 0:00:01\r\u001b[K     |███▊                            | 40kB 12.4MB/s eta 0:00:01\r\u001b[K     |████▋                           | 51kB 8.5MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 61kB 9.9MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 71kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 81kB 10.0MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 92kB 9.5MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 102kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 112kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████████                     | 122kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████                    | 133kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 143kB 8.2MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 153kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 163kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 174kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 184kB 8.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 194kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 204kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 215kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 225kB 8.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 235kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 245kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 256kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 266kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 276kB 8.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 286kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 296kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 307kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 317kB 8.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 327kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 337kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 348kB 8.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 358kB 8.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from shap) (1.19.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from shap) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from shap) (0.22.2.post1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from shap) (1.1.5)\n",
            "Requirement already satisfied: tqdm>4.25.0 in /usr/local/lib/python3.7/dist-packages (from shap) (4.41.1)\n",
            "Collecting slicer==0.0.7\n",
            "  Downloading https://files.pythonhosted.org/packages/78/c2/b3f55dfdb8af9812fdb9baf70cacf3b9e82e505b2bd4324d588888b81202/slicer-0.0.7-py3-none-any.whl\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.7/dist-packages (from shap) (0.51.2)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from shap) (1.3.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->shap) (1.0.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->shap) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->shap) (2.8.1)\n",
            "Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba->shap) (0.34.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba->shap) (57.0.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->shap) (1.15.0)\n",
            "Building wheels for collected packages: shap\n",
            "  Building wheel for shap (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for shap: filename=shap-0.39.0-cp37-cp37m-linux_x86_64.whl size=491627 sha256=084c2c08b760af2ebfcd11192bfd738f2bbe54096291961edfb9f7affd35d353\n",
            "  Stored in directory: /root/.cache/pip/wheels/15/27/f5/a8ab9da52fd159aae6477b5ede6eaaec69fd130fa0fa59f283\n",
            "Successfully built shap\n",
            "Installing collected packages: slicer, shap\n",
            "Successfully installed shap-0.39.0 slicer-0.0.7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "iGINmoRNrHCt",
        "outputId": "997eaf16-3634-4ffd-a266-f51464b39dac"
      },
      "source": [
        "row = x_test_.iloc[[1]]  # 중첩 brackets을 사용하면 결과물이 DataFrame입니다\n",
        "row"
      ],
      "execution_count": 236,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NASDAQCOM</th>\n",
              "      <th>BAMLH0A0HYM2</th>\n",
              "      <th>UMCSENT</th>\n",
              "      <th>ICSA</th>\n",
              "      <th>NG</th>\n",
              "      <th>HG</th>\n",
              "      <th>USD/KRW</th>\n",
              "      <th>USD/EUR</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6031</th>\n",
              "      <td>4997.46</td>\n",
              "      <td>5.13</td>\n",
              "      <td>93.1</td>\n",
              "      <td>292000.0</td>\n",
              "      <td>2.716</td>\n",
              "      <td>2.3805</td>\n",
              "      <td>1134.8</td>\n",
              "      <td>0.9081</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      NASDAQCOM  BAMLH0A0HYM2  UMCSENT  ...      HG  USD/KRW  USD/EUR\n",
              "6031    4997.46          5.13     93.1  ...  2.3805   1134.8   0.9081\n",
              "\n",
              "[1 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 236
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nOCNq-Weq_Do",
        "outputId": "b9a2b1be-6815-4d8e-a14d-bb76e15dae65"
      },
      "source": [
        "y_test_.iloc[[1]]"
      ],
      "execution_count": 232,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6031    0.25\n",
              "Name: DFEDTAR, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 232
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eU3qwshqrCif",
        "outputId": "ff32c720-e5f3-439f-817d-18105f9224da"
      },
      "source": [
        "# 모델 예측값\n",
        "pipe_.predict(row)"
      ],
      "execution_count": 234,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.2575])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 234
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lf9A9Hq8UEfv",
        "outputId": "ccfd7fd8-1e12-451c-c3a2-41b088ade2b8"
      },
      "source": [
        "explainer"
      ],
      "execution_count": 213,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<shap.explainers._tree.Tree at 0x7ff3d7262950>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 213
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gyYKngcFn9K5"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}