FinRAD_Sentence_FinBERTembedding_Extraction_Modeling_13K.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FinRAD_Sentence_FinBERTembedding_Extraction_Modeling_13K.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "SKyArqps4MRr"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "QMvrAjQijsTY"
      },
      "outputs": [],
      "source": [
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "ZR2o6dYY4BFH"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import f1_score"
      ],
      "metadata": {
        "id": "zTLDGE8I6Jc2"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle"
      ],
      "metadata": {
        "id": "pNiXxyUsANUs"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('term_unique_13K_all_in_one_NEW_CLEANED_FINAL_12_columns.csv')"
      ],
      "metadata": {
        "id": "eGMhhSsIkBRp"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (Optional) Embedding creation"
      ],
      "metadata": {
        "id": "SKyArqps4MRr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentence_transformers"
      ],
      "metadata": {
        "id": "Wcv8pGrKkhgE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer"
      ],
      "metadata": {
        "id": "LvNGvKqckPTR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 868
        },
        "id": "wM7xUXH8kQcw",
        "outputId": "0253f9e6-f1b9-4fb9-aa4e-ebef0c262254"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-930dcf97-fa71-4e05-8397-1cbccc1ad73b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>terms</th>\n",
              "      <th>definitions</th>\n",
              "      <th>source</th>\n",
              "      <th>assigned_readability</th>\n",
              "      <th>flesch_reading_ease</th>\n",
              "      <th>flesch_kincaid_grade</th>\n",
              "      <th>smog_index</th>\n",
              "      <th>coleman_liau_index</th>\n",
              "      <th>automated_readability_index</th>\n",
              "      <th>dale_chall_readability_score</th>\n",
              "      <th>linsear_write_formula</th>\n",
              "      <th>gunning_fog</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>adjusted present value (apv)</td>\n",
              "      <td>net present value of an asset if financed sole...</td>\n",
              "      <td>prin</td>\n",
              "      <td>0</td>\n",
              "      <td>59.64</td>\n",
              "      <td>9.9</td>\n",
              "      <td>0.0</td>\n",
              "      <td>9.40</td>\n",
              "      <td>10.5</td>\n",
              "      <td>10.16</td>\n",
              "      <td>12.000000</td>\n",
              "      <td>12.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>agency costs</td>\n",
              "      <td>costs that arise when an agent (e.g., a manage...</td>\n",
              "      <td>prin</td>\n",
              "      <td>0</td>\n",
              "      <td>66.07</td>\n",
              "      <td>9.5</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.78</td>\n",
              "      <td>12.3</td>\n",
              "      <td>10.47</td>\n",
              "      <td>15.000000</td>\n",
              "      <td>14.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>annual percentage rate (apr)</td>\n",
              "      <td>the interest rate per period (e.g., per month)...</td>\n",
              "      <td>prin</td>\n",
              "      <td>0</td>\n",
              "      <td>62.68</td>\n",
              "      <td>8.7</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6.67</td>\n",
              "      <td>8.4</td>\n",
              "      <td>10.98</td>\n",
              "      <td>12.500000</td>\n",
              "      <td>16.21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>annuity</td>\n",
              "      <td>investment that produces a level stream of cas...</td>\n",
              "      <td>prin</td>\n",
              "      <td>0</td>\n",
              "      <td>56.25</td>\n",
              "      <td>9.1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>9.56</td>\n",
              "      <td>8.7</td>\n",
              "      <td>9.64</td>\n",
              "      <td>10.500000</td>\n",
              "      <td>14.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>annuity due</td>\n",
              "      <td>annuity whose payments occur at the start of e...</td>\n",
              "      <td>prin</td>\n",
              "      <td>0</td>\n",
              "      <td>61.33</td>\n",
              "      <td>7.2</td>\n",
              "      <td>0.0</td>\n",
              "      <td>8.50</td>\n",
              "      <td>6.2</td>\n",
              "      <td>10.45</td>\n",
              "      <td>6.000000</td>\n",
              "      <td>12.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13107</th>\n",
              "      <td>ZERO COUPON SWAP</td>\n",
              "      <td>An OVER-THE-COUNTER SWAP involvingtheexchange ...</td>\n",
              "      <td>palgrave</td>\n",
              "      <td>0</td>\n",
              "      <td>35.78</td>\n",
              "      <td>12.9</td>\n",
              "      <td>0.0</td>\n",
              "      <td>13.46</td>\n",
              "      <td>13.4</td>\n",
              "      <td>10.10</td>\n",
              "      <td>13.250000</td>\n",
              "      <td>12.81</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13108</th>\n",
              "      <td>ZERO COUPON YIELD CURVE</td>\n",
              "      <td>A YIELD CURVE representing DISCOUNTRATES acros...</td>\n",
              "      <td>palgrave</td>\n",
              "      <td>0</td>\n",
              "      <td>52.19</td>\n",
              "      <td>10.7</td>\n",
              "      <td>14.1</td>\n",
              "      <td>15.66</td>\n",
              "      <td>15.5</td>\n",
              "      <td>10.95</td>\n",
              "      <td>13.166667</td>\n",
              "      <td>15.32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13109</th>\n",
              "      <td>ZERO MINUS TICK</td>\n",
              "      <td>Sale of a SECURITY at the same price as the la...</td>\n",
              "      <td>palgrave</td>\n",
              "      <td>0</td>\n",
              "      <td>66.74</td>\n",
              "      <td>7.2</td>\n",
              "      <td>10.5</td>\n",
              "      <td>6.83</td>\n",
              "      <td>6.2</td>\n",
              "      <td>9.14</td>\n",
              "      <td>7.166667</td>\n",
              "      <td>9.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13110</th>\n",
              "      <td>ZERO PLUS TICK</td>\n",
              "      <td>Sale of a SECURITY at the same price as the la...</td>\n",
              "      <td>palgrave</td>\n",
              "      <td>0</td>\n",
              "      <td>66.74</td>\n",
              "      <td>7.2</td>\n",
              "      <td>10.5</td>\n",
              "      <td>6.54</td>\n",
              "      <td>6.0</td>\n",
              "      <td>8.73</td>\n",
              "      <td>7.166667</td>\n",
              "      <td>9.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13111</th>\n",
              "      <td>ZOMBIE</td>\n",
              "      <td>A company that is in technical INSOLVENCY but ...</td>\n",
              "      <td>palgrave</td>\n",
              "      <td>0</td>\n",
              "      <td>-6.02</td>\n",
              "      <td>18.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>20.12</td>\n",
              "      <td>19.2</td>\n",
              "      <td>12.42</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>22.76</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13112 rows × 12 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-930dcf97-fa71-4e05-8397-1cbccc1ad73b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-930dcf97-fa71-4e05-8397-1cbccc1ad73b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-930dcf97-fa71-4e05-8397-1cbccc1ad73b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                              terms  ... gunning_fog\n",
              "0      adjusted present value (apv)  ...       12.00\n",
              "1                      agency costs  ...       14.25\n",
              "2      annual percentage rate (apr)  ...       16.21\n",
              "3                           annuity  ...       14.00\n",
              "4                       annuity due  ...       12.00\n",
              "...                             ...  ...         ...\n",
              "13107              ZERO COUPON SWAP  ...       12.81\n",
              "13108       ZERO COUPON YIELD CURVE  ...       15.32\n",
              "13109               ZERO MINUS TICK  ...        9.30\n",
              "13110                ZERO PLUS TICK  ...        9.30\n",
              "13111                        ZOMBIE  ...       22.76\n",
              "\n",
              "[13112 rows x 12 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = SentenceTransformer('ProsusAI/finbert')"
      ],
      "metadata": {
        "id": "jZpyIvlikc43"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corpus = list(data['definitions'].fillna('').values)\n",
        "finbert_embed = model.encode(corpus)"
      ],
      "metadata": {
        "id": "pnK21hLakfUC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "finbert_df = pd.DataFrame(finbert_embed)"
      ],
      "metadata": {
        "id": "hoU0LrnWkt3p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data[['assigned_readability']].to_csv('assigned_redability_scores_13K.csv', index = False)"
      ],
      "metadata": {
        "id": "LiV29jutvmz8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "finbert_df.to_pickle('finbert_embeddings_of_definitions_13K.pickle')"
      ],
      "metadata": {
        "id": "GN8-VXJLw6oc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading Embeddings and Target variables"
      ],
      "metadata": {
        "id": "KLMZZhZcwbXq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X = pd.read_pickle('finbert_embeddings_of_definitions_13K.pickle') # ../embeddings_and_labels/finbert_embeddings_of_definitions_13K.pickle\n",
        "y = pd.read_csv('assigned_redability_scores_13K.csv') # ../embeddings_and_labels/assigned_redability_scores_13K.csv"
      ],
      "metadata": {
        "id": "yFL-4UH021oL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "id": "xAftOjpX3DNZ",
        "outputId": "490b20c6-b7af-4a8e-dccc-1740891803dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-7ac5cd00-8025-4730-948d-c934b3246412\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>...</th>\n",
              "      <th>728</th>\n",
              "      <th>729</th>\n",
              "      <th>730</th>\n",
              "      <th>731</th>\n",
              "      <th>732</th>\n",
              "      <th>733</th>\n",
              "      <th>734</th>\n",
              "      <th>735</th>\n",
              "      <th>736</th>\n",
              "      <th>737</th>\n",
              "      <th>738</th>\n",
              "      <th>739</th>\n",
              "      <th>740</th>\n",
              "      <th>741</th>\n",
              "      <th>742</th>\n",
              "      <th>743</th>\n",
              "      <th>744</th>\n",
              "      <th>745</th>\n",
              "      <th>746</th>\n",
              "      <th>747</th>\n",
              "      <th>748</th>\n",
              "      <th>749</th>\n",
              "      <th>750</th>\n",
              "      <th>751</th>\n",
              "      <th>752</th>\n",
              "      <th>753</th>\n",
              "      <th>754</th>\n",
              "      <th>755</th>\n",
              "      <th>756</th>\n",
              "      <th>757</th>\n",
              "      <th>758</th>\n",
              "      <th>759</th>\n",
              "      <th>760</th>\n",
              "      <th>761</th>\n",
              "      <th>762</th>\n",
              "      <th>763</th>\n",
              "      <th>764</th>\n",
              "      <th>765</th>\n",
              "      <th>766</th>\n",
              "      <th>767</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.184883</td>\n",
              "      <td>0.963402</td>\n",
              "      <td>-0.113459</td>\n",
              "      <td>-0.238247</td>\n",
              "      <td>0.228393</td>\n",
              "      <td>-0.657534</td>\n",
              "      <td>-0.159602</td>\n",
              "      <td>0.790045</td>\n",
              "      <td>0.284261</td>\n",
              "      <td>0.270735</td>\n",
              "      <td>-0.286424</td>\n",
              "      <td>-0.099882</td>\n",
              "      <td>0.328200</td>\n",
              "      <td>0.009324</td>\n",
              "      <td>-0.380326</td>\n",
              "      <td>-0.026808</td>\n",
              "      <td>0.069056</td>\n",
              "      <td>0.006242</td>\n",
              "      <td>0.523684</td>\n",
              "      <td>-0.187701</td>\n",
              "      <td>0.053000</td>\n",
              "      <td>0.666077</td>\n",
              "      <td>-0.168280</td>\n",
              "      <td>0.353479</td>\n",
              "      <td>0.666040</td>\n",
              "      <td>0.335737</td>\n",
              "      <td>0.537289</td>\n",
              "      <td>0.456613</td>\n",
              "      <td>0.456570</td>\n",
              "      <td>0.385766</td>\n",
              "      <td>0.997439</td>\n",
              "      <td>0.673123</td>\n",
              "      <td>0.247810</td>\n",
              "      <td>-0.063953</td>\n",
              "      <td>0.479946</td>\n",
              "      <td>-0.552191</td>\n",
              "      <td>0.294324</td>\n",
              "      <td>-0.451335</td>\n",
              "      <td>-0.681898</td>\n",
              "      <td>-0.144674</td>\n",
              "      <td>...</td>\n",
              "      <td>0.183947</td>\n",
              "      <td>0.322111</td>\n",
              "      <td>0.033319</td>\n",
              "      <td>-0.246759</td>\n",
              "      <td>-0.593302</td>\n",
              "      <td>-0.076825</td>\n",
              "      <td>0.052730</td>\n",
              "      <td>0.695216</td>\n",
              "      <td>-0.372869</td>\n",
              "      <td>0.505815</td>\n",
              "      <td>0.054873</td>\n",
              "      <td>0.523658</td>\n",
              "      <td>-0.045578</td>\n",
              "      <td>0.181468</td>\n",
              "      <td>0.247852</td>\n",
              "      <td>-1.099165</td>\n",
              "      <td>-0.476844</td>\n",
              "      <td>-0.227893</td>\n",
              "      <td>-0.545093</td>\n",
              "      <td>0.497437</td>\n",
              "      <td>0.383686</td>\n",
              "      <td>0.323929</td>\n",
              "      <td>0.200648</td>\n",
              "      <td>-0.122827</td>\n",
              "      <td>1.422405</td>\n",
              "      <td>0.089364</td>\n",
              "      <td>-0.189113</td>\n",
              "      <td>0.672834</td>\n",
              "      <td>0.370699</td>\n",
              "      <td>0.155610</td>\n",
              "      <td>-0.017009</td>\n",
              "      <td>0.330071</td>\n",
              "      <td>0.507158</td>\n",
              "      <td>-0.688259</td>\n",
              "      <td>-0.477652</td>\n",
              "      <td>0.103034</td>\n",
              "      <td>0.089814</td>\n",
              "      <td>-0.584482</td>\n",
              "      <td>-0.567774</td>\n",
              "      <td>0.718144</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.242829</td>\n",
              "      <td>0.680479</td>\n",
              "      <td>-0.060378</td>\n",
              "      <td>-0.196382</td>\n",
              "      <td>0.077879</td>\n",
              "      <td>-0.413284</td>\n",
              "      <td>0.127389</td>\n",
              "      <td>0.917318</td>\n",
              "      <td>0.083452</td>\n",
              "      <td>0.082668</td>\n",
              "      <td>-0.175501</td>\n",
              "      <td>-0.210223</td>\n",
              "      <td>0.084094</td>\n",
              "      <td>0.133842</td>\n",
              "      <td>-0.338829</td>\n",
              "      <td>0.311936</td>\n",
              "      <td>0.247770</td>\n",
              "      <td>0.237924</td>\n",
              "      <td>-0.074485</td>\n",
              "      <td>-0.228161</td>\n",
              "      <td>0.099722</td>\n",
              "      <td>0.606247</td>\n",
              "      <td>-0.069841</td>\n",
              "      <td>0.223901</td>\n",
              "      <td>0.511090</td>\n",
              "      <td>-0.065129</td>\n",
              "      <td>0.278169</td>\n",
              "      <td>0.357795</td>\n",
              "      <td>0.449496</td>\n",
              "      <td>0.193122</td>\n",
              "      <td>0.754209</td>\n",
              "      <td>0.572877</td>\n",
              "      <td>0.080695</td>\n",
              "      <td>-0.116159</td>\n",
              "      <td>0.731938</td>\n",
              "      <td>-0.115331</td>\n",
              "      <td>0.304042</td>\n",
              "      <td>-0.352987</td>\n",
              "      <td>-0.439339</td>\n",
              "      <td>0.063462</td>\n",
              "      <td>...</td>\n",
              "      <td>0.108002</td>\n",
              "      <td>0.097358</td>\n",
              "      <td>0.200539</td>\n",
              "      <td>-0.061815</td>\n",
              "      <td>-0.528963</td>\n",
              "      <td>-0.043847</td>\n",
              "      <td>0.509497</td>\n",
              "      <td>1.002764</td>\n",
              "      <td>-0.529102</td>\n",
              "      <td>0.362646</td>\n",
              "      <td>0.103270</td>\n",
              "      <td>0.741056</td>\n",
              "      <td>-0.301473</td>\n",
              "      <td>0.170660</td>\n",
              "      <td>0.608756</td>\n",
              "      <td>-0.928778</td>\n",
              "      <td>-0.752666</td>\n",
              "      <td>-0.186905</td>\n",
              "      <td>-0.182519</td>\n",
              "      <td>0.252048</td>\n",
              "      <td>0.804839</td>\n",
              "      <td>0.067022</td>\n",
              "      <td>0.025321</td>\n",
              "      <td>-0.058781</td>\n",
              "      <td>1.102862</td>\n",
              "      <td>-0.234377</td>\n",
              "      <td>0.048481</td>\n",
              "      <td>0.331052</td>\n",
              "      <td>0.742415</td>\n",
              "      <td>0.054227</td>\n",
              "      <td>0.235442</td>\n",
              "      <td>0.050984</td>\n",
              "      <td>0.081753</td>\n",
              "      <td>-0.897964</td>\n",
              "      <td>-0.700977</td>\n",
              "      <td>0.100685</td>\n",
              "      <td>0.469065</td>\n",
              "      <td>-0.542681</td>\n",
              "      <td>-0.172880</td>\n",
              "      <td>0.511789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.161858</td>\n",
              "      <td>0.389614</td>\n",
              "      <td>0.371562</td>\n",
              "      <td>-0.274890</td>\n",
              "      <td>0.325236</td>\n",
              "      <td>-0.289743</td>\n",
              "      <td>0.003597</td>\n",
              "      <td>1.158318</td>\n",
              "      <td>-0.097860</td>\n",
              "      <td>0.210648</td>\n",
              "      <td>-0.167418</td>\n",
              "      <td>-0.518324</td>\n",
              "      <td>-0.059521</td>\n",
              "      <td>0.435780</td>\n",
              "      <td>-0.046144</td>\n",
              "      <td>0.310456</td>\n",
              "      <td>0.030572</td>\n",
              "      <td>-0.168012</td>\n",
              "      <td>-0.114892</td>\n",
              "      <td>0.249773</td>\n",
              "      <td>0.218560</td>\n",
              "      <td>0.885349</td>\n",
              "      <td>-0.072886</td>\n",
              "      <td>-0.064065</td>\n",
              "      <td>0.375222</td>\n",
              "      <td>-0.304826</td>\n",
              "      <td>0.412837</td>\n",
              "      <td>0.073889</td>\n",
              "      <td>0.172669</td>\n",
              "      <td>0.132819</td>\n",
              "      <td>0.479669</td>\n",
              "      <td>0.439182</td>\n",
              "      <td>0.105781</td>\n",
              "      <td>0.230065</td>\n",
              "      <td>1.026393</td>\n",
              "      <td>-0.492289</td>\n",
              "      <td>0.298767</td>\n",
              "      <td>-0.784319</td>\n",
              "      <td>-0.400964</td>\n",
              "      <td>-0.220388</td>\n",
              "      <td>...</td>\n",
              "      <td>0.376578</td>\n",
              "      <td>0.147067</td>\n",
              "      <td>-0.331118</td>\n",
              "      <td>-0.043380</td>\n",
              "      <td>-0.343244</td>\n",
              "      <td>-0.291331</td>\n",
              "      <td>0.607986</td>\n",
              "      <td>0.528104</td>\n",
              "      <td>-0.303974</td>\n",
              "      <td>0.164009</td>\n",
              "      <td>-0.324436</td>\n",
              "      <td>0.730373</td>\n",
              "      <td>0.266155</td>\n",
              "      <td>-0.317194</td>\n",
              "      <td>0.442139</td>\n",
              "      <td>-0.696930</td>\n",
              "      <td>-0.245127</td>\n",
              "      <td>0.047178</td>\n",
              "      <td>-0.480287</td>\n",
              "      <td>0.301644</td>\n",
              "      <td>0.549728</td>\n",
              "      <td>0.210458</td>\n",
              "      <td>-0.113074</td>\n",
              "      <td>-0.305841</td>\n",
              "      <td>1.193745</td>\n",
              "      <td>0.240389</td>\n",
              "      <td>0.010169</td>\n",
              "      <td>0.330152</td>\n",
              "      <td>0.250107</td>\n",
              "      <td>-0.284011</td>\n",
              "      <td>0.390322</td>\n",
              "      <td>0.151351</td>\n",
              "      <td>-0.102673</td>\n",
              "      <td>-0.592789</td>\n",
              "      <td>-0.484857</td>\n",
              "      <td>-0.153056</td>\n",
              "      <td>0.080858</td>\n",
              "      <td>-0.228636</td>\n",
              "      <td>-0.036314</td>\n",
              "      <td>0.441195</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.023223</td>\n",
              "      <td>0.445641</td>\n",
              "      <td>0.016909</td>\n",
              "      <td>0.166205</td>\n",
              "      <td>0.483727</td>\n",
              "      <td>-0.435553</td>\n",
              "      <td>-0.254949</td>\n",
              "      <td>0.528690</td>\n",
              "      <td>0.017388</td>\n",
              "      <td>-0.342903</td>\n",
              "      <td>-0.492616</td>\n",
              "      <td>-0.134340</td>\n",
              "      <td>0.244819</td>\n",
              "      <td>-0.538804</td>\n",
              "      <td>0.032028</td>\n",
              "      <td>0.232134</td>\n",
              "      <td>0.516510</td>\n",
              "      <td>-0.143379</td>\n",
              "      <td>0.043878</td>\n",
              "      <td>-0.035306</td>\n",
              "      <td>0.060544</td>\n",
              "      <td>0.005003</td>\n",
              "      <td>-0.286677</td>\n",
              "      <td>0.296558</td>\n",
              "      <td>0.359438</td>\n",
              "      <td>0.037744</td>\n",
              "      <td>0.107464</td>\n",
              "      <td>0.930401</td>\n",
              "      <td>0.107573</td>\n",
              "      <td>0.246667</td>\n",
              "      <td>0.488973</td>\n",
              "      <td>0.439756</td>\n",
              "      <td>-0.088450</td>\n",
              "      <td>0.113551</td>\n",
              "      <td>0.661401</td>\n",
              "      <td>-0.683219</td>\n",
              "      <td>0.605379</td>\n",
              "      <td>0.116923</td>\n",
              "      <td>-0.405467</td>\n",
              "      <td>-0.271648</td>\n",
              "      <td>...</td>\n",
              "      <td>0.517980</td>\n",
              "      <td>0.002232</td>\n",
              "      <td>0.378510</td>\n",
              "      <td>0.190717</td>\n",
              "      <td>-0.363660</td>\n",
              "      <td>-0.418434</td>\n",
              "      <td>-0.018251</td>\n",
              "      <td>0.566143</td>\n",
              "      <td>-0.288505</td>\n",
              "      <td>0.473909</td>\n",
              "      <td>-0.398661</td>\n",
              "      <td>-0.069614</td>\n",
              "      <td>-0.389753</td>\n",
              "      <td>0.284137</td>\n",
              "      <td>0.292468</td>\n",
              "      <td>-1.221428</td>\n",
              "      <td>-0.600689</td>\n",
              "      <td>-0.273175</td>\n",
              "      <td>-0.223456</td>\n",
              "      <td>0.279699</td>\n",
              "      <td>0.395662</td>\n",
              "      <td>0.363241</td>\n",
              "      <td>-0.052759</td>\n",
              "      <td>-0.270714</td>\n",
              "      <td>1.488481</td>\n",
              "      <td>-0.375759</td>\n",
              "      <td>0.454230</td>\n",
              "      <td>0.472612</td>\n",
              "      <td>0.179239</td>\n",
              "      <td>0.090694</td>\n",
              "      <td>-0.279148</td>\n",
              "      <td>-0.179137</td>\n",
              "      <td>0.372667</td>\n",
              "      <td>-0.202656</td>\n",
              "      <td>-0.557891</td>\n",
              "      <td>-0.041573</td>\n",
              "      <td>0.231946</td>\n",
              "      <td>-0.785041</td>\n",
              "      <td>-0.764352</td>\n",
              "      <td>0.203464</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-0.277175</td>\n",
              "      <td>0.317595</td>\n",
              "      <td>0.411673</td>\n",
              "      <td>-0.258846</td>\n",
              "      <td>0.333753</td>\n",
              "      <td>-0.333010</td>\n",
              "      <td>0.295542</td>\n",
              "      <td>0.999050</td>\n",
              "      <td>0.254497</td>\n",
              "      <td>0.048463</td>\n",
              "      <td>-0.332334</td>\n",
              "      <td>-0.025842</td>\n",
              "      <td>-0.032343</td>\n",
              "      <td>0.042468</td>\n",
              "      <td>-0.240808</td>\n",
              "      <td>0.083356</td>\n",
              "      <td>-0.271210</td>\n",
              "      <td>0.050999</td>\n",
              "      <td>-0.222975</td>\n",
              "      <td>-0.132725</td>\n",
              "      <td>0.352255</td>\n",
              "      <td>0.304582</td>\n",
              "      <td>-0.266617</td>\n",
              "      <td>0.159570</td>\n",
              "      <td>0.258637</td>\n",
              "      <td>-0.073498</td>\n",
              "      <td>0.257358</td>\n",
              "      <td>0.251949</td>\n",
              "      <td>0.180690</td>\n",
              "      <td>0.098515</td>\n",
              "      <td>0.552765</td>\n",
              "      <td>0.454875</td>\n",
              "      <td>0.368944</td>\n",
              "      <td>0.005718</td>\n",
              "      <td>0.975146</td>\n",
              "      <td>-0.269984</td>\n",
              "      <td>0.036238</td>\n",
              "      <td>-0.362672</td>\n",
              "      <td>-0.281072</td>\n",
              "      <td>-0.116108</td>\n",
              "      <td>...</td>\n",
              "      <td>0.570795</td>\n",
              "      <td>0.286619</td>\n",
              "      <td>-0.001280</td>\n",
              "      <td>-0.042452</td>\n",
              "      <td>-0.508884</td>\n",
              "      <td>-0.465914</td>\n",
              "      <td>0.565026</td>\n",
              "      <td>0.553120</td>\n",
              "      <td>0.057060</td>\n",
              "      <td>0.602830</td>\n",
              "      <td>-0.051018</td>\n",
              "      <td>0.374319</td>\n",
              "      <td>0.307036</td>\n",
              "      <td>-0.567812</td>\n",
              "      <td>1.254878</td>\n",
              "      <td>-1.321846</td>\n",
              "      <td>-0.326154</td>\n",
              "      <td>0.141492</td>\n",
              "      <td>-0.532748</td>\n",
              "      <td>0.486383</td>\n",
              "      <td>0.493066</td>\n",
              "      <td>0.212075</td>\n",
              "      <td>0.441086</td>\n",
              "      <td>-0.105270</td>\n",
              "      <td>1.322911</td>\n",
              "      <td>-0.183309</td>\n",
              "      <td>-0.207935</td>\n",
              "      <td>0.183314</td>\n",
              "      <td>0.369047</td>\n",
              "      <td>-0.629212</td>\n",
              "      <td>0.722259</td>\n",
              "      <td>0.252352</td>\n",
              "      <td>-0.185299</td>\n",
              "      <td>-0.841254</td>\n",
              "      <td>-0.561818</td>\n",
              "      <td>0.155261</td>\n",
              "      <td>0.255316</td>\n",
              "      <td>-0.050675</td>\n",
              "      <td>-0.243014</td>\n",
              "      <td>0.630817</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13107</th>\n",
              "      <td>0.021163</td>\n",
              "      <td>0.331648</td>\n",
              "      <td>0.097957</td>\n",
              "      <td>-0.044651</td>\n",
              "      <td>0.397460</td>\n",
              "      <td>-0.246158</td>\n",
              "      <td>-0.148983</td>\n",
              "      <td>0.647735</td>\n",
              "      <td>0.221966</td>\n",
              "      <td>0.252533</td>\n",
              "      <td>-0.309565</td>\n",
              "      <td>0.053896</td>\n",
              "      <td>0.272400</td>\n",
              "      <td>-0.049390</td>\n",
              "      <td>-0.344258</td>\n",
              "      <td>-0.143515</td>\n",
              "      <td>0.173908</td>\n",
              "      <td>-0.160543</td>\n",
              "      <td>-0.077118</td>\n",
              "      <td>-0.066772</td>\n",
              "      <td>0.332805</td>\n",
              "      <td>0.684235</td>\n",
              "      <td>-0.076209</td>\n",
              "      <td>0.212857</td>\n",
              "      <td>0.100115</td>\n",
              "      <td>0.020390</td>\n",
              "      <td>0.154811</td>\n",
              "      <td>0.087563</td>\n",
              "      <td>0.255231</td>\n",
              "      <td>0.289032</td>\n",
              "      <td>0.723061</td>\n",
              "      <td>0.249036</td>\n",
              "      <td>0.416077</td>\n",
              "      <td>-0.039980</td>\n",
              "      <td>0.725918</td>\n",
              "      <td>-0.128038</td>\n",
              "      <td>0.300892</td>\n",
              "      <td>-0.413751</td>\n",
              "      <td>-0.416851</td>\n",
              "      <td>-0.387010</td>\n",
              "      <td>...</td>\n",
              "      <td>0.303299</td>\n",
              "      <td>0.388174</td>\n",
              "      <td>0.229467</td>\n",
              "      <td>-0.232867</td>\n",
              "      <td>-0.405656</td>\n",
              "      <td>-0.292352</td>\n",
              "      <td>0.434300</td>\n",
              "      <td>0.710381</td>\n",
              "      <td>-0.284060</td>\n",
              "      <td>0.633546</td>\n",
              "      <td>-0.094310</td>\n",
              "      <td>0.134642</td>\n",
              "      <td>0.132167</td>\n",
              "      <td>-0.142419</td>\n",
              "      <td>0.540762</td>\n",
              "      <td>-1.019609</td>\n",
              "      <td>-0.457518</td>\n",
              "      <td>0.006361</td>\n",
              "      <td>-0.427495</td>\n",
              "      <td>0.244350</td>\n",
              "      <td>0.639600</td>\n",
              "      <td>0.326619</td>\n",
              "      <td>0.156354</td>\n",
              "      <td>-0.278068</td>\n",
              "      <td>1.282086</td>\n",
              "      <td>-0.135513</td>\n",
              "      <td>-0.128157</td>\n",
              "      <td>0.548470</td>\n",
              "      <td>-0.201803</td>\n",
              "      <td>-0.485339</td>\n",
              "      <td>0.108834</td>\n",
              "      <td>0.040428</td>\n",
              "      <td>-0.086953</td>\n",
              "      <td>-0.695931</td>\n",
              "      <td>-0.450496</td>\n",
              "      <td>0.051134</td>\n",
              "      <td>-0.157590</td>\n",
              "      <td>0.006986</td>\n",
              "      <td>-0.434256</td>\n",
              "      <td>0.656598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13108</th>\n",
              "      <td>-0.187862</td>\n",
              "      <td>0.212622</td>\n",
              "      <td>0.138334</td>\n",
              "      <td>-0.080471</td>\n",
              "      <td>0.348631</td>\n",
              "      <td>-0.103958</td>\n",
              "      <td>-0.212263</td>\n",
              "      <td>0.367935</td>\n",
              "      <td>0.567376</td>\n",
              "      <td>-0.186444</td>\n",
              "      <td>-0.156049</td>\n",
              "      <td>-0.072839</td>\n",
              "      <td>0.206802</td>\n",
              "      <td>-0.051871</td>\n",
              "      <td>0.019011</td>\n",
              "      <td>0.177611</td>\n",
              "      <td>0.187596</td>\n",
              "      <td>-0.182136</td>\n",
              "      <td>0.123549</td>\n",
              "      <td>0.105473</td>\n",
              "      <td>0.085287</td>\n",
              "      <td>0.679156</td>\n",
              "      <td>-0.200505</td>\n",
              "      <td>0.342952</td>\n",
              "      <td>0.384540</td>\n",
              "      <td>-0.111028</td>\n",
              "      <td>0.122931</td>\n",
              "      <td>0.068794</td>\n",
              "      <td>0.076331</td>\n",
              "      <td>0.290271</td>\n",
              "      <td>0.346695</td>\n",
              "      <td>0.318418</td>\n",
              "      <td>0.223632</td>\n",
              "      <td>-0.490634</td>\n",
              "      <td>0.525421</td>\n",
              "      <td>-0.412621</td>\n",
              "      <td>0.048449</td>\n",
              "      <td>-0.150551</td>\n",
              "      <td>-0.292101</td>\n",
              "      <td>-0.218417</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.364646</td>\n",
              "      <td>0.083237</td>\n",
              "      <td>-0.023131</td>\n",
              "      <td>0.138374</td>\n",
              "      <td>-0.364378</td>\n",
              "      <td>-0.109004</td>\n",
              "      <td>0.355341</td>\n",
              "      <td>0.472508</td>\n",
              "      <td>-0.570895</td>\n",
              "      <td>0.178758</td>\n",
              "      <td>0.089382</td>\n",
              "      <td>0.143989</td>\n",
              "      <td>-0.218513</td>\n",
              "      <td>0.203486</td>\n",
              "      <td>0.305163</td>\n",
              "      <td>-0.807924</td>\n",
              "      <td>-0.513458</td>\n",
              "      <td>0.148365</td>\n",
              "      <td>-0.264189</td>\n",
              "      <td>0.399860</td>\n",
              "      <td>0.355850</td>\n",
              "      <td>0.414548</td>\n",
              "      <td>-0.209196</td>\n",
              "      <td>0.035620</td>\n",
              "      <td>1.012324</td>\n",
              "      <td>0.273522</td>\n",
              "      <td>-0.284049</td>\n",
              "      <td>0.329157</td>\n",
              "      <td>0.049142</td>\n",
              "      <td>-0.380106</td>\n",
              "      <td>0.247228</td>\n",
              "      <td>0.249030</td>\n",
              "      <td>-0.229585</td>\n",
              "      <td>-0.574909</td>\n",
              "      <td>-0.640928</td>\n",
              "      <td>-0.235551</td>\n",
              "      <td>-0.128866</td>\n",
              "      <td>-0.341619</td>\n",
              "      <td>-0.360839</td>\n",
              "      <td>0.401239</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13109</th>\n",
              "      <td>0.232175</td>\n",
              "      <td>0.139506</td>\n",
              "      <td>0.006346</td>\n",
              "      <td>-0.115720</td>\n",
              "      <td>0.194042</td>\n",
              "      <td>0.065195</td>\n",
              "      <td>-0.062542</td>\n",
              "      <td>0.751114</td>\n",
              "      <td>0.366908</td>\n",
              "      <td>0.065513</td>\n",
              "      <td>-0.188477</td>\n",
              "      <td>-0.474425</td>\n",
              "      <td>-0.081461</td>\n",
              "      <td>0.201449</td>\n",
              "      <td>-0.091342</td>\n",
              "      <td>0.088053</td>\n",
              "      <td>-0.011597</td>\n",
              "      <td>-0.223374</td>\n",
              "      <td>0.220098</td>\n",
              "      <td>0.060098</td>\n",
              "      <td>0.368565</td>\n",
              "      <td>1.031158</td>\n",
              "      <td>-0.057572</td>\n",
              "      <td>0.111284</td>\n",
              "      <td>0.236639</td>\n",
              "      <td>0.080211</td>\n",
              "      <td>0.292216</td>\n",
              "      <td>-0.117506</td>\n",
              "      <td>-0.125316</td>\n",
              "      <td>0.305051</td>\n",
              "      <td>0.520381</td>\n",
              "      <td>0.357687</td>\n",
              "      <td>0.190173</td>\n",
              "      <td>-0.141151</td>\n",
              "      <td>0.374471</td>\n",
              "      <td>-0.417710</td>\n",
              "      <td>0.412437</td>\n",
              "      <td>-0.091998</td>\n",
              "      <td>-0.350800</td>\n",
              "      <td>-0.353843</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.290798</td>\n",
              "      <td>-0.148217</td>\n",
              "      <td>-0.220408</td>\n",
              "      <td>0.001389</td>\n",
              "      <td>-0.426606</td>\n",
              "      <td>0.009722</td>\n",
              "      <td>0.348430</td>\n",
              "      <td>0.718493</td>\n",
              "      <td>-0.357420</td>\n",
              "      <td>0.194281</td>\n",
              "      <td>-0.095499</td>\n",
              "      <td>0.327751</td>\n",
              "      <td>0.145683</td>\n",
              "      <td>-0.029311</td>\n",
              "      <td>0.078217</td>\n",
              "      <td>-1.093480</td>\n",
              "      <td>-0.368556</td>\n",
              "      <td>0.150657</td>\n",
              "      <td>-0.184456</td>\n",
              "      <td>0.331329</td>\n",
              "      <td>0.743703</td>\n",
              "      <td>0.792232</td>\n",
              "      <td>-0.022615</td>\n",
              "      <td>-0.150573</td>\n",
              "      <td>1.296555</td>\n",
              "      <td>0.004470</td>\n",
              "      <td>-0.129560</td>\n",
              "      <td>0.226173</td>\n",
              "      <td>0.138514</td>\n",
              "      <td>-0.270262</td>\n",
              "      <td>0.528386</td>\n",
              "      <td>0.255743</td>\n",
              "      <td>-0.103669</td>\n",
              "      <td>-0.456029</td>\n",
              "      <td>-0.271310</td>\n",
              "      <td>-0.056394</td>\n",
              "      <td>0.173910</td>\n",
              "      <td>-0.084195</td>\n",
              "      <td>-0.031481</td>\n",
              "      <td>0.496607</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13110</th>\n",
              "      <td>0.203303</td>\n",
              "      <td>0.017677</td>\n",
              "      <td>0.105035</td>\n",
              "      <td>-0.009308</td>\n",
              "      <td>0.176142</td>\n",
              "      <td>0.125231</td>\n",
              "      <td>-0.034156</td>\n",
              "      <td>0.560181</td>\n",
              "      <td>0.372004</td>\n",
              "      <td>0.062158</td>\n",
              "      <td>-0.220316</td>\n",
              "      <td>-0.371438</td>\n",
              "      <td>-0.090994</td>\n",
              "      <td>0.165835</td>\n",
              "      <td>-0.068784</td>\n",
              "      <td>0.154269</td>\n",
              "      <td>0.104465</td>\n",
              "      <td>-0.268656</td>\n",
              "      <td>0.148881</td>\n",
              "      <td>0.117132</td>\n",
              "      <td>0.439617</td>\n",
              "      <td>0.922795</td>\n",
              "      <td>0.011157</td>\n",
              "      <td>0.117145</td>\n",
              "      <td>0.225179</td>\n",
              "      <td>0.109822</td>\n",
              "      <td>0.298109</td>\n",
              "      <td>-0.099077</td>\n",
              "      <td>-0.210599</td>\n",
              "      <td>0.371557</td>\n",
              "      <td>0.494324</td>\n",
              "      <td>0.380823</td>\n",
              "      <td>0.002952</td>\n",
              "      <td>-0.123375</td>\n",
              "      <td>0.298069</td>\n",
              "      <td>-0.338508</td>\n",
              "      <td>0.419182</td>\n",
              "      <td>-0.082242</td>\n",
              "      <td>-0.381123</td>\n",
              "      <td>-0.380525</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.282394</td>\n",
              "      <td>-0.172521</td>\n",
              "      <td>-0.210246</td>\n",
              "      <td>-0.162646</td>\n",
              "      <td>-0.301045</td>\n",
              "      <td>-0.012515</td>\n",
              "      <td>0.270225</td>\n",
              "      <td>0.633970</td>\n",
              "      <td>-0.429181</td>\n",
              "      <td>0.254226</td>\n",
              "      <td>-0.094158</td>\n",
              "      <td>0.285457</td>\n",
              "      <td>0.215410</td>\n",
              "      <td>-0.096119</td>\n",
              "      <td>-0.033630</td>\n",
              "      <td>-1.218847</td>\n",
              "      <td>-0.273695</td>\n",
              "      <td>0.010020</td>\n",
              "      <td>-0.281258</td>\n",
              "      <td>0.213125</td>\n",
              "      <td>0.812164</td>\n",
              "      <td>0.888290</td>\n",
              "      <td>-0.106968</td>\n",
              "      <td>-0.099163</td>\n",
              "      <td>1.266647</td>\n",
              "      <td>-0.062871</td>\n",
              "      <td>-0.174459</td>\n",
              "      <td>0.139911</td>\n",
              "      <td>0.108533</td>\n",
              "      <td>-0.358623</td>\n",
              "      <td>0.430703</td>\n",
              "      <td>0.350374</td>\n",
              "      <td>0.032217</td>\n",
              "      <td>-0.429955</td>\n",
              "      <td>-0.367603</td>\n",
              "      <td>-0.040143</td>\n",
              "      <td>0.174703</td>\n",
              "      <td>-0.095335</td>\n",
              "      <td>0.094141</td>\n",
              "      <td>0.473806</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13111</th>\n",
              "      <td>-0.549440</td>\n",
              "      <td>0.002461</td>\n",
              "      <td>0.398878</td>\n",
              "      <td>-0.316565</td>\n",
              "      <td>0.813447</td>\n",
              "      <td>0.031025</td>\n",
              "      <td>0.492473</td>\n",
              "      <td>0.612148</td>\n",
              "      <td>-0.128865</td>\n",
              "      <td>0.129941</td>\n",
              "      <td>-0.364990</td>\n",
              "      <td>-0.644180</td>\n",
              "      <td>0.275995</td>\n",
              "      <td>0.722479</td>\n",
              "      <td>-0.061328</td>\n",
              "      <td>0.256556</td>\n",
              "      <td>0.109354</td>\n",
              "      <td>0.160724</td>\n",
              "      <td>0.239262</td>\n",
              "      <td>0.251696</td>\n",
              "      <td>-0.290292</td>\n",
              "      <td>0.235854</td>\n",
              "      <td>0.069166</td>\n",
              "      <td>0.494916</td>\n",
              "      <td>0.289053</td>\n",
              "      <td>-0.316784</td>\n",
              "      <td>-0.218791</td>\n",
              "      <td>-0.337396</td>\n",
              "      <td>-0.426572</td>\n",
              "      <td>-0.184128</td>\n",
              "      <td>-0.021364</td>\n",
              "      <td>-0.092326</td>\n",
              "      <td>-0.006793</td>\n",
              "      <td>0.030584</td>\n",
              "      <td>0.746363</td>\n",
              "      <td>-0.027652</td>\n",
              "      <td>-0.255142</td>\n",
              "      <td>-0.217362</td>\n",
              "      <td>-0.737086</td>\n",
              "      <td>-0.131872</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.162203</td>\n",
              "      <td>-0.763273</td>\n",
              "      <td>-0.435070</td>\n",
              "      <td>0.032795</td>\n",
              "      <td>-0.364614</td>\n",
              "      <td>0.236522</td>\n",
              "      <td>0.149270</td>\n",
              "      <td>0.269347</td>\n",
              "      <td>-0.404875</td>\n",
              "      <td>0.538478</td>\n",
              "      <td>-0.284155</td>\n",
              "      <td>0.781907</td>\n",
              "      <td>-0.029453</td>\n",
              "      <td>0.106128</td>\n",
              "      <td>0.118717</td>\n",
              "      <td>-0.784383</td>\n",
              "      <td>-0.338728</td>\n",
              "      <td>0.129703</td>\n",
              "      <td>-0.139588</td>\n",
              "      <td>-0.253445</td>\n",
              "      <td>0.107002</td>\n",
              "      <td>0.134742</td>\n",
              "      <td>-0.107807</td>\n",
              "      <td>-0.144779</td>\n",
              "      <td>1.144725</td>\n",
              "      <td>-0.383157</td>\n",
              "      <td>-0.461921</td>\n",
              "      <td>0.225701</td>\n",
              "      <td>0.384148</td>\n",
              "      <td>-0.675798</td>\n",
              "      <td>0.112448</td>\n",
              "      <td>0.337801</td>\n",
              "      <td>0.112309</td>\n",
              "      <td>-0.882201</td>\n",
              "      <td>-0.331987</td>\n",
              "      <td>-0.869165</td>\n",
              "      <td>0.260543</td>\n",
              "      <td>-0.484959</td>\n",
              "      <td>-0.211000</td>\n",
              "      <td>0.101889</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13112 rows × 768 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7ac5cd00-8025-4730-948d-c934b3246412')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7ac5cd00-8025-4730-948d-c934b3246412 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7ac5cd00-8025-4730-948d-c934b3246412');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "            0         1         2    ...       765       766       767\n",
              "0      0.184883  0.963402 -0.113459  ... -0.584482 -0.567774  0.718144\n",
              "1     -0.242829  0.680479 -0.060378  ... -0.542681 -0.172880  0.511789\n",
              "2     -0.161858  0.389614  0.371562  ... -0.228636 -0.036314  0.441195\n",
              "3     -0.023223  0.445641  0.016909  ... -0.785041 -0.764352  0.203464\n",
              "4     -0.277175  0.317595  0.411673  ... -0.050675 -0.243014  0.630817\n",
              "...         ...       ...       ...  ...       ...       ...       ...\n",
              "13107  0.021163  0.331648  0.097957  ...  0.006986 -0.434256  0.656598\n",
              "13108 -0.187862  0.212622  0.138334  ... -0.341619 -0.360839  0.401239\n",
              "13109  0.232175  0.139506  0.006346  ... -0.084195 -0.031481  0.496607\n",
              "13110  0.203303  0.017677  0.105035  ... -0.095335  0.094141  0.473806\n",
              "13111 -0.549440  0.002461  0.398878  ... -0.484959 -0.211000  0.101889\n",
              "\n",
              "[13112 rows x 768 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "ME5A9Fib3oAW",
        "outputId": "3e6010e3-d8ca-4384-e6ef-ff57c77adc8c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-d5d10f84-4233-494c-ac33-8f3b3a56b6bb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>assigned_readability</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13107</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13108</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13109</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13110</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13111</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13112 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d5d10f84-4233-494c-ac33-8f3b3a56b6bb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d5d10f84-4233-494c-ac33-8f3b3a56b6bb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d5d10f84-4233-494c-ac33-8f3b3a56b6bb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       assigned_readability\n",
              "0                         0\n",
              "1                         0\n",
              "2                         0\n",
              "3                         0\n",
              "4                         0\n",
              "...                     ...\n",
              "13107                     0\n",
              "13108                     0\n",
              "13109                     0\n",
              "13110                     0\n",
              "13111                     0\n",
              "\n",
              "[13112 rows x 1 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.33, random_state=42)"
      ],
      "metadata": {
        "id": "AEObjGwc3pAn"
      },
      "execution_count": 9,
      "outputs": []
    },
    
    {
      "cell_type": "markdown",
      "source": [
        "# Models"
      ],
      "metadata": {
        "id": "Xqf5AKwI4YDw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def model(clf, train_X, train_y, valid_X, valid_y):\n",
        "    clf.fit(train_X, train_y)\n",
        "    pred_tr = clf.predict(train_X)\n",
        "    pred_valid = clf.predict(valid_X)\n",
        "    pred_tr_prob = clf.predict_proba(train_X)[:,1]\n",
        "    pred_valid_prob = clf.predict_proba(valid_X)[:,1]\n",
        "    print(\"\\nTraining F1:{}\".format(f1_score(train_y, pred_tr, average=\"weighted\")))\n",
        "    print(\"Training Confusion Matrix \\n{}\".format(confusion_matrix(train_y, pred_tr)))\n",
        "    print(\"Classification Report Train: \\n{}\".format(classification_report(train_y, pred_tr)))\n",
        "    print(\"AUC Train\", roc_auc_score(train_y, pred_tr_prob))\n",
        "\n",
        "    print(\n",
        "        \"\\nValidation F1:{}\".format(f1_score(valid_y, pred_valid, average=\"weighted\"))\n",
        "    )\n",
        "    print(\n",
        "        \"Validation Confusion Matrix \\n{}\".format(confusion_matrix(valid_y, pred_valid))\n",
        "    )\n",
        "    print(\n",
        "        \"Classification Report: \\n{}\".format(classification_report(valid_y, pred_valid))\n",
        "    )\n",
        "    print(\"AUC Valid\", roc_auc_score(valid_y, pred_valid_prob))"
      ],
      "metadata": {
        "id": "bwQhiBF3-dqG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Logistic Regression"
      ],
      "metadata": {
        "id": "AfD5twN-COx2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "lr_clf = LogisticRegression(solver=\"lbfgs\", n_jobs=-1)\n",
        "model(lr_clf, train_X, train_y, valid_X, valid_y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "txU9I4xL-eTU",
        "outputId": "a5cd3635-e172-4c5f-d7b7-d1a13c62cea6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py:985: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Training F1:0.9456293084590777\n",
            "Training Confusion Matrix \n",
            "[[3458  219]\n",
            " [ 259 4849]]\n",
            "Classification Report Train: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.94      0.94      3677\n",
            "           1       0.96      0.95      0.95      5108\n",
            "\n",
            "    accuracy                           0.95      8785\n",
            "   macro avg       0.94      0.94      0.94      8785\n",
            "weighted avg       0.95      0.95      0.95      8785\n",
            "\n",
            "AUC Train 0.9868565661078867\n",
            "\n",
            "Validation F1:0.9198866139528612\n",
            "Validation Confusion Matrix \n",
            "[[1672  159]\n",
            " [ 188 2308]]\n",
            "Classification Report: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.90      0.91      0.91      1831\n",
            "           1       0.94      0.92      0.93      2496\n",
            "\n",
            "    accuracy                           0.92      4327\n",
            "   macro avg       0.92      0.92      0.92      4327\n",
            "weighted avg       0.92      0.92      0.92      4327\n",
            "\n",
            "AUC Valid 0.9692768943690572\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('logistic_regression_classifier.pkl', 'wb') as f:\n",
        "    pickle.dump(lr_clf, f)"
      ],
      "metadata": {
        "id": "Jc3d0_ft_zaB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Random Forest"
      ],
      "metadata": {
        "id": "rh4plBq-CVHy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "rf_clf = RandomForestClassifier(max_depth=10, random_state=0, n_estimators=100, class_weight=\"balanced\")\n",
        "model(rf_clf, train_X, train_y, valid_X, valid_y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PPDPmbGW-jZ-",
        "outputId": "2c540369-e4ed-41e0-f22e-230e4ee5d956"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
            "  \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Training F1:0.9849709233897832\n",
            "Training Confusion Matrix \n",
            "[[3605   72]\n",
            " [  60 5048]]\n",
            "Classification Report Train: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.98      0.98      3677\n",
            "           1       0.99      0.99      0.99      5108\n",
            "\n",
            "    accuracy                           0.98      8785\n",
            "   macro avg       0.98      0.98      0.98      8785\n",
            "weighted avg       0.98      0.98      0.98      8785\n",
            "\n",
            "AUC Train 0.9987011314380125\n",
            "\n",
            "Validation F1:0.8698083596755274\n",
            "Validation Confusion Matrix \n",
            "[[1501  330]\n",
            " [ 231 2265]]\n",
            "Classification Report: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.87      0.82      0.84      1831\n",
            "           1       0.87      0.91      0.89      2496\n",
            "\n",
            "    accuracy                           0.87      4327\n",
            "   macro avg       0.87      0.86      0.87      4327\n",
            "weighted avg       0.87      0.87      0.87      4327\n",
            "\n",
            "AUC Valid 0.9435107094343851\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('random_forest_classifier.pkl', 'wb') as f:\n",
        "    pickle.dump(rf_clf, f)"
      ],
      "metadata": {
        "id": "2N5TBonfARH-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## GBM"
      ],
      "metadata": {
        "id": "sR-fIAytAgOY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gbm_clf = GradientBoostingClassifier(max_depth=10, random_state=0, n_estimators=100)\n",
        "model(gbm_clf, train_X, train_y, valid_X, valid_y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FZKq3BQaChFS",
        "outputId": "74d05200-78ca-4b94-a3cb-d2a2e6d42dd9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_gb.py:494: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Training F1:0.9998861717738574\n",
            "Training Confusion Matrix \n",
            "[[3677    0]\n",
            " [   1 5107]]\n",
            "Classification Report Train: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      1.00      1.00      3677\n",
            "           1       1.00      1.00      1.00      5108\n",
            "\n",
            "    accuracy                           1.00      8785\n",
            "   macro avg       1.00      1.00      1.00      8785\n",
            "weighted avg       1.00      1.00      1.00      8785\n",
            "\n",
            "AUC Train 0.9999999733789313\n",
            "\n",
            "Validation F1:0.8829240299638225\n",
            "Validation Confusion Matrix \n",
            "[[1540  291]\n",
            " [ 214 2282]]\n",
            "Classification Report: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.84      0.86      1831\n",
            "           1       0.89      0.91      0.90      2496\n",
            "\n",
            "    accuracy                           0.88      4327\n",
            "   macro avg       0.88      0.88      0.88      4327\n",
            "weighted avg       0.88      0.88      0.88      4327\n",
            "\n",
            "AUC Valid 0.9523591651612543\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('gbm_classifier.pkl', 'wb') as f:\n",
        "    pickle.dump(gbm_clf, f)"
      ],
      "metadata": {
        "id": "r3Vh6i6tD1-7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "l-UMM9FhD-Mz"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}