handling_outliers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO5YzTf2J+qan8BJcRnnTA+",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/muajnstu/phishing_URL_detection_using_transductive_learning-/blob/main/handling_outliers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UhA3IjAlAtww"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "url = 'https://media.githubusercontent.com/media/shahriariit/opendataset/master/PhiUSIIL_phishing.csv'\n",
        "data=pd.read_csv(url)\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "1Aof5JHn3xFp",
        "outputId": "40920bec-2e8c-4bd4-e680-56da5e36a6ce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   URLLength  DomainLength  TLD  URLSimilarityIndex  CharContinuationRate  \\\n",
              "0         31            24  231               100.0              1.000000   \n",
              "1         23            16  254               100.0              0.666667   \n",
              "2         29            22  647               100.0              0.866667   \n",
              "3         26            19  231               100.0              1.000000   \n",
              "4         33            26  503               100.0              1.000000   \n",
              "\n",
              "   TLDLegitimateProb  URLCharProb  TLDLength  NoOfSubDomain  HasObfuscation  \\\n",
              "0           0.522907     0.061933          3              1               0   \n",
              "1           0.032650     0.050207          2              1               0   \n",
              "2           0.028555     0.064129          2              2               0   \n",
              "3           0.522907     0.057606          3              1               0   \n",
              "4           0.079963     0.059441          3              1               0   \n",
              "\n",
              "   ...  Pay  Crypto  HasCopyrightInfo  NoOfImage  NoOfCSS  NoOfJS  \\\n",
              "0  ...    0       0                 1         34       20      28   \n",
              "1  ...    0       0                 1         50        9       8   \n",
              "2  ...    0       0                 1         10        2       7   \n",
              "3  ...    1       1                 1          3       27      15   \n",
              "4  ...    1       0                 1        244       15      34   \n",
              "\n",
              "   NoOfSelfRef  NoOfEmptyRef  NoOfExternalRef  label  \n",
              "0          119             0              124      1  \n",
              "1           39             0              217      1  \n",
              "2           42             2                5      1  \n",
              "3           22             1               31      1  \n",
              "4           72             1               85      1  \n",
              "\n",
              "[5 rows x 49 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c2ee2e92-15a3-4ab8-956e-1603f251ada1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URLLength</th>\n",
              "      <th>DomainLength</th>\n",
              "      <th>TLD</th>\n",
              "      <th>URLSimilarityIndex</th>\n",
              "      <th>CharContinuationRate</th>\n",
              "      <th>TLDLegitimateProb</th>\n",
              "      <th>URLCharProb</th>\n",
              "      <th>TLDLength</th>\n",
              "      <th>NoOfSubDomain</th>\n",
              "      <th>HasObfuscation</th>\n",
              "      <th>...</th>\n",
              "      <th>Pay</th>\n",
              "      <th>Crypto</th>\n",
              "      <th>HasCopyrightInfo</th>\n",
              "      <th>NoOfImage</th>\n",
              "      <th>NoOfCSS</th>\n",
              "      <th>NoOfJS</th>\n",
              "      <th>NoOfSelfRef</th>\n",
              "      <th>NoOfEmptyRef</th>\n",
              "      <th>NoOfExternalRef</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>31</td>\n",
              "      <td>24</td>\n",
              "      <td>231</td>\n",
              "      <td>100.0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.061933</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>34</td>\n",
              "      <td>20</td>\n",
              "      <td>28</td>\n",
              "      <td>119</td>\n",
              "      <td>0</td>\n",
              "      <td>124</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>23</td>\n",
              "      <td>16</td>\n",
              "      <td>254</td>\n",
              "      <td>100.0</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.032650</td>\n",
              "      <td>0.050207</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>50</td>\n",
              "      <td>9</td>\n",
              "      <td>8</td>\n",
              "      <td>39</td>\n",
              "      <td>0</td>\n",
              "      <td>217</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>29</td>\n",
              "      <td>22</td>\n",
              "      <td>647</td>\n",
              "      <td>100.0</td>\n",
              "      <td>0.866667</td>\n",
              "      <td>0.028555</td>\n",
              "      <td>0.064129</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "      <td>42</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>26</td>\n",
              "      <td>19</td>\n",
              "      <td>231</td>\n",
              "      <td>100.0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.057606</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>27</td>\n",
              "      <td>15</td>\n",
              "      <td>22</td>\n",
              "      <td>1</td>\n",
              "      <td>31</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>33</td>\n",
              "      <td>26</td>\n",
              "      <td>503</td>\n",
              "      <td>100.0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.079963</td>\n",
              "      <td>0.059441</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>244</td>\n",
              "      <td>15</td>\n",
              "      <td>34</td>\n",
              "      <td>72</td>\n",
              "      <td>1</td>\n",
              "      <td>85</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 49 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c2ee2e92-15a3-4ab8-956e-1603f251ada1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c2ee2e92-15a3-4ab8-956e-1603f251ada1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c2ee2e92-15a3-4ab8-956e-1603f251ada1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fcb6faf8-e3ff-469a-b7ed-56f8ae925eb1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fcb6faf8-e3ff-469a-b7ed-56f8ae925eb1')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fcb6faf8-e3ff-469a-b7ed-56f8ae925eb1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349
        },
        "id": "eXtjfbVN4N_f",
        "outputId": "8e5cc1ad-1681-4647-878c-07aa64294d02"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           URLLength   DomainLength            TLD  URLSimilarityIndex  \\\n",
              "count  235795.000000  235795.000000  235795.000000       235795.000000   \n",
              "mean       34.573095      21.470396     319.494234           78.430778   \n",
              "std        41.314153       9.150793     144.660289           28.976055   \n",
              "min        13.000000       4.000000       0.000000            0.155574   \n",
              "25%        23.000000      16.000000     231.000000           57.024793   \n",
              "50%        27.000000      20.000000     231.000000          100.000000   \n",
              "75%        34.000000      24.000000     443.000000          100.000000   \n",
              "max      6097.000000     110.000000     694.000000          100.000000   \n",
              "\n",
              "       CharContinuationRate  TLDLegitimateProb    URLCharProb      TLDLength  \\\n",
              "count         235795.000000      235795.000000  235795.000000  235795.000000   \n",
              "mean               0.845508           0.260423       0.055747       2.764456   \n",
              "std                0.216632           0.251628       0.010587       0.599739   \n",
              "min                0.000000           0.000000       0.001083       2.000000   \n",
              "25%                0.680000           0.005977       0.050747       2.000000   \n",
              "50%                1.000000           0.079963       0.057970       3.000000   \n",
              "75%                1.000000           0.522907       0.062875       3.000000   \n",
              "max                1.000000           0.522907       0.090824      13.000000   \n",
              "\n",
              "       NoOfSubDomain  HasObfuscation  ...            Pay         Crypto  \\\n",
              "count  235795.000000   235795.000000  ...  235795.000000  235795.000000   \n",
              "mean        1.164758        0.002057  ...       0.237007       0.023474   \n",
              "std         0.600969        0.045306  ...       0.425247       0.151403   \n",
              "min         0.000000        0.000000  ...       0.000000       0.000000   \n",
              "25%         1.000000        0.000000  ...       0.000000       0.000000   \n",
              "50%         1.000000        0.000000  ...       0.000000       0.000000   \n",
              "75%         1.000000        0.000000  ...       0.000000       0.000000   \n",
              "max        10.000000        1.000000  ...       1.000000       1.000000   \n",
              "\n",
              "       HasCopyrightInfo      NoOfImage        NoOfCSS         NoOfJS  \\\n",
              "count     235795.000000  235795.000000  235795.000000  235795.000000   \n",
              "mean           0.486775      26.075689       6.333111      10.522305   \n",
              "std            0.499826      79.411815      74.866296      22.312192   \n",
              "min            0.000000       0.000000       0.000000       0.000000   \n",
              "25%            0.000000       0.000000       0.000000       0.000000   \n",
              "50%            0.000000       8.000000       2.000000       6.000000   \n",
              "75%            1.000000      29.000000       8.000000      15.000000   \n",
              "max            1.000000    8956.000000   35820.000000    6957.000000   \n",
              "\n",
              "         NoOfSelfRef   NoOfEmptyRef  NoOfExternalRef          label  \n",
              "count  235795.000000  235795.000000    235795.000000  235795.000000  \n",
              "mean       65.071113       2.377629        49.262516       0.571895  \n",
              "std       176.687539      17.641097       161.027430       0.494805  \n",
              "min         0.000000       0.000000         0.000000       0.000000  \n",
              "25%         0.000000       0.000000         1.000000       0.000000  \n",
              "50%        12.000000       0.000000        10.000000       1.000000  \n",
              "75%        88.000000       1.000000        57.000000       1.000000  \n",
              "max     27397.000000    4887.000000     27516.000000       1.000000  \n",
              "\n",
              "[8 rows x 49 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c3660038-e4be-4458-bb6c-099b7fadcb45\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URLLength</th>\n",
              "      <th>DomainLength</th>\n",
              "      <th>TLD</th>\n",
              "      <th>URLSimilarityIndex</th>\n",
              "      <th>CharContinuationRate</th>\n",
              "      <th>TLDLegitimateProb</th>\n",
              "      <th>URLCharProb</th>\n",
              "      <th>TLDLength</th>\n",
              "      <th>NoOfSubDomain</th>\n",
              "      <th>HasObfuscation</th>\n",
              "      <th>...</th>\n",
              "      <th>Pay</th>\n",
              "      <th>Crypto</th>\n",
              "      <th>HasCopyrightInfo</th>\n",
              "      <th>NoOfImage</th>\n",
              "      <th>NoOfCSS</th>\n",
              "      <th>NoOfJS</th>\n",
              "      <th>NoOfSelfRef</th>\n",
              "      <th>NoOfEmptyRef</th>\n",
              "      <th>NoOfExternalRef</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "      <td>235795.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>34.573095</td>\n",
              "      <td>21.470396</td>\n",
              "      <td>319.494234</td>\n",
              "      <td>78.430778</td>\n",
              "      <td>0.845508</td>\n",
              "      <td>0.260423</td>\n",
              "      <td>0.055747</td>\n",
              "      <td>2.764456</td>\n",
              "      <td>1.164758</td>\n",
              "      <td>0.002057</td>\n",
              "      <td>...</td>\n",
              "      <td>0.237007</td>\n",
              "      <td>0.023474</td>\n",
              "      <td>0.486775</td>\n",
              "      <td>26.075689</td>\n",
              "      <td>6.333111</td>\n",
              "      <td>10.522305</td>\n",
              "      <td>65.071113</td>\n",
              "      <td>2.377629</td>\n",
              "      <td>49.262516</td>\n",
              "      <td>0.571895</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>41.314153</td>\n",
              "      <td>9.150793</td>\n",
              "      <td>144.660289</td>\n",
              "      <td>28.976055</td>\n",
              "      <td>0.216632</td>\n",
              "      <td>0.251628</td>\n",
              "      <td>0.010587</td>\n",
              "      <td>0.599739</td>\n",
              "      <td>0.600969</td>\n",
              "      <td>0.045306</td>\n",
              "      <td>...</td>\n",
              "      <td>0.425247</td>\n",
              "      <td>0.151403</td>\n",
              "      <td>0.499826</td>\n",
              "      <td>79.411815</td>\n",
              "      <td>74.866296</td>\n",
              "      <td>22.312192</td>\n",
              "      <td>176.687539</td>\n",
              "      <td>17.641097</td>\n",
              "      <td>161.027430</td>\n",
              "      <td>0.494805</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>13.000000</td>\n",
              "      <td>4.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.155574</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.001083</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>23.000000</td>\n",
              "      <td>16.000000</td>\n",
              "      <td>231.000000</td>\n",
              "      <td>57.024793</td>\n",
              "      <td>0.680000</td>\n",
              "      <td>0.005977</td>\n",
              "      <td>0.050747</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>27.000000</td>\n",
              "      <td>20.000000</td>\n",
              "      <td>231.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.079963</td>\n",
              "      <td>0.057970</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>8.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>6.000000</td>\n",
              "      <td>12.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>34.000000</td>\n",
              "      <td>24.000000</td>\n",
              "      <td>443.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.062875</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>29.000000</td>\n",
              "      <td>8.000000</td>\n",
              "      <td>15.000000</td>\n",
              "      <td>88.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>57.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>6097.000000</td>\n",
              "      <td>110.000000</td>\n",
              "      <td>694.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.090824</td>\n",
              "      <td>13.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>8956.000000</td>\n",
              "      <td>35820.000000</td>\n",
              "      <td>6957.000000</td>\n",
              "      <td>27397.000000</td>\n",
              "      <td>4887.000000</td>\n",
              "      <td>27516.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 49 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c3660038-e4be-4458-bb6c-099b7fadcb45')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c3660038-e4be-4458-bb6c-099b7fadcb45 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c3660038-e4be-4458-bb6c-099b7fadcb45');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-245e2996-e3fe-4358-a9ea-e463e150536f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-245e2996-e3fe-4358-a9ea-e463e150536f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-245e2996-e3fe-4358-a9ea-e463e150536f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.shape\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "550XgGYM4dTH",
        "outputId": "e6200b7e-d4f9-47b1-e562-1c8c73346bb6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(235795, 49)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oAvWwHHX4hti",
        "outputId": "a43f8f43-9550-4d6b-e49b-6e02d4b9f06b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['URLLength', 'DomainLength', 'TLD', 'URLSimilarityIndex',\n",
              "       'CharContinuationRate', 'TLDLegitimateProb', 'URLCharProb', 'TLDLength',\n",
              "       'NoOfSubDomain', 'HasObfuscation', 'ObfuscationRatio',\n",
              "       'NoOfLettersInURL', 'LetterRatioInURL', 'NoOfDegitsInURL',\n",
              "       'DegitRatioInURL', 'NoOfEqualsInURL', 'NoOfQMarkInURL',\n",
              "       'NoOfOtherSpecialCharsInURL', 'SpacialCharRatioInURL', 'IsHTTPS',\n",
              "       'LineOfCode', 'LargestLineLength', 'HasTitle', 'DomainTitleMatchScore',\n",
              "       'URLTitleMatchScore', 'HasFavicon', 'Robots', 'IsResponsive',\n",
              "       'NoOfURLRedirect', 'NoOfSelfRedirect', 'HasDescription', 'NoOfPopup',\n",
              "       'NoOfiFrame', 'HasExternalFormSubmit', 'HasSocialNet',\n",
              "       'HasSubmitButton', 'HasHiddenFields', 'HasPasswordField', 'Bank', 'Pay',\n",
              "       'Crypto', 'HasCopyrightInfo', 'NoOfImage', 'NoOfCSS', 'NoOfJS',\n",
              "       'NoOfSelfRef', 'NoOfEmptyRef', 'NoOfExternalRef', 'label'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Removing outlier from individual column\n"
      ],
      "metadata": {
        "id": "YoMXFKH9Lpni"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# removing outliers using z-score methods\n",
        "upper_limit = data['URLLength'].mean() + 3*data['URLLength'].std()\n",
        "lower_limit = data['URLLength'].mean() - 3*data['URLLength'].std()"
      ],
      "metadata": {
        "id": "bYnAVyiD4mrk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.loc[(data['URLLength'] >upper_limit) | (data['URLLength'] <lower_limit)  ]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "id": "Md_IDSTA5tgf",
        "outputId": "b4692407-b396-4890-dd78-74231a7fcb1f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        URLLength  DomainLength  TLD  URLSimilarityIndex  \\\n",
              "401           191            13  116            4.813395   \n",
              "458           184            27  231            5.672426   \n",
              "939           251            52  632            5.877449   \n",
              "1093          188            14  231            6.619948   \n",
              "1205          163            13   18            5.769865   \n",
              "...           ...           ...  ...                 ...   \n",
              "234393       1427            25  481            1.411391   \n",
              "234994        199            19  226            9.988233   \n",
              "235468        487            22  231            3.144015   \n",
              "235634        426            21  231            2.637135   \n",
              "235722        198            21  231            7.922275   \n",
              "\n",
              "        CharContinuationRate  TLDLegitimateProb  URLCharProb  TLDLength  \\\n",
              "401                 0.400000           0.000000     0.042737          2   \n",
              "458                 0.391304           0.522907     0.036274          3   \n",
              "939                 0.363636           0.000275     0.046974          3   \n",
              "1093                1.000000           0.522907     0.035170          3   \n",
              "1205                0.444444           0.000000     0.046133          3   \n",
              "...                      ...                ...          ...        ...   \n",
              "234393              0.809524           0.038420     0.031619          3   \n",
              "234994              1.000000           0.005977     0.035174          2   \n",
              "235468              0.555556           0.522907     0.032263          3   \n",
              "235634              0.411765           0.522907     0.037928          3   \n",
              "235722              0.647059           0.522907     0.043377          3   \n",
              "\n",
              "        NoOfSubDomain  HasObfuscation  ...  Pay  Crypto  HasCopyrightInfo  \\\n",
              "401                 2               0  ...    0       0                 0   \n",
              "458                 3               0  ...    0       0                 0   \n",
              "939                 5               0  ...    0       0                 1   \n",
              "1093                1               1  ...    0       0                 0   \n",
              "1205                2               0  ...    0       0                 0   \n",
              "...               ...             ...  ...  ...     ...               ...   \n",
              "234393              2               0  ...    0       0                 0   \n",
              "234994              0               0  ...    0       0                 0   \n",
              "235468              1               0  ...    0       0                 1   \n",
              "235634              1               0  ...    0       0                 0   \n",
              "235722              0               0  ...    0       0                 0   \n",
              "\n",
              "        NoOfImage  NoOfCSS  NoOfJS  NoOfSelfRef  NoOfEmptyRef  \\\n",
              "401             0        0       0            0             0   \n",
              "458             0        0       0            1             0   \n",
              "939            10        2       1            0             0   \n",
              "1093            0        0       0            1             0   \n",
              "1205            0        0       0            0             0   \n",
              "...           ...      ...     ...          ...           ...   \n",
              "234393          0        0       0            0             0   \n",
              "234994          2        1       0            0             1   \n",
              "235468          4       10      15           23             7   \n",
              "235634          0        0       0            0             0   \n",
              "235722         18        1       0            0             0   \n",
              "\n",
              "        NoOfExternalRef  label  \n",
              "401                   0      0  \n",
              "458                   1      0  \n",
              "939                   0      0  \n",
              "1093                  1      0  \n",
              "1205                  0      0  \n",
              "...                 ...    ...  \n",
              "234393                0      0  \n",
              "234994                0      0  \n",
              "235468               24      0  \n",
              "235634                0      0  \n",
              "235722                0      0  \n",
              "\n",
              "[2198 rows x 49 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ef5ac086-796e-4bcc-8509-bbdea03fa49f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URLLength</th>\n",
              "      <th>DomainLength</th>\n",
              "      <th>TLD</th>\n",
              "      <th>URLSimilarityIndex</th>\n",
              "      <th>CharContinuationRate</th>\n",
              "      <th>TLDLegitimateProb</th>\n",
              "      <th>URLCharProb</th>\n",
              "      <th>TLDLength</th>\n",
              "      <th>NoOfSubDomain</th>\n",
              "      <th>HasObfuscation</th>\n",
              "      <th>...</th>\n",
              "      <th>Pay</th>\n",
              "      <th>Crypto</th>\n",
              "      <th>HasCopyrightInfo</th>\n",
              "      <th>NoOfImage</th>\n",
              "      <th>NoOfCSS</th>\n",
              "      <th>NoOfJS</th>\n",
              "      <th>NoOfSelfRef</th>\n",
              "      <th>NoOfEmptyRef</th>\n",
              "      <th>NoOfExternalRef</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>401</th>\n",
              "      <td>191</td>\n",
              "      <td>13</td>\n",
              "      <td>116</td>\n",
              "      <td>4.813395</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.042737</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458</th>\n",
              "      <td>184</td>\n",
              "      <td>27</td>\n",
              "      <td>231</td>\n",
              "      <td>5.672426</td>\n",
              "      <td>0.391304</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.036274</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>939</th>\n",
              "      <td>251</td>\n",
              "      <td>52</td>\n",
              "      <td>632</td>\n",
              "      <td>5.877449</td>\n",
              "      <td>0.363636</td>\n",
              "      <td>0.000275</td>\n",
              "      <td>0.046974</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1093</th>\n",
              "      <td>188</td>\n",
              "      <td>14</td>\n",
              "      <td>231</td>\n",
              "      <td>6.619948</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.035170</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1205</th>\n",
              "      <td>163</td>\n",
              "      <td>13</td>\n",
              "      <td>18</td>\n",
              "      <td>5.769865</td>\n",
              "      <td>0.444444</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.046133</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234393</th>\n",
              "      <td>1427</td>\n",
              "      <td>25</td>\n",
              "      <td>481</td>\n",
              "      <td>1.411391</td>\n",
              "      <td>0.809524</td>\n",
              "      <td>0.038420</td>\n",
              "      <td>0.031619</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234994</th>\n",
              "      <td>199</td>\n",
              "      <td>19</td>\n",
              "      <td>226</td>\n",
              "      <td>9.988233</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.005977</td>\n",
              "      <td>0.035174</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>235468</th>\n",
              "      <td>487</td>\n",
              "      <td>22</td>\n",
              "      <td>231</td>\n",
              "      <td>3.144015</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.032263</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>10</td>\n",
              "      <td>15</td>\n",
              "      <td>23</td>\n",
              "      <td>7</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>235634</th>\n",
              "      <td>426</td>\n",
              "      <td>21</td>\n",
              "      <td>231</td>\n",
              "      <td>2.637135</td>\n",
              "      <td>0.411765</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.037928</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>235722</th>\n",
              "      <td>198</td>\n",
              "      <td>21</td>\n",
              "      <td>231</td>\n",
              "      <td>7.922275</td>\n",
              "      <td>0.647059</td>\n",
              "      <td>0.522907</td>\n",
              "      <td>0.043377</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>18</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2198 rows × 49 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ef5ac086-796e-4bcc-8509-bbdea03fa49f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ef5ac086-796e-4bcc-8509-bbdea03fa49f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ef5ac086-796e-4bcc-8509-bbdea03fa49f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a048056a-b075-4c57-a57b-63c939ad9b06\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a048056a-b075-4c57-a57b-63c939ad9b06')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a048056a-b075-4c57-a57b-63c939ad9b06 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#trimming the outliers\n",
        "\n",
        "new_data=data.loc[(data['URLLength'] <upper_limit) & (data['URLLength'] >lower_limit)  ]"
      ],
      "metadata": {
        "id": "AlYGMtXR60cA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('before removing outliers: ', len(data))\n",
        "print('new data: ', len(new_data))\n",
        "print('outliers: ', len(data)- len(new_data))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-h5dS4Dr71Za",
        "outputId": "5f3cd5f2-9eec-4045-937f-e56a95836b12"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "before removing outliers:  235795\n",
            "new data:  233597\n",
            "outliers:  2198\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Removing outlier from multiple columns"
      ],
      "metadata": {
        "id": "LhBM87YwMBFi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Function to detect and remove outliers\n",
        "def remove_outliers(df, columns, threshold=3):\n",
        "    z_scores = np.abs((df[columns] - df[columns].mean()) / df[columns].std())\n",
        "    mask = (z_scores < threshold).all(axis=1)\n",
        "    return df[mask]\n",
        "\n",
        "# Specify columns to detect outliers\n",
        "columns_to_check = ['URLLength','DomainLength']\n",
        "\n",
        "# Remove outliers\n",
        "data_no_outliers = remove_outliers(data, columns_to_check)\n",
        "\n",
        "# Display the result\n",
        "print(\"Original DataFrame shape:\", data.shape)\n",
        "print(\"DataFrame shape after removing outliers:\", data_no_outliers.shape)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "53E4R5Kx-41z",
        "outputId": "a2695201-1218-4870-9bca-9587c515b4f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original DataFrame shape: (235795, 49)\n",
            "DataFrame shape after removing outliers: (230215, 49)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Removing outlier from entire dataset"
      ],
      "metadata": {
        "id": "kVcYB7ZKMbhP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to detect and remove outliers\n",
        "def remove_outliers(df, threshold=3):\n",
        "    z_scores = np.abs((df - df.mean()) / df.std())\n",
        "    mask = (z_scores < threshold).all(axis=1)\n",
        "    return df[mask]\n",
        "\n",
        "# Remove outliers\n",
        "data_no_outliers = remove_outliers(data)\n",
        "\n",
        "# Display the result\n",
        "print(\"Original DataFrame shape:\", data.shape)\n",
        "print(\"DataFrame shape after removing outliers:\", data_no_outliers.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G0PmQOM0_4LS",
        "outputId": "397b7ebc-c08c-4fca-fff7-baf8df29663a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original DataFrame shape: (235795, 49)\n",
            "DataFrame shape after removing outliers: (176278, 49)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "# Function to detect and remove outliers using IQR method\n",
        "def remove_outliers_iqr(df, k=1.5):\n",
        "    Q1 = df.quantile(0.25)\n",
        "    Q3 = df.quantile(0.75)\n",
        "    IQR = Q3 - Q1\n",
        "    lower_bound = Q1 - k * IQR\n",
        "    upper_bound = Q3 + k * IQR\n",
        "    return df[~((df < lower_bound) | (df > upper_bound)).any(axis=1)]\n",
        "\n",
        "# Remove outliers\n",
        "data_no_outliers = remove_outliers_iqr(data)\n",
        "\n",
        "# Display the result\n",
        "print(\"Original DataFrame shape:\", data.shape)\n",
        "print(\"DataFrame shape after removing outliers using IQR method:\", data_no_outliers.shape)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BNgj2yeqAuub",
        "outputId": "04c802d5-093f-4e28-b6a8-f762d0998486"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original DataFrame shape: (235795, 49)\n",
            "DataFrame shape after removing outliers using IQR method: (26607, 49)\n"
          ]
        }
      ]
    }
  ]
}