main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "019ca3ac-06e5-4586-8506-b0024bcff255",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "from tqdm import tqdm\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dcd6ccd3-1f3a-4bde-9959-54dccfe9664d",
   "metadata": {},
   "outputs": [],
   "source": [
    "hosts = pd.read_csv(\"data/host.csv\", header=None)\n",
    "hosts.columns = [\"url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "697fed42-9fcc-4db9-a785-5befcd2ac8bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>api.youla.io</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>favicon.yandex.net</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w-74721.fp.kaspersky-labs.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>questtime.net</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>passport-authproxy.taxi.yandex.net</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  url\n",
       "0                        api.youla.io\n",
       "1                  favicon.yandex.net\n",
       "2       w-74721.fp.kaspersky-labs.com\n",
       "3                       questtime.net\n",
       "4  passport-authproxy.taxi.yandex.net"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hosts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "25ab9a21-3403-4007-8596-462141113214",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(199944, 1)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hosts.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72d628bc-e7cf-404d-95ee-6b633eb08558",
   "metadata": {},
   "source": [
    "### Анализ датасета\n",
    "* Очень много хостов, которые являются облаками крупных компаний\n",
    "* Явный дисбаланс в сторону \"технических\" хостов"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60dcf7d9-ed27-45ea-a35b-4bac110034fc",
   "metadata": {},
   "source": [
    "#### В качестве train сета будем использовать собранный командой датасет\n",
    "target: \n",
    "* 0 - ответ, привычный для пользователя\n",
    "* 1 - ответ, соответствующий служебному (техническому) взаимодействию \n",
    "\n",
    "Также в качестве попытки устранить дисбаланс на обучающей выборке, был добавлен сабсет, содержащий обычные пользовательские сайты (спарсили по запросу популярных сайтов в поисковике)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1df860a7-b4a3-41c7-9e89-e6a98931cb90",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b2d9a62c-9f51-4483-836a-b8f087eaaff8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>api.youla.io</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>favicon.yandex.net</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w-74721.fp.kaspersky-labs.com</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>questtime.net</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>passport-authproxy.taxi.yandex.net</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  url  target\n",
       "0                        api.youla.io       0\n",
       "1                  favicon.yandex.net       1\n",
       "2       w-74721.fp.kaspersky-labs.com       1\n",
       "3                       questtime.net       0\n",
       "4  passport-authproxy.taxi.yandex.net       1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "58bfba5a-c6e7-48cd-9571-526d2df6b81c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1426, 2)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a75ae9b8-fd3e-47f7-9060-fbcd7255a96a",
   "metadata": {},
   "source": [
    "#### Посмортим, какие компоненты чаще всего встречаются в url'ах"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2394693e-67b7-4d45-a8ed-2b99e22d7c68",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_components(urls):\n",
    "    words = []\n",
    "    for url in tqdm(urls):\n",
    "        words += url.split(\".\") \n",
    "    words = pd.Series(words)\n",
    "    return words.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c2364c61-08f3-4f1b-ae40-cd0cce0b64b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 199944/199944 [00:00<00:00, 891005.46it/s]\n"
     ]
    }
   ],
   "source": [
    "top_components = get_top_components(hosts[\"url\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9c2ea21f-e2d4-4564-aace-183785c0c372",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "com                  81315\n",
       "ru                   47350\n",
       "net                  22317\n",
       "googlesyndication    19807\n",
       "safeframe            19802\n",
       "yandex               10472\n",
       "0                     8436\n",
       "mts                   8194\n",
       "match                 8013\n",
       "verify                7940\n",
       "whiteboxdigital       7111\n",
       "mitdmp                7110\n",
       "org                   5992\n",
       "local                 5457\n",
       "me                    4779\n",
       "io                    4689\n",
       "www                   4653\n",
       "cc                    4140\n",
       "googlevideo           4137\n",
       "pushy                 3661\n",
       "biz                   3570\n",
       "in                    3373\n",
       "cdn                   2767\n",
       "2                     2194\n",
       "arpa                  2110\n",
       "in-addr               2109\n",
       "1                     2103\n",
       "v1                    1787\n",
       "a                     1750\n",
       "e                     1747\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_components.head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "901710bb-f8ec-42b0-ae87-788a352e060e",
   "metadata": {},
   "source": [
    "#### Для train датасета добавим фичи, которые не требуют выхода в онлай"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7a1c3c01-9dbc-4205-9c5e-9148678c4335",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_offline_features(t):\n",
    "    df = t.copy()\n",
    "    df[\"start_with_api\"] = df[\"url\"].str.contains(\"^api\", regex=True).astype(int)\n",
    "    df[\"has_userapi\"] = df[\"url\"].str.contains(\"userapi\").astype(int)\n",
    "    df[\"has_googleapis\"] = df[\"url\"].str.contains(\"googleapis\").astype(int)\n",
    "    df[\"size_of_url\"] = df[\"url\"].apply(lambda x: len(x))\n",
    "    df[\"size_of_url_split\"] = df[\"url\"].apply(lambda x: len(x.split(\".\")))\n",
    "    df[\"clear_url\"] =  df[\"url\"].apply(lambda x: x.replace(\".\", \" \"))\n",
    "    df[\"minus_count\"] = df[\"url\"].str.count(\"-\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "105c71ac-3f8a-42f5-b17f-1617fc0794c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = get_offline_features(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8129fbfb-5ac2-47b2-95df-df4601b06123",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>target</th>\n",
       "      <th>start_with_api</th>\n",
       "      <th>has_userapi</th>\n",
       "      <th>has_googleapis</th>\n",
       "      <th>size_of_url</th>\n",
       "      <th>size_of_url_split</th>\n",
       "      <th>clear_url</th>\n",
       "      <th>minus_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>api.youla.io</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>api youla io</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>favicon.yandex.net</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>favicon yandex net</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w-74721.fp.kaspersky-labs.com</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>w-74721 fp kaspersky-labs com</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>questtime.net</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>questtime net</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>passport-authproxy.taxi.yandex.net</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>passport-authproxy taxi yandex net</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  url  target  start_with_api  has_userapi  \\\n",
       "0                        api.youla.io       0               1            0   \n",
       "1                  favicon.yandex.net       1               0            0   \n",
       "2       w-74721.fp.kaspersky-labs.com       1               0            0   \n",
       "3                       questtime.net       0               0            0   \n",
       "4  passport-authproxy.taxi.yandex.net       1               0            0   \n",
       "\n",
       "   has_googleapis  size_of_url  size_of_url_split  \\\n",
       "0               0           12                  3   \n",
       "1               0           18                  3   \n",
       "2               0           29                  4   \n",
       "3               0           13                  2   \n",
       "4               0           34                  4   \n",
       "\n",
       "                            clear_url  minus_count  \n",
       "0                        api youla io            0  \n",
       "1                  favicon yandex net            0  \n",
       "2       w-74721 fp kaspersky-labs com            2  \n",
       "3                       questtime net            0  \n",
       "4  passport-authproxy taxi yandex net            1  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0e090a2-f372-4f96-8c2a-d6d579629b1d",
   "metadata": {},
   "source": [
    "#### Теперь создадим модель на основе CatBoost\n",
    "Обучать модель мы будем несколько раз, поэтому обернём её в функцию"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3a7b007b-57c8-4eea-a6a9-0001d0d65285",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(train, cat_features, text_features):\n",
    "    \n",
    "    # Сначала разделим выборку на две части\n",
    "    x_train, x_validation, y_train, y_validation = train_test_split(train.drop(columns=[\"url\", \"target\"]), \n",
    "                                                                train[\"target\"], \n",
    "                                                                stratify=train[\"target\"],\n",
    "                                                                test_size=0.33, \n",
    "                                                                random_state=127\n",
    "                                                               )\n",
    "    # Создадим экземпляр модели\n",
    "    # CatBoost выполнит препроцессинг для текста (колонка clear_url) за нас, главное - передать настройки\n",
    "    # Аналогично нет необходимости задумываться о категориальных фичах\n",
    "    model = CatBoostClassifier(iterations=500,\n",
    "#                             depth = 6,\n",
    "                            learning_rate = 0.07,\n",
    "#                             l2_leaf_reg = 4,\n",
    "                            eval_metric=\"F1\",\n",
    "                            loss_function = \"Logloss\",\n",
    "                            task_type=\"GPU\",\n",
    "                            # fold_permutation_block = 2,\n",
    "                            # fold_len_multiplier = 1.5,\n",
    "                            leaf_estimation_iterations = 10,\n",
    "                            max_ctr_complexity = 5,\n",
    "                            random_seed= 127,\n",
    "                               \n",
    "                            cat_features = cat_features,\n",
    "                            text_features = [\"clear_url\"],\n",
    "                               \n",
    "                            text_processing = {\n",
    "                                \"tokenizers\" : [{\n",
    "                                \"tokenizer_id\" : \"Space\",\n",
    "                                \"separator_type\" : \"ByDelimiter\",\n",
    "                                \"delimiter\" : \" \"\n",
    "                            }],\n",
    "\n",
    "                            \"dictionaries\" : [{\n",
    "                                \"dictionary_id\" : \"BiGram\",\n",
    "                                \"token_level_type\": \"Letter\",\n",
    "                                \"max_dictionary_size\" : \"150000\",\n",
    "                                \"occurrence_lower_bound\" : \"1\",\n",
    "                                \"gram_order\" : \"2\"\n",
    "                            }, {\n",
    "                                \"dictionary_id\" : \"Trigram\",\n",
    "                                \"max_dictionary_size\" : \"150000\",\n",
    "                                \"token_level_type\": \"Letter\",\n",
    "                                \"occurrence_lower_bound\" : \"1\",\n",
    "                                \"gram_order\" : \"3\"\n",
    "                            }, {\n",
    "                                \"dictionary_id\" : \"Fourgram\",\n",
    "                                \"max_dictionary_size\" : \"150000\",\n",
    "                                \"token_level_type\": \"Letter\",\n",
    "                                \"occurrence_lower_bound\" : \"1\",\n",
    "                                \"gram_order\" : \"4\"\n",
    "                            }, {\n",
    "                                \"dictionary_id\" : \"Word\",\n",
    "                                \"max_dictionary_size\" : \"30000\",\n",
    "                                \"occurrence_lower_bound\" : \"3\",\n",
    "                                \"gram_order\" : \"1\"\n",
    "                            }, {\n",
    "                                \"dictionary_id\" : \"Fivegram\",\n",
    "                                \"max_dictionary_size\" : \"150000\",\n",
    "                                \"token_level_type\": \"Letter\",\n",
    "                                \"occurrence_lower_bound\" : \"1\",\n",
    "                                \"gram_order\" : \"5\"\n",
    "                            }, {\n",
    "                                \"dictionary_id\" : \"Sixgram\",\n",
    "                                \"max_dictionary_size\" : \"150000\",\n",
    "                                \"token_level_type\": \"Letter\",\n",
    "                                \"occurrence_lower_bound\" : \"1\",\n",
    "                                \"gram_order\" : \"6\"\n",
    "                            }\n",
    "                            ],\n",
    "\n",
    "                            \"feature_processing\" : {\n",
    "                                \"default\" : [\n",
    "                                        {\n",
    "                                        \"dictionaries_names\" : [\"BiGram\", \"Trigram\", \"Fourgram\", \"Word\", \"Fivegram\", \"Sixgram\"],\n",
    "                                        \"feature_calcers\" : [\"BoW\"],\n",
    "                                        \"tokenizers_names\" : [\"Space\"]\n",
    "                                    },{\n",
    "                                        \"dictionaries_names\" : [\"BiGram\", \"Trigram\", \"Fourgram\", \"Word\", \"Fivegram\", \"Sixgram\"],\n",
    "                                        \"feature_calcers\" : [\"NaiveBayes\"],\n",
    "                                        \"tokenizers_names\" : [\"Space\"]\n",
    "                                    },{\n",
    "                                        \"dictionaries_names\" : [ \"BiGram\", \"Trigram\", \"Fourgram\", \"Fivegram\", \"Sixgram\"],\n",
    "                                        \"feature_calcers\" : [\"BM25\"],\n",
    "                                        \"tokenizers_names\" : [\"Space\"]\n",
    "                                    }\n",
    "                                ],\n",
    "                            }\n",
    "                        }\n",
    "      )\n",
    "    \n",
    "    #Запустим обучение модели\n",
    "    model.fit(x_train, \n",
    "          y_train, \n",
    "          eval_set=(x_validation, y_validation), \n",
    "          use_best_model=True, \n",
    "#           early_stopping_rounds=300,  \n",
    "#           plot=True, \n",
    "          verbose=50\n",
    "          )\n",
    "    \n",
    "    print(\"\\nVS\\nConst 1: \", f1_score(y_validation, np.ones(len(y_validation))))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b818b309-baf6-45bc-acfc-d4ee9895eb13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.8788660\ttest: 0.8825065\tbest: 0.8825065 (0)\ttotal: 116ms\tremaining: 57.7s\n",
      "50:\tlearn: 0.9502762\ttest: 0.8975069\tbest: 0.9005525 (40)\ttotal: 4.86s\tremaining: 42.8s\n",
      "100:\tlearn: 0.9625520\ttest: 0.9060773\tbest: 0.9085873 (78)\ttotal: 8.79s\tremaining: 34.7s\n",
      "150:\tlearn: 0.9762238\ttest: 0.9060773\tbest: 0.9090909 (104)\ttotal: 12.9s\tremaining: 29.9s\n",
      "200:\tlearn: 0.9817159\ttest: 0.8938547\tbest: 0.9090909 (104)\ttotal: 17.2s\tremaining: 25.6s\n",
      "250:\tlearn: 0.9915254\ttest: 0.8969359\tbest: 0.9090909 (104)\ttotal: 21.4s\tremaining: 21.3s\n",
      "300:\tlearn: 0.9929279\ttest: 0.8932584\tbest: 0.9090909 (104)\ttotal: 25.6s\tremaining: 17s\n",
      "350:\tlearn: 0.9971671\ttest: 0.8938547\tbest: 0.9090909 (104)\ttotal: 30.1s\tremaining: 12.8s\n",
      "400:\tlearn: 0.9971671\ttest: 0.8907563\tbest: 0.9090909 (104)\ttotal: 34.5s\tremaining: 8.53s\n",
      "450:\tlearn: 0.9985856\ttest: 0.8901408\tbest: 0.9090909 (104)\ttotal: 38.9s\tremaining: 4.22s\n",
      "499:\tlearn: 0.9985856\ttest: 0.8901408\tbest: 0.9090909 (104)\ttotal: 43.2s\tremaining: 0us\n",
      "bestTest = 0.9090909091\n",
      "bestIteration = 104\n",
      "Shrink model to first 105 iterations.\n",
      "\n",
      "VS\n",
      "Const 1:  0.541795665634675\n"
     ]
    }
   ],
   "source": [
    "cat_features = ['start_with_api', 'has_userapi', 'has_googleapis']\n",
    "text_features = [\"clear_url\"]\n",
    "\n",
    "model_v1 = get_model(train, cat_features, text_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0b0a419c-ce14-4025-9150-2f7104d73528",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature Id</th>\n",
       "      <th>Importances</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>size_of_url_split</td>\n",
       "      <td>48.573407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clear_url</td>\n",
       "      <td>43.578420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>size_of_url</td>\n",
       "      <td>7.577916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>minus_count</td>\n",
       "      <td>0.270257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>start_with_api</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Feature Id  Importances\n",
       "0  size_of_url_split    48.573407\n",
       "1          clear_url    43.578420\n",
       "2        size_of_url     7.577916\n",
       "3        minus_count     0.270257\n",
       "4     start_with_api     0.000000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_v1.get_feature_importance(prettified=True).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39ea3150-ce2b-4182-a6e3-1b4e747003db",
   "metadata": {},
   "source": [
    "#### Средний результат нескольких запусков: 0.89032\n",
    "* Против 0.54180 при константном предсказании 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5195621e-0f40-4b5d-9b46-3ae0f7a5fcb7",
   "metadata": {},
   "source": [
    "### 1 вариант улучшения модели\n",
    "- Используя обученную модель, предсказываем метки класса оставшихся записей из hosts\n",
    "- Используем полученный датасет для обучения новой модели\n",
    "- Проверяем её качество на исходном датасете (собранном участниками команды)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0fc97f2c-b5d8-4b0d-a0f2-6d6438d4746c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape:  (199058, 8)\n"
     ]
    }
   ],
   "source": [
    "train_v2 = get_offline_features(hosts[~hosts[\"url\"].isin(train[\"url\"])])\n",
    "print(\"Shape: \",train_v2.shape)\n",
    "predict = model_v1.predict(train_v2.drop(columns=[\"url\"]))\n",
    "train_v2[\"target\"] = predict\n",
    "\n",
    "cat_features = ['start_with_api', 'has_userapi', 'has_googleapis']\n",
    "text_features = [\"clear_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cf65817e-2376-4cb5-88a8-bd3f7afa61e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9664572\ttest: 0.9691368\tbest: 0.9691368 (0)\ttotal: 114ms\tremaining: 56.7s\n",
      "50:\tlearn: 0.9740429\ttest: 0.9755576\tbest: 0.9755576 (50)\ttotal: 4.83s\tremaining: 42.5s\n",
      "100:\tlearn: 0.9772020\ttest: 0.9779606\tbest: 0.9779606 (99)\ttotal: 8.87s\tremaining: 35s\n",
      "150:\tlearn: 0.9788014\ttest: 0.9796558\tbest: 0.9796558 (150)\ttotal: 12.9s\tremaining: 29.8s\n",
      "200:\tlearn: 0.9803010\ttest: 0.9809753\tbest: 0.9809753 (200)\ttotal: 16.9s\tremaining: 25.1s\n",
      "250:\tlearn: 0.9813624\ttest: 0.9816421\tbest: 0.9816421 (250)\ttotal: 20.7s\tremaining: 20.5s\n",
      "300:\tlearn: 0.9823618\ttest: 0.9822627\tbest: 0.9822627 (300)\ttotal: 24.5s\tremaining: 16.2s\n",
      "350:\tlearn: 0.9829798\ttest: 0.9827042\tbest: 0.9827042 (350)\ttotal: 28.4s\tremaining: 12.1s\n",
      "400:\tlearn: 0.9836181\ttest: 0.9830171\tbest: 0.9830282 (395)\ttotal: 32.2s\tremaining: 7.95s\n",
      "450:\tlearn: 0.9842085\ttest: 0.9833707\tbest: 0.9833707 (450)\ttotal: 35.9s\tremaining: 3.9s\n",
      "499:\tlearn: 0.9846707\ttest: 0.9835768\tbest: 0.9836177 (488)\ttotal: 39.5s\tremaining: 0us\n",
      "bestTest = 0.9836176658\n",
      "bestIteration = 488\n",
      "Shrink model to first 489 iterations.\n",
      "\n",
      "VS\n",
      "Const 1:  0.8641236685572002\n",
      "\n",
      "\n",
      "Final score:  0.926605504587156\n",
      "Constant prediction 1:  0.5411764705882353\n"
     ]
    }
   ],
   "source": [
    "model_v2 = get_model(train_v2, cat_features, text_features)\n",
    "\n",
    "predicts = model_v2.predict(train.drop(columns=[\"url\", \"target\"]))\n",
    "print(\"\\n\\nFinal score: \", f1_score(train[\"target\"], predicts))\n",
    "print(\"Constant prediction 1: \", f1_score(train[\"target\"], np.ones(len(train[\"target\"]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62788df0-5dad-4abb-bb49-461f6992833b",
   "metadata": {},
   "source": [
    "#### Средний результат нескольких запусков: 0.89840\n",
    "* Против 0.89032 предыдущей модели\n",
    "* Против 0.54118 при константном предсказании 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8847f32c-d1f3-4355-bb6e-f77b7ac577a4",
   "metadata": {},
   "source": [
    "### 2 вариант улучшения модели\n",
    "Попробуем добавить в исходный обучающий датасет фичи, которые можно получить, обратившись к хосту  \n",
    "Из них мы выбрали:\n",
    "* Content-Type ответа\n",
    "* Status code\n",
    "* Можно ли спарсить ответ JSON'ом\n",
    "* и другие\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "911c7c04-adcd-406a-8d4e-1db4b613a998",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_online_features(t):\n",
    "    df = t.copy()\n",
    "    \n",
    "    status_codes = []\n",
    "    content_types = []\n",
    "    is_json = []\n",
    "    is_redirect = []\n",
    "    size_of_cookies = []\n",
    "    encodings = []\n",
    "    \n",
    "    def process_url(url):\n",
    "        try:\n",
    "            response = requests.get(\"http://\"+url, timeout=0.5)\n",
    "\n",
    "            try: \n",
    "                status_codes.append(response.status_code)\n",
    "            except:\n",
    "                status_codes.append(-1)\n",
    "\n",
    "            try:\n",
    "                content_types.append(response.headers[\"Content-Type\"].split(\";\")[0])\n",
    "            except:\n",
    "                content_types.append(\"none\")\n",
    "\n",
    "            try: \n",
    "                response.json()\n",
    "                is_json.append(1)\n",
    "            except:\n",
    "                is_json.append(0)\n",
    "\n",
    "            try:\n",
    "                is_redirect.append(int(response.is_redirect))\n",
    "            except:\n",
    "                is_redirect.append(-1)\n",
    "\n",
    "            try:\n",
    "                size_of_cookies.append(len(response.cookies))\n",
    "            except:\n",
    "                size_of_cookies.append(-1)\n",
    "\n",
    "            try:\n",
    "                encodings.append(response.encoding.lower() if len(response.encoding) != 0 else \"none\")\n",
    "            except: \n",
    "                encodings.append(\"none\")\n",
    "\n",
    "        except:\n",
    "            status_codes.append(-1)\n",
    "            content_types.append(\"none\")\n",
    "            is_json.append(-1)\n",
    "            is_redirect.append(-1)\n",
    "            size_of_cookies.append(-1)\n",
    "            encodings.append(\"none\")\n",
    "            \n",
    "    df[\"url\"].progress_apply(process_url)\n",
    "    \n",
    "    df[\"content_type\"] = content_types\n",
    "    df[\"status_code\"] = status_codes\n",
    "    df[\"is_json\"] = is_json\n",
    "    df[\"is_redirect\"] = is_redirect\n",
    "    df[\"encoding\"] = encodings\n",
    "    df[\"size_of_cookies\"] = size_of_cookies\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2ff24c25-dfe6-490e-a732-7a64ee67c3e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 1426/1426 [17:44<00:00,  1.34it/s]\n"
     ]
    }
   ],
   "source": [
    "train_v3 = get_online_features(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f38d23f5-683f-44f5-9eee-ca0a94f04fdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>target</th>\n",
       "      <th>start_with_api</th>\n",
       "      <th>has_userapi</th>\n",
       "      <th>has_googleapis</th>\n",
       "      <th>size_of_url</th>\n",
       "      <th>size_of_url_split</th>\n",
       "      <th>clear_url</th>\n",
       "      <th>minus_count</th>\n",
       "      <th>content_type</th>\n",
       "      <th>status_code</th>\n",
       "      <th>is_json</th>\n",
       "      <th>is_redirect</th>\n",
       "      <th>encoding</th>\n",
       "      <th>size_of_cookies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>api.youla.io</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>api youla io</td>\n",
       "      <td>0</td>\n",
       "      <td>text/html</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>utf-8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>favicon.yandex.net</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>favicon yandex net</td>\n",
       "      <td>0</td>\n",
       "      <td>none</td>\n",
       "      <td>404</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>none</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w-74721.fp.kaspersky-labs.com</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>w-74721 fp kaspersky-labs com</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>none</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>questtime.net</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>questtime net</td>\n",
       "      <td>0</td>\n",
       "      <td>text/html</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>utf-8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>passport-authproxy.taxi.yandex.net</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>passport-authproxy taxi yandex net</td>\n",
       "      <td>1</td>\n",
       "      <td>application/json</td>\n",
       "      <td>404</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>utf-8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  url  target  start_with_api  has_userapi  \\\n",
       "0                        api.youla.io       0               1            0   \n",
       "1                  favicon.yandex.net       1               0            0   \n",
       "2       w-74721.fp.kaspersky-labs.com       1               0            0   \n",
       "3                       questtime.net       0               0            0   \n",
       "4  passport-authproxy.taxi.yandex.net       1               0            0   \n",
       "\n",
       "   has_googleapis  size_of_url  size_of_url_split  \\\n",
       "0               0           12                  3   \n",
       "1               0           18                  3   \n",
       "2               0           29                  4   \n",
       "3               0           13                  2   \n",
       "4               0           34                  4   \n",
       "\n",
       "                            clear_url  minus_count      content_type  \\\n",
       "0                        api youla io            0         text/html   \n",
       "1                  favicon yandex net            0              none   \n",
       "2       w-74721 fp kaspersky-labs com            2              none   \n",
       "3                       questtime net            0         text/html   \n",
       "4  passport-authproxy taxi yandex net            1  application/json   \n",
       "\n",
       "   status_code  is_json  is_redirect encoding  size_of_cookies  \n",
       "0          200        0            0    utf-8                0  \n",
       "1          404        0            0     none                0  \n",
       "2           -1       -1           -1     none               -1  \n",
       "3          200        0            0    utf-8                1  \n",
       "4          404        1            0    utf-8                0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_v3.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ebf9d10-f6ff-4f2f-b5f8-19bc2e3f18e1",
   "metadata": {},
   "source": [
    "#### Снова обучим модель"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ea670d06-227f-4091-9244-d80261500ed6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.8056738\ttest: 0.7635328\tbest: 0.7635328 (0)\ttotal: 136ms\tremaining: 1m 7s\n",
      "50:\tlearn: 0.9116022\ttest: 0.8994413\tbest: 0.8994413 (48)\ttotal: 4.4s\tremaining: 38.8s\n",
      "100:\tlearn: 0.9279778\ttest: 0.9025070\tbest: 0.9025070 (96)\ttotal: 8.43s\tremaining: 33.3s\n",
      "150:\tlearn: 0.9374131\ttest: 0.8975069\tbest: 0.9030471 (121)\ttotal: 12.7s\tremaining: 29.3s\n",
      "200:\tlearn: 0.9415042\ttest: 0.9030471\tbest: 0.9055556 (199)\ttotal: 17s\tremaining: 25.2s\n",
      "250:\tlearn: 0.9415042\ttest: 0.9000000\tbest: 0.9055556 (199)\ttotal: 20.7s\tremaining: 20.5s\n",
      "300:\tlearn: 0.9413408\ttest: 0.8938547\tbest: 0.9055556 (199)\ttotal: 24.4s\tremaining: 16.1s\n",
      "350:\tlearn: 0.9439776\ttest: 0.8938547\tbest: 0.9055556 (199)\ttotal: 27.8s\tremaining: 11.8s\n",
      "400:\tlearn: 0.9466292\ttest: 0.8938547\tbest: 0.9055556 (199)\ttotal: 31.5s\tremaining: 7.78s\n",
      "450:\tlearn: 0.9523810\ttest: 0.8938547\tbest: 0.9055556 (199)\ttotal: 34.9s\tremaining: 3.79s\n",
      "499:\tlearn: 0.9523810\ttest: 0.8963585\tbest: 0.9055556 (199)\ttotal: 38.3s\tremaining: 0us\n",
      "bestTest = 0.9055555556\n",
      "bestIteration = 199\n",
      "Shrink model to first 200 iterations.\n",
      "\n",
      "VS\n",
      "Const 1:  0.541795665634675\n"
     ]
    }
   ],
   "source": [
    "cat_features = ['start_with_api', 'has_userapi', 'has_googleapis', 'content_type', 'status_code',\n",
    "       'is_json', 'is_redirect', 'encoding']\n",
    "text_features = [\"clear_url\"]\n",
    "\n",
    "model_v3 = get_model(train_v3, cat_features, text_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "48384abf-ef6f-4f76-88dc-27ba1414f3d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature Id</th>\n",
       "      <th>Importances</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>size_of_url_split</td>\n",
       "      <td>31.611967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clear_url</td>\n",
       "      <td>26.815538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>status_code</td>\n",
       "      <td>16.799770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>content_type</td>\n",
       "      <td>14.643261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>encoding</td>\n",
       "      <td>5.347574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>size_of_url</td>\n",
       "      <td>2.715102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>is_json</td>\n",
       "      <td>2.066788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>start_with_api</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>has_userapi</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>has_googleapis</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>minus_count</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>is_redirect</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>size_of_cookies</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Feature Id  Importances\n",
       "0   size_of_url_split    31.611967\n",
       "1           clear_url    26.815538\n",
       "2         status_code    16.799770\n",
       "3        content_type    14.643261\n",
       "4            encoding     5.347574\n",
       "5         size_of_url     2.715102\n",
       "6             is_json     2.066788\n",
       "7      start_with_api     0.000000\n",
       "8         has_userapi     0.000000\n",
       "9      has_googleapis     0.000000\n",
       "10        minus_count     0.000000\n",
       "11        is_redirect     0.000000\n",
       "12    size_of_cookies     0.000000"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_v3.get_feature_importance(prettified=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50ecc36d-2a62-4a8c-ad02-4864e00e430d",
   "metadata": {},
   "source": [
    "#### Средний результат после нескольких запусков: 0.90181\n",
    "* Против 0.89032 первой модели\n",
    "* Против 0.89840 второй модели\n",
    "* Против 0.54118 константного предсказания 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d27e3c5-5e09-4aa9-8d92-62bbf7ac8cc8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}