0003 Algorithm Selection/03. 1 Hyperparameter Optimization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from scipy.stats import randint as sp_randint\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import uniform as sp_randFloat\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from time import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import randint as sp_randInt\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, PredefinedSplit\n",
    "from sklearn.metrics import make_scorer\n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['pck_size', 'Ether_type', 'LLC_ctrl', 'EAPOL_version', 'EAPOL_type', 'IP_ihl', 'IP_tos', 'IP_len', 'IP_flags', 'IP_DF', 'IP_ttl', 'IP_options', 'ICMP_code', 'TCP_dataofs', 'TCP_FIN', 'TCP_ACK', 'TCP_window', 'UDP_len', 'DHCP_options', 'BOOTP_hlen', 'BOOTP_flags', 'BOOTP_sname', 'BOOTP_file', 'BOOTP_options', 'DNS_qr', 'DNS_rd', 'DNS_qdcount', 'dport_class', 'payload_bytes', 'entropy', 'Label']\n",
    "\n",
    "df=pd.read_csv(\"Aalto_train_IoTDevID.csv\",usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "df=pd.read_csv(\"Aalto_validation_IoTDevID.csv\",usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(62923, 30) (20971, 30)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aalto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                                                                            F1 Score             Time     No      \n",
      "default                                                                                    0.7216903208093847   10.534   24      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 29, 'min_samples_split': 2}       0.7270150996774428   14.381   0       \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 29, 'min_samples_split': 9}    0.724980440036087    15.236   1       \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 13, 'min_samples_split': 2}       0.7252618968864211   9.218    2       \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 16, 'min_samples_split': 9}       0.7245593084832818   10.306   3       \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 27, 'min_samples_split': 6}       0.7251107950372347   13.949   4       \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 20, 'min_samples_split': 9}    0.7244973995041137   13.091   5       \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 16, 'min_samples_split': 2}    0.7231765310218701   12.385   6       \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 21, 'min_samples_split': 9}       0.7247943363351329   12.858   7       \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 18, 'min_samples_split': 9}       0.7248002515917541   12.961   8       \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 8, 'min_samples_split': 5}        0.723367681399762    9.64     9       \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 27, 'min_samples_split': 8}       0.7244262045209183   14.53    10      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 19, 'min_samples_split': 2}    0.7236455988378891   13.168   11      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 20, 'min_samples_split': 8}    0.723866087060152    14.352   12      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 8, 'min_samples_split': 8}     0.7237479956150793   11.079   13      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 7, 'min_samples_split': 9}     0.7241165685092735   9.799    14      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 20, 'min_samples_split': 2}       0.7257924263102572   12.802   15      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 15, 'min_samples_split': 8}    0.7232163736578001   12.453   16      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 18, 'min_samples_split': 5}       0.7231862602417674   12.192   17      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 12, 'min_samples_split': 8}    0.7242180589439537   12.623   18      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 8, 'min_samples_split': 9}     0.7237966278378923   10.075   19      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 13, 'min_samples_split': 6}       0.7253347945229431   10.037   20      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 26, 'min_samples_split': 6}       0.725302991802814    12.981   21      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 17, 'min_samples_split': 3}       0.7248940115045128   10.636   22      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 20, 'min_samples_split': 4}    0.7209539428720394   11.647   23      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 9, 'min_samples_split': 8}     0.722450295095256    8.9      24      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 18, 'min_samples_split': 3}       0.7250459496902333   10.466   25      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 6, 'min_samples_split': 5}        0.724028654176219    7.67     26      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 14, 'min_samples_split': 6}       0.725094307029027    9.462    27      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 26, 'min_samples_split': 9}    0.7251236464849294   13.486   28      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 12, 'min_samples_split': 6}       0.7246663845617409   9.562    29      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 7, 'min_samples_split': 4}        0.7228604700446687   8.017    30      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 18, 'min_samples_split': 4}    0.7234831398596584   10.994   31      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 22, 'min_samples_split': 8}    0.7242566965576555   12.191   32      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 25, 'min_samples_split': 8}    0.724689598515635    14.339   33      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 10, 'min_samples_split': 6}    0.7231456172138455   9.195    34      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 12, 'min_samples_split': 6}       0.7186509273016304   9.17     35      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 27, 'min_samples_split': 3}       0.7246350578939911   13.181   36      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 22, 'min_samples_split': 5}       0.7260497250645431   11.054   37      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 24, 'min_samples_split': 8}       0.7190099059589314   11.804   38      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 29, 'min_samples_split': 5}       0.7251969331434204   13.558   39      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 11, 'min_samples_split': 5}       0.7247805981039107   8.714    40      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 18, 'min_samples_split': 8}       0.7250182827569855   9.906    41      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 23, 'min_samples_split': 5}       0.7256816573850342   12.52    42      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 18, 'min_samples_split': 7}       0.7244911364951246   9.992    43      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 29, 'min_samples_split': 5}       0.7271805851132336   14.186   44      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 29, 'min_samples_split': 8}       0.725989247364035    13.222   45      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 16, 'min_samples_split': 8}    0.7235095351931019   10.251   46      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 26, 'min_samples_split': 2}       0.7267338625528005   13.176   47      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 26, 'min_samples_split': 2}       0.7266859440773427   13.061   48      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 27, 'min_samples_split': 3}       0.7268408251957146   12.488   49      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 8, 'min_samples_split': 5}     0.7221796067233227   7.971    50      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 20, 'min_samples_split': 6}       0.7250337431216138   10.596   51      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 23, 'min_samples_split': 2}       0.7211636749895688   11.646   52      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 3, 'min_samples_split': 2}        0.7234949727863585   6.745    53      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 22, 'min_samples_split': 6}       0.7271052189634853   11.83    54      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 14, 'min_samples_split': 7}       0.7237087117579789   8.427    55      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 12, 'min_samples_split': 8}    0.724436783952094    9.811    56      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 29, 'min_samples_split': 9}       0.7252723672583222   14.184   57      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 24, 'min_samples_split': 4}       0.725737715800568    11.45    58      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 1, 'min_samples_split': 9}        0.7234410867960476   6.701    59      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 10, 'min_samples_split': 2}       0.7251610309852468   9.171    60      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 17, 'min_samples_split': 9}    0.7245209535453699   10.934   61      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 21, 'min_samples_split': 4}       0.7252661448423163   11.706   62      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 13, 'min_samples_split': 9}       0.7225537598391735   9.053    63      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 27, 'min_samples_split': 5}       0.7248912901016741   13.058   64      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 23, 'min_samples_split': 6}    0.7222811428893577   12.971   65      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 28, 'min_samples_split': 2}       0.7268905598898969   13.104   66      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 18, 'min_samples_split': 9}       0.7244995620159433   10.208   67      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 21, 'min_samples_split': 2}    0.7239913735218103   11.833   68      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 23, 'min_samples_split': 4}       0.7241297461139777   11.426   69      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 15, 'min_samples_split': 4}       0.724625077473432    9.182    70      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 8, 'min_samples_split': 6}        0.7247149176717109   7.941    71      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 15, 'min_samples_split': 6}       0.7247305293670678   9.09     72      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 22, 'min_samples_split': 2}    0.7235299969799498   12.24    73      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 7, 'min_samples_split': 8}     0.7236558399915668   7.947    74      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 12, 'min_samples_split': 2}    0.7236569635372426   9.819    75      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 7}    0.7227461799970353   9.034    76      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 25, 'min_samples_split': 6}       0.7252167937146276   12.331   77      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 14, 'min_samples_split': 6}       0.7232444298701134   9.156    78      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 10, 'min_samples_split': 5}       0.7252999045367694   8.308    79      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 21, 'min_samples_split': 2}       0.7257054442069462   10.634   80      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 10, 'min_samples_split': 8}       0.7245298897087266   8.02     81      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 9, 'min_samples_split': 7}        0.7232855600935136   7.917    82      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 24, 'min_samples_split': 9}       0.7256216396347231   11.994   83      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 2, 'min_samples_split': 4}        0.7229663908544157   7.899    84      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 29, 'min_samples_split': 2}       0.7254681527942194   13.47    85      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 11, 'min_samples_split': 3}       0.7246605726260489   7.522    86      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 28, 'min_samples_split': 9}       0.7253759939620084   12.474   87      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 12, 'min_samples_split': 4}    0.7233360084857234   9.059    88      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 4, 'min_samples_split': 5}        0.7245381374390141   7.239    89      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 19, 'min_samples_split': 5}       0.7263807196564451   10.377   90      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 23, 'min_samples_split': 8}    0.723393216463202    12.619   91      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 6, 'min_samples_split': 8}     0.7234701314767534   8.436    92      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 16, 'min_samples_split': 6}       0.723029960791053    9.903    93      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 1, 'min_samples_split': 3}        0.7224739304434817   6.316    94      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 27, 'min_samples_split': 7}       0.726070962368767    12.085   95      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 22, 'min_samples_split': 9}    0.724964971235338    12.925   96      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 14, 'min_samples_split': 9}    0.7247520978573495   9.778    97      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 2, 'min_samples_split': 7}        0.7224310549691185   6.494    98      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 25, 'min_samples_split': 8}    0.7242836477642917   13.794   99      \n"
     ]
    }
   ],
   "source": [
    "print ('%-90s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "  \n",
    "\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(25):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GridSearchCV DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 14384 candidates, totalling 14384 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    2.3s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:    7.9s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:   21.4s\n",
      "[Parallel(n_jobs=-1)]: Done 792 tasks      | elapsed:   41.8s\n",
      "[Parallel(n_jobs=-1)]: Done 1242 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1792 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 2442 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 3192 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 4042 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 4992 tasks      | elapsed:  7.3min\n",
      "[Parallel(n_jobs=-1)]: Done 6042 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=-1)]: Done 7192 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=-1)]: Done 8442 tasks      | elapsed: 13.2min\n",
      "[Parallel(n_jobs=-1)]: Done 9792 tasks      | elapsed: 16.3min\n",
      "[Parallel(n_jobs=-1)]: Done 11242 tasks      | elapsed: 22.2min\n",
      "[Parallel(n_jobs=-1)]: Done 12792 tasks      | elapsed: 28.4min\n",
      "[Parallel(n_jobs=-1)]: Done 14384 out of 14384 | elapsed: 34.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier(criterion='entropy', max_depth=23, max_features=21)\n"
     ]
    }
   ],
   "source": [
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":list(range(1,32)),\n",
    "                 \"min_samples_split\":list(range(2,10)),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" :list(range(1,X_train.shape[1]))}\n",
    "\n",
    "\n",
    "\n",
    "nbModel_grid = GridSearchCV(estimator=DecisionTreeClassifier(), param_grid=param_grid, verbose=1, cv=ps, n_jobs=-1)\n",
    "nbModel_grid.fit(X, y)\n",
    "print(nbModel_grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "default                                                                                    0.7227299454671903   10.844   0       \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 2, 'min_samples_split': 10, 'n_estimators': 156} 0.7289502435293613   159.548  0       \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 25.0, 'max_features': 1, 'min_samples_split': 8, 'n_estimators': 113} 0.7262619535233411   193.489  1       \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 10, 'min_samples_split': 10, 'n_estimators': 148} 0.7236580905885828   183.427  2       \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 4, 'min_samples_split': 10, 'n_estimators': 82} 0.727171992889616    113.511  3       \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 3, 'min_samples_split': 7, 'n_estimators': 90} 0.7246079997419235   147.829  4       \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 17.0, 'max_features': 3, 'min_samples_split': 7, 'n_estimators': 179} 0.7261607773967084   141.951  5       \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 8, 'min_samples_split': 9, 'n_estimators': 71} 0.7274836873217096   159.158  6       \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 2, 'min_samples_split': 10, 'n_estimators': 44} 0.7272760895770615   125.654  7       \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 31.0, 'max_features': 5, 'min_samples_split': 7, 'n_estimators': 57} 0.7223700356865719   110.93   8       \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 6, 'min_samples_split': 7, 'n_estimators': 170} 0.726462732206969    197.449  9       \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 15.0, 'max_features': 7, 'min_samples_split': 8, 'n_estimators': 65} 0.7271774612920666   103.825  10      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 15.0, 'max_features': 3, 'min_samples_split': 9, 'n_estimators': 192} 0.7259226691785232   118.783  11      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 5, 'min_samples_split': 10, 'n_estimators': 101} 0.7264429266577636   139.107  12      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 29.0, 'max_features': 1, 'min_samples_split': 9, 'n_estimators': 168} 0.7273870266442394   159.972  13      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 20.0, 'max_features': 4, 'min_samples_split': 2, 'n_estimators': 42} 0.721795284493693    192.037  14      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 7, 'min_samples_split': 2, 'n_estimators': 179} 0.725948017769752    226.695  15      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 9, 'min_samples_split': 10, 'n_estimators': 88} 0.7268684877036256   177.318  16      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 16.0, 'max_features': 5, 'min_samples_split': 3, 'n_estimators': 40} 0.7275582826119787   158.053  17      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 29.0, 'max_features': 10, 'min_samples_split': 4, 'n_estimators': 131} 0.7261681352406946   150.095  18      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 16.0, 'max_features': 10, 'min_samples_split': 8, 'n_estimators': 51} 0.7254586605586391   106.204  19      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 6, 'min_samples_split': 7, 'n_estimators': 97} 0.7260312420099452   84.556   20      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 7, 'min_samples_split': 10, 'n_estimators': 72} 0.7281730893615443   94.277   21      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 6, 'min_samples_split': 10, 'n_estimators': 87} 0.7271905907490757   128.626  22      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 28.0, 'max_features': 7, 'min_samples_split': 9, 'n_estimators': 48} 0.7270876642348131   105.576  23      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 3, 'min_samples_split': 3, 'n_estimators': 75} 0.7256443228361186   66.597   24      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 10, 'min_samples_split': 7, 'n_estimators': 170} 0.7288270925057012   134.969  25      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 29.0, 'max_features': 3, 'min_samples_split': 5, 'n_estimators': 150} 0.7244428726674976   116.494  26      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 18.0, 'max_features': 8, 'min_samples_split': 8, 'n_estimators': 173} 0.7262569086702089   163.22   27      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 10, 'min_samples_split': 7, 'n_estimators': 58} 0.7270239359734808   99.236   28      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 5, 'min_samples_split': 6, 'n_estimators': 40} 0.7236789322288364   89.597   29      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 27.0, 'max_features': 5, 'min_samples_split': 5, 'n_estimators': 78} 0.7226860148579904   67.1     30      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 22.0, 'max_features': 2, 'min_samples_split': 8, 'n_estimators': 158} 0.7251134866724035   107.267  31      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 9, 'min_samples_split': 8, 'n_estimators': 137} 0.7282017198879392   154.268  32      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 8, 'min_samples_split': 9, 'n_estimators': 72} 0.7239421297620993   100.141  33      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 7, 'min_samples_split': 8, 'n_estimators': 114} 0.7272023152708532   107.083  34      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 21.0, 'max_features': 2, 'min_samples_split': 3, 'n_estimators': 153} 0.7243882279289854   89.127   35      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 9, 'min_samples_split': 9, 'n_estimators': 69} 0.7268893812726851   96.224   36      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 25.0, 'max_features': 3, 'min_samples_split': 7, 'n_estimators': 60} 0.7270006719672821   79.197   37      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 8, 'min_samples_split': 10, 'n_estimators': 150} 0.7269748732859105   140.889  38      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 18.0, 'max_features': 8, 'min_samples_split': 9, 'n_estimators': 96} 0.729092339928399    82.865   39      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 10, 'min_samples_split': 4, 'n_estimators': 134} 0.7279725366435981   99.794   40      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 19.0, 'max_features': 7, 'min_samples_split': 4, 'n_estimators': 27} 0.726429423617312    89.002   41      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 25.0, 'max_features': 1, 'min_samples_split': 10, 'n_estimators': 24} 0.7278031608325315   121.184  42      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 17.0, 'max_features': 7, 'min_samples_split': 9, 'n_estimators': 60} 0.7268951830128271   126.641  43      \n",
      "{'bootstrap': False, 'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 4, 'min_samples_split': 9, 'n_estimators': 154} 0.72544300622277     137.125  44      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 2, 'min_samples_split': 8, 'n_estimators': 123} 0.7267975800525236   103.93   45      \n",
      "{'bootstrap': True, 'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 8, 'min_samples_split': 10, 'n_estimators': 52} 0.725383068130239    98.231   46      \n",
      "{'bootstrap': True, 'criterion': 'gini', 'max_depth': 21.0, 'max_features': 8, 'min_samples_split': 10, 'n_estimators': 139} 0.727666811668462    115.969  47      \n",
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 20.0, 'max_features': 7, 'min_samples_split': 9, 'n_estimators': 93} 0.7276667091992742   113.684  48      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': False, 'criterion': 'gini', 'max_depth': 19.0, 'max_features': 8, 'min_samples_split': 9, 'n_estimators': 159} 0.727076835646918    110.671  49      \n"
     ]
    }
   ],
   "source": [
    "# use a full grid over all parameters\n",
    "param_grid = {\"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "              \"n_estimators\" : sp_randint(1, 200),\n",
    "              \"max_features\": sp_randint(1, 11),\n",
    "              \"min_samples_split\":sp_randint(2, 11),\n",
    "              \"bootstrap\": [True, False],\n",
    "              \"criterion\": [\"gini\", \"entropy\"]}\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=RandomForestClassifier()\n",
    "for ii in range(1):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(50):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(RandomForestClassifier(),param_grid,X,y)\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1=sklearn.metrics.f1_score(y_test, predict,average= \"macro\") \n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "default                                                                                    0.7097573078498426   31.042   49      \n",
      "{'algorithm': 'auto', 'leaf_size': 30, 'n_neighbors': 63, 'weights': 'distance'}           0.7169344588030341   211.957  0       \n",
      "{'algorithm': 'auto', 'leaf_size': 16, 'n_neighbors': 47, 'weights': 'distance'}           0.7176466921242463   245.564  1       \n",
      "{'algorithm': 'brute', 'leaf_size': 18, 'n_neighbors': 44, 'weights': 'distance'}          0.7169670826387976   273.454  2       \n",
      "{'algorithm': 'brute', 'leaf_size': 2, 'n_neighbors': 48, 'weights': 'distance'}           0.718017143478749    227.629  3       \n",
      "{'algorithm': 'auto', 'leaf_size': 43, 'n_neighbors': 46, 'weights': 'distance'}           0.7176244604190383   228.35   4       \n",
      "{'algorithm': 'auto', 'leaf_size': 18, 'n_neighbors': 40, 'weights': 'distance'}           0.7175984782540666   257.862  5       \n",
      "{'algorithm': 'brute', 'leaf_size': 20, 'n_neighbors': 61, 'weights': 'distance'}          0.7170127067216242   234.887  6       \n",
      "{'algorithm': 'kd_tree', 'leaf_size': 1, 'n_neighbors': 46, 'weights': 'distance'}         0.7104694436974722   202.743  7       \n",
      "{'algorithm': 'brute', 'leaf_size': 35, 'n_neighbors': 7, 'weights': 'uniform'}            0.7081961280125751   209.964  8       \n",
      "{'algorithm': 'brute', 'leaf_size': 23, 'n_neighbors': 52, 'weights': 'distance'}          0.7173685344914941   212.04   9       \n",
      "{'algorithm': 'auto', 'leaf_size': 37, 'n_neighbors': 54, 'weights': 'distance'}           0.7177129935879529   224.004  10      \n",
      "{'algorithm': 'brute', 'leaf_size': 4, 'n_neighbors': 62, 'weights': 'distance'}           0.7170097076382455   207.909  11      \n",
      "{'algorithm': 'brute', 'leaf_size': 8, 'n_neighbors': 63, 'weights': 'distance'}           0.7169344588030341   271.655  12      \n",
      "{'algorithm': 'brute', 'leaf_size': 17, 'n_neighbors': 43, 'weights': 'distance'}          0.7178903666497938   228.07   13      \n",
      "{'algorithm': 'brute', 'leaf_size': 14, 'n_neighbors': 42, 'weights': 'distance'}          0.7176872387392771   208.902  14      \n",
      "{'algorithm': 'kd_tree', 'leaf_size': 4, 'n_neighbors': 44, 'weights': 'distance'}         0.7102768121197465   252.865  15      \n",
      "{'algorithm': 'brute', 'leaf_size': 45, 'n_neighbors': 23, 'weights': 'distance'}          0.7114519450706117   230.086  16      \n",
      "{'algorithm': 'auto', 'leaf_size': 7, 'n_neighbors': 16, 'weights': 'distance'}            0.7131093630276617   202.224  17      \n",
      "{'algorithm': 'auto', 'leaf_size': 13, 'n_neighbors': 48, 'weights': 'distance'}           0.718017143478749    288.878  18      \n",
      "{'algorithm': 'brute', 'leaf_size': 31, 'n_neighbors': 56, 'weights': 'distance'}          0.7176132491846418   167.248  19      \n",
      "{'algorithm': 'kd_tree', 'leaf_size': 35, 'n_neighbors': 56, 'weights': 'distance'}        0.7079073271193573   190.033  20      \n",
      "{'algorithm': 'brute', 'leaf_size': 7, 'n_neighbors': 44, 'weights': 'distance'}           0.7169670826387976   251.166  21      \n",
      "{'algorithm': 'auto', 'leaf_size': 15, 'n_neighbors': 54, 'weights': 'distance'}           0.7177129935879529   232.68   22      \n",
      "{'algorithm': 'brute', 'leaf_size': 13, 'n_neighbors': 51, 'weights': 'distance'}          0.7173184534887421   230.976  23      \n",
      "{'algorithm': 'auto', 'leaf_size': 3, 'n_neighbors': 38, 'weights': 'distance'}            0.7167155412508265   246.683  24      \n",
      "{'algorithm': 'brute', 'leaf_size': 11, 'n_neighbors': 32, 'weights': 'distance'}          0.7102365799223443   229.454  25      \n",
      "{'algorithm': 'brute', 'leaf_size': 24, 'n_neighbors': 50, 'weights': 'distance'}          0.717631726693687    206.894  26      \n",
      "{'algorithm': 'ball_tree', 'leaf_size': 49, 'n_neighbors': 33, 'weights': 'distance'}      0.7103457394465497   166.123  27      \n",
      "{'algorithm': 'brute', 'leaf_size': 41, 'n_neighbors': 48, 'weights': 'distance'}          0.718017143478749    255.024  28      \n",
      "{'algorithm': 'kd_tree', 'leaf_size': 16, 'n_neighbors': 34, 'weights': 'distance'}        0.7111808408837775   191.456  29      \n",
      "{'algorithm': 'auto', 'leaf_size': 26, 'n_neighbors': 54, 'weights': 'distance'}           0.7177129935879529   291.692  30      \n",
      "{'algorithm': 'auto', 'leaf_size': 16, 'n_neighbors': 62, 'weights': 'distance'}           0.7170097076382455   211.092  31      \n",
      "{'algorithm': 'auto', 'leaf_size': 11, 'n_neighbors': 54, 'weights': 'distance'}           0.7177129935879529   198.097  32      \n",
      "{'algorithm': 'brute', 'leaf_size': 28, 'n_neighbors': 45, 'weights': 'distance'}          0.7174484819687488   206.553  33      \n",
      "{'algorithm': 'auto', 'leaf_size': 14, 'n_neighbors': 22, 'weights': 'distance'}           0.7113581356843974   170.384  34      \n",
      "{'algorithm': 'ball_tree', 'leaf_size': 16, 'n_neighbors': 42, 'weights': 'distance'}      0.7102057347512681   171.394  35      \n",
      "{'algorithm': 'brute', 'leaf_size': 25, 'n_neighbors': 44, 'weights': 'distance'}          0.7169670826387976   253.003  36      \n",
      "{'algorithm': 'brute', 'leaf_size': 38, 'n_neighbors': 45, 'weights': 'distance'}          0.7174484819687488   220.847  37      \n",
      "{'algorithm': 'brute', 'leaf_size': 24, 'n_neighbors': 38, 'weights': 'distance'}          0.7167155412508265   239.582  38      \n",
      "{'algorithm': 'brute', 'leaf_size': 32, 'n_neighbors': 61, 'weights': 'distance'}          0.7170127067216242   246.425  39      \n",
      "{'algorithm': 'ball_tree', 'leaf_size': 15, 'n_neighbors': 55, 'weights': 'distance'}      0.7066496694373934   200.188  40      \n",
      "{'algorithm': 'brute', 'leaf_size': 40, 'n_neighbors': 29, 'weights': 'distance'}          0.7102025057910922   254.251  41      \n",
      "{'algorithm': 'auto', 'leaf_size': 13, 'n_neighbors': 63, 'weights': 'distance'}           0.7169344588030341   253.98   42      \n",
      "{'algorithm': 'brute', 'leaf_size': 1, 'n_neighbors': 44, 'weights': 'distance'}           0.7169670826387976   194.761  43      \n",
      "{'algorithm': 'brute', 'leaf_size': 24, 'n_neighbors': 45, 'weights': 'distance'}          0.7174484819687488   249.6    44      \n",
      "{'algorithm': 'auto', 'leaf_size': 23, 'n_neighbors': 19, 'weights': 'distance'}           0.7141141781681122   221.987  45      \n",
      "{'algorithm': 'brute', 'leaf_size': 28, 'n_neighbors': 15, 'weights': 'distance'}          0.7131475504599976   231.722  46      \n",
      "{'algorithm': 'brute', 'leaf_size': 14, 'n_neighbors': 52, 'weights': 'distance'}          0.7173685344914941   205.543  47      \n",
      "{'algorithm': 'auto', 'leaf_size': 36, 'n_neighbors': 56, 'weights': 'distance'}           0.7176132491846418   218.377  48      \n",
      "{'algorithm': 'brute', 'leaf_size': 4, 'n_neighbors': 60, 'weights': 'distance'}           0.7169916986321065   228.11   49      \n"
     ]
    }
   ],
   "source": [
    "# use a full grid over all parameters\n",
    "param_grid = {\"n_neighbors\" : sp_randint(1,64) ,  \n",
    "             \"leaf_size\": sp_randint(1,50) , \n",
    "              \"algorithm\" : [\"auto\", \"ball_tree\", \"kd_tree\", \"brute\"],\n",
    "              \"weights\" : [\"uniform\", \"distance\"]}\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=KNeighborsClassifier()\n",
    "for ii in range(1):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),i))\n",
    "\n",
    "\n",
    "for i in range(50):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(KNeighborsClassifier(),param_grid,X,y)\n",
    "\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1=sklearn.metrics.f1_score(y_test, predict,average= \"macro\") \n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "default                                                                                    0.6247629147200783   323.341  0       \n",
      "{'learning_rate': 0.1838641631843394, 'max_depth': 6, 'n_estimators': 535, 'subsample': 0.7134682210818548} 0.010075373269035157 41527.706 0       \n"
     ]
    }
   ],
   "source": [
    "# use a full grid over all parameters\n",
    "param_grid =  {\"learning_rate\": sp_randFloat(),\n",
    "          \"subsample\"    : sp_randFloat(),\n",
    "          \"n_estimators\" : sp_randInt(100, 1000),\n",
    "          \"max_depth\"    : sp_randInt(4, 10)\n",
    "         }\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=GradientBoostingClassifier()\n",
    "for ii in range(1):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "\n",
    "for i in range(1):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(GradientBoostingClassifier(),param_grid,X,y)\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1=sklearn.metrics.f1_score(y_test, predict,average= \"macro\") \n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 20 candidates, totalling 20 fits\n",
      "SVC(C=10, gamma=1)\n"
     ]
    }
   ],
   "source": [
    "param_grid =  {'C': [0.001, 0.01, 0.1, 1, 10], 'gamma' : [0.001, 0.01, 0.1, 1]}    \n",
    "nbModel_grid = GridSearchCV(estimator=svm.SVC(), param_grid=param_grid, verbose=1, cv=ps, n_jobs=-1)\n",
    "nbModel_grid.fit(X, y)\n",
    "print(nbModel_grid.best_estimator_)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gamma': 1, 'C': 10}                                                                      0.6492980530251848   6214.957 0.64929805\n"
     ]
    }
   ],
   "source": [
    "param_grid =  {'C': [0.001, 0.01, 0.1, 1, 10], 'gamma' : [0.001, 0.01, 0.1, 1]}  \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "second=time()\n",
    "a,b,clf=run_random_search(svm.SVC(),param_grid,X,y)\n",
    "clf.fit(X_train, y_train)\n",
    "predict =clf.predict(X_test)\n",
    "f1=(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "\n",
    "print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),b))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10594.689 0       \n",
      "{'gamma': 1, 'C': 1}                                                                       0.6346047487263045   9958.364 1       \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10890.422 2       \n",
      "{'gamma': 0.1, 'C': 10}                                                                    0.6368980755609428   10017.98 3       \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9705.726 4       \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10802.529 5       \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10383.928 6       \n",
      "{'gamma': 0.1, 'C': 1}                                                                     0.6179754190670083   11097.563 7       \n",
      "{'gamma': 1, 'C': 1}                                                                       0.6346047487263045   9510.031 8       \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9953.25  9       \n",
      "{'gamma': 1, 'C': 1}                                                                       0.6346047487263045   11350.142 10      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9939.749 11      \n",
      "{'gamma': 0.1, 'C': 10}                                                                    0.6368980755609428   8435.837 12      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9327.144 13      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9242.359 14      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9632.738 15      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9295.685 16      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10715.839 17      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10964.279 18      \n",
      "{'gamma': 0.1, 'C': 10}                                                                    0.6368980755609428   10740.248 19      \n",
      "{'gamma': 1, 'C': 0.1}                                                                     0.5961635493517783   12507.846 20      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    10097.318 21      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9778.545 22      \n",
      "{'gamma': 0.1, 'C': 10}                                                                    0.6368980755609428   10118.065 23      \n",
      "{'gamma': 1, 'C': 1}                                                                       0.6346047487263045   9299.756 24      \n",
      "{'gamma': 0.1, 'C': 10}                                                                    0.6368980755609428   11395.01 25      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    8425.38  26      \n",
      "{'gamma': 0.1, 'C': 1}                                                                     0.6179754190670083   10058.017 27      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9216.197 28      \n",
      "{'gamma': 1, 'C': 10}                                                                      0.649298053025185    9412.471 29      \n",
      "{'gamma': 0.1, 'C': 10}                                                                    0.6368980755609428   8840.529 30      \n",
      "{'gamma': 0.01, 'C': 10}                                                                   0.616569367062346    10968.527 31      \n"
     ]
    }
   ],
   "source": [
    "param_grid =  {'C': [0.001, 0.01, 0.1, 1, 10], 'gamma' : [0.001, 0.01, 0.1, 1]}  \n",
    "\n",
    "\n",
    "\n",
    "for i in range(33):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(svm.SVC(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(10):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "param_grid = {\"C\": stats.uniform(0.001, 10),\n",
    "             \"gamma\": stats.uniform(0.001, 1)}\n",
    "\n",
    "for i in range(33):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(svm.SVC(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(10):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#   NB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 200 candidates, totalling 200 fits\n",
      "CategoricalNB(alpha=1e-09, fit_prior='True')\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import CategoricalNB\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid_nb = {\n",
    "    'alpha': np.logspace(0,-9, num=100),\n",
    "    \"fit_prior\":[\"True\",\"False\"]\n",
    "}\n",
    "nbModel_grid = GridSearchCV(estimator=CategoricalNB(), param_grid=param_grid_nb, verbose=1, cv=ps, n_jobs=-1)\n",
    "nbModel_grid.fit(X, y)\n",
    "print(nbModel_grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 200 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   23.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:  2.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CategoricalNB(alpha=1e-09, fit_prior='True')\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import CategoricalNB\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid_nb = {\n",
    "    'alpha': np.logspace(0,-9, num=100),\n",
    "    \"fit_prior\":[\"True\",\"False\"]\n",
    "}\n",
    "nbModel_grid = GridSearchCV(estimator=CategoricalNB(), param_grid=param_grid_nb, verbose=1, cv=ps, n_jobs=-1)\n",
    "nbModel_grid.fit(X, y)\n",
    "print(nbModel_grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV NB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'fit_prior': 'True', 'alpha': 1.873817422860383e-07}                                      0.5585787416692073   8.41     0.55857874\n"
     ]
    }
   ],
   "source": [
    "\n",
    "second=time()\n",
    "\n",
    "param_grid = {\n",
    "    'alpha': np.logspace(0,-9, num=100),\n",
    "    \"fit_prior\":[\"True\",\"False\"]\n",
    "}\n",
    "\n",
    "a,b,clf=run_random_search(CategoricalNB(),param_grid,X,y)\n",
    "clf.fit(X_train, y_train)\n",
    "predict =clf.predict(X_test)\n",
    "f1=(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "\n",
    "print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),b))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import CategoricalNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'fit_prior': 'False', 'alpha': 8.111308307896873e-08}                                     0.5585527392583353   27.113   0       \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   26.625   1       \n",
      "{'fit_prior': 'False', 'alpha': 1.873817422860383e-08}                                     0.5585237755311087   27.33    2       \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-07}                                     0.5585787416692072   26.851   3       \n",
      "{'fit_prior': 'False', 'alpha': 1.232846739442066e-07}                                     0.5585527392583353   28.996   4       \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   30.334   5       \n",
      "{'fit_prior': 'False', 'alpha': 1.873817422860383e-07}                                     0.5585787416692072   25.975   6       \n",
      "{'fit_prior': 'True', 'alpha': 2.848035868435799e-07}                                      0.5585787416692072   30.5     7       \n",
      "{'fit_prior': 'True', 'alpha': 8.111308307896856e-09}                                      0.5585237755311087   28.78    8       \n",
      "{'fit_prior': 'False', 'alpha': 8.111308307896872e-07}                                     0.5585042848171752   29.407   9       \n",
      "{'fit_prior': 'False', 'alpha': 1.232846739442066e-08}                                     0.5585237755311087   27.218   10      \n",
      "{'fit_prior': 'False', 'alpha': 1.873817422860383e-07}                                     0.5585787416692072   28.401   11      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-07}                                     0.5585684317597219   26.954   12      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860383e-07}                                      0.5585787416692072   26.104   13      \n",
      "{'fit_prior': 'False', 'alpha': 1e-09}                                                     0.5587468285337281   26.859   14      \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-07}                                     0.5585787416692072   29.226   15      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860387e-09}                                      0.5585322432937763   29.316   16      \n",
      "{'fit_prior': 'True', 'alpha': 4.3287612810830526e-07}                                     0.5585321247002658   28.82    17      \n",
      "{'fit_prior': 'True', 'alpha': 1.2328467394420635e-09}                                     0.5585322432937763   29.354   18      \n",
      "{'fit_prior': 'False', 'alpha': 1.2328467394420635e-09}                                    0.5585322432937763   27.082   19      \n",
      "{'fit_prior': 'False', 'alpha': 1e-09}                                                     0.5587468285337281   27.493   20      \n",
      "{'fit_prior': 'True', 'alpha': 1e-09}                                                      0.5587468285337281   30.35    21      \n",
      "{'fit_prior': 'True', 'alpha': 6.579332246575682e-07}                                      0.5585042848171752   30.642   22      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-07}                                     0.5585684317597219   27.113   23      \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-07}                                     0.5585787416692072   25.374   24      \n",
      "{'fit_prior': 'True', 'alpha': 1e-09}                                                      0.5587468285337281   24.098   25      \n",
      "{'fit_prior': 'True', 'alpha': 1.519911082952933e-07}                                      0.5585684317597219   25.978   26      \n",
      "{'fit_prior': 'False', 'alpha': 3.5111917342151273e-09}                                    0.5585237755311087   26.621   27      \n",
      "{'fit_prior': 'False', 'alpha': 1.2328467394420635e-09}                                    0.5585322432937763   27.302   28      \n",
      "{'fit_prior': 'False', 'alpha': 1.873817422860387e-09}                                     0.5585322432937763   27.096   29      \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-06}                                     0.5583590427195376   27.005   30      \n",
      "{'fit_prior': 'True', 'alpha': 2.848035868435799e-07}                                      0.5585787416692072   28.666   31      \n",
      "{'fit_prior': 'False', 'alpha': 6.579332246575682e-08}                                     0.55850429035763     27.525   32      \n",
      "{'fit_prior': 'True', 'alpha': 3.5111917342151277e-07}                                     0.5585321247002658   24.992   33      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-07}                                      0.5585787416692072   23.676   34      \n",
      "{'fit_prior': 'True', 'alpha': 8.111308307896873e-08}                                      0.5585527392583353   24.551   35      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-07}                                      0.5585787416692072   23.799   36      \n",
      "{'fit_prior': 'True', 'alpha': 1e-08}                                                      0.5585237755311087   24.209   37      \n",
      "{'fit_prior': 'True', 'alpha': 1.2328467394420635e-09}                                     0.5585322432937763   24.771   38      \n",
      "{'fit_prior': 'True', 'alpha': 1.519911082952933e-08}                                      0.5585237755311087   23.929   39      \n",
      "{'fit_prior': 'False', 'alpha': 2.848035868435799e-07}                                     0.5585787416692072   27.103   40      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-09}                                     0.5585322432937763   27.803   41      \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   27.139   42      \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   26.77    43      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860383e-07}                                      0.5585787416692072   28.174   44      \n",
      "{'fit_prior': 'True', 'alpha': 8.111308307896873e-08}                                      0.5585527392583353   25.43    45      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860383e-07}                                      0.5585787416692072   27.594   46      \n",
      "{'fit_prior': 'False', 'alpha': 1.873817422860387e-09}                                     0.5585322432937763   28.726   47      \n",
      "{'fit_prior': 'True', 'alpha': 1.232846739442066e-07}                                      0.5585527392583353   26.621   48      \n",
      "{'fit_prior': 'False', 'alpha': 8.111308307896873e-08}                                     0.5585527392583353   32.861   49      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860383e-08}                                      0.5585237755311087   57.316   50      \n",
      "{'fit_prior': 'True', 'alpha': 1.519911082952933e-08}                                      0.5585237755311087   23.86    51      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-07}                                      0.5585787416692072   27.31    52      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-07}                                      0.5585787416692072   28.759   53      \n",
      "{'fit_prior': 'True', 'alpha': 1.232846739442066e-08}                                      0.5585237755311087   28.147   54      \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   30.088   55      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-09}                                     0.5585322432937763   28.261   56      \n",
      "{'fit_prior': 'True', 'alpha': 1.232846739442066e-08}                                      0.5585237755311087   24.938   57      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860387e-09}                                      0.5585322432937763   24.441   58      \n",
      "{'fit_prior': 'True', 'alpha': 3.5111917342151277e-08}                                     0.5585237755311087   26.641   59      \n",
      "{'fit_prior': 'True', 'alpha': 1.519911082952933e-07}                                      0.5585684317597219   25.333   60      \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-07}                                     0.5585787416692072   25.032   61      \n",
      "{'fit_prior': 'False', 'alpha': 8.111308307896873e-08}                                     0.5585527392583353   25.714   62      \n",
      "{'fit_prior': 'True', 'alpha': 1.232846739442066e-07}                                      0.5585527392583353   26.345   63      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'fit_prior': 'False', 'alpha': 1.232846739442066e-07}                                     0.5585527392583353   25.202   64      \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-07}                                     0.5585787416692072   29.14    65      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-07}                                      0.5585787416692072   27.861   66      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-09}                                     0.5585322432937763   25.329   67      \n",
      "{'fit_prior': 'False', 'alpha': 1.232846739442066e-08}                                     0.5585237755311087   28.716   68      \n",
      "{'fit_prior': 'False', 'alpha': 1e-07}                                                     0.5585527392583353   25.47    69      \n",
      "{'fit_prior': 'False', 'alpha': 8.111308307896873e-08}                                     0.5585527392583353   26.286   70      \n",
      "{'fit_prior': 'True', 'alpha': 1.519911082952933e-07}                                      0.5585684317597219   24.25    71      \n",
      "{'fit_prior': 'True', 'alpha': 8.111308307896873e-08}                                      0.5585527392583353   23.598   72      \n",
      "{'fit_prior': 'False', 'alpha': 1.232846739442066e-07}                                     0.5585527392583353   24.13    73      \n",
      "{'fit_prior': 'True', 'alpha': 1.519911082952933e-09}                                      0.5585322432937763   23.182   74      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-09}                                      0.5585237755311087   21.41    75      \n",
      "{'fit_prior': 'True', 'alpha': 2.848035868435799e-07}                                      0.5585787416692072   21.284   76      \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   21.24    77      \n",
      "{'fit_prior': 'True', 'alpha': 1e-09}                                                      0.5587468285337281   22.208   78      \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-07}                                     0.5585787416692072   21.328   79      \n",
      "{'fit_prior': 'False', 'alpha': 3.5111917342151277e-07}                                    0.5585321247002658   21.259   80      \n",
      "{'fit_prior': 'True', 'alpha': 1e-09}                                                      0.5587468285337281   21.242   81      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860387e-09}                                      0.5585322432937763   21.216   82      \n",
      "{'fit_prior': 'False', 'alpha': 1e-07}                                                     0.5585527392583353   21.285   83      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-07}                                      0.5585787416692072   21.569   84      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860387e-09}                                      0.5585322432937763   21.238   85      \n",
      "{'fit_prior': 'True', 'alpha': 8.111308307896873e-08}                                      0.5585527392583353   21.221   86      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-07}                                     0.5585684317597219   21.278   87      \n",
      "{'fit_prior': 'True', 'alpha': 2.310129700083158e-08}                                      0.5585237755311087   25.438   88      \n",
      "{'fit_prior': 'True', 'alpha': 1.873817422860383e-08}                                      0.5585237755311087   25.128   89      \n",
      "{'fit_prior': 'False', 'alpha': 1e-09}                                                     0.5587468285337281   21.916   90      \n",
      "{'fit_prior': 'False', 'alpha': 1.519911082952933e-08}                                     0.5585237755311087   21.333   91      \n",
      "{'fit_prior': 'True', 'alpha': 2.848035868435799e-07}                                      0.5585787416692072   21.347   92      \n",
      "{'fit_prior': 'False', 'alpha': 5.336699231206313e-07}                                     0.5585321247002658   21.264   93      \n",
      "{'fit_prior': 'True', 'alpha': 2.848035868435805e-09}                                      0.5585237755311087   21.223   94      \n",
      "{'fit_prior': 'False', 'alpha': 2.310129700083158e-08}                                     0.5585237755311087   21.328   95      \n",
      "{'fit_prior': 'True', 'alpha': 8.111308307896873e-08}                                      0.5585527392583353   25.477   96      \n",
      "{'fit_prior': 'False', 'alpha': 1.232846739442066e-08}                                     0.5585237755311087   24.622   97      \n",
      "{'fit_prior': 'False', 'alpha': 4.3287612810830526e-07}                                    0.5585321247002658   28.394   98      \n",
      "{'fit_prior': 'False', 'alpha': 1e-09}                                                     0.5587468285337281   27.397   99      \n"
     ]
    }
   ],
   "source": [
    "\n",
    "second=time()\n",
    "\n",
    "param_grid = {\n",
    "    'alpha': np.logspace(0,-9, num=100),\n",
    "    \"fit_prior\":[\"True\",\"False\"]\n",
    "}\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(CategoricalNB(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list= {'AaDevID': ['pck_size', 'Ether_type', 'LLC_ctrl', 'EAPOL_version', 'EAPOL_type', 'IP_ihl', 'IP_tos', 'IP_len', 'IP_flags', 'IP_DF', 'IP_ttl', 'IP_options', 'ICMP_code', 'TCP_dataofs', 'TCP_FIN', 'TCP_ACK', 'TCP_window', 'UDP_len', 'DHCP_options', 'BOOTP_hlen', 'BOOTP_flags', 'BOOTP_sname', 'BOOTP_file', 'BOOTP_options', 'DNS_qr', 'DNS_rd', 'DNS_qdcount', 'dport_class', 'payload_bytes', 'entropy', 'Label']}\n",
    "\n",
    "file_list={'AaDevID': ['./CSVs/Aalto_train_IoTDevID.csv','./CSVs/Aalto_validation_IoTDevID.csv']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AaDevID\n",
      "default                             0.4697629020250319   124.319  0       \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [7:41:50<00:00, 27710.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'solver': 'adam', 'learning_rate': 'constant', 'hidden_layer_sizes': (1220, 1965), 'alpha': 0.1, 'activation': 'relu'} 0.30080728295490694  27709.979 0       \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'tabulate' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [9], line 107\u001b[0m\n\u001b[0;32m    103\u001b[0m results \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame (lines[\u001b[38;5;241m1\u001b[39m:], columns \u001b[38;5;241m=\u001b[39m lines[\u001b[38;5;241m0\u001b[39m])\n\u001b[0;32m    104\u001b[0m results\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mann_HPO.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m,index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m--> 107\u001b[0m \u001b[38;5;28mprint\u001b[39m (\u001b[43mtabulate\u001b[49m(results, headers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlist\u001b[39m(results\u001b[38;5;241m.\u001b[39mcolumns)))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'tabulate' is not defined"
     ]
    }
   ],
   "source": [
    "# MLP\n",
    "from tqdm import tqdm\n",
    "import scipy.stats\n",
    "\n",
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "\n",
    "param_dist = [\n",
    "    {'hidden_layer_sizes': (scipy.stats.randint(low=2**5, high=2**11+1).rvs(), )}\n",
    "    for _ in range(100)] + [\n",
    "    {'hidden_layer_sizes': (\n",
    "            scipy.stats.randint(low=2 ** 5, high=2 ** 11 + 1).rvs(),\n",
    "            scipy.stats.randint(low=2 ** 5, high=2 ** 11 + 1).rvs(),\n",
    "    )} for i in range(100)\n",
    "]\n",
    "\n",
    "layers=[]\n",
    "\n",
    "for i in param_dist:\n",
    "    layers.append(i['hidden_layer_sizes'])\n",
    "\n",
    "\n",
    "\n",
    "param_grid =  {\n",
    "    'hidden_layer_sizes': layers,\n",
    "    'solver': ['sgd', 'adam', 'lbfgs'],\n",
    "    'activation': ['relu', 'tanh'],\n",
    "    'alpha': [0.0001,0.001,0.01,0.1, 0.05],\n",
    "    'learning_rate': ['constant','adaptive']\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "lines=[['solver', 'learning_rate','hidden_layer_sizes','alpha','activation', \"F1\",\"Std\",\"Time\",\"No\",\"Attack\"]]\n",
    "\n",
    "\n",
    "for j in file_list:\n",
    "    print(j)\n",
    "    \n",
    "    \n",
    "    df=pd.read_csv(file_list[j][0],usecols=feature_list[j]) \n",
    "    X_train = df.iloc[:,0:-1]\n",
    "    df['Label'] = df['Label'].astype('category')\n",
    "    y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "    df=pd.read_csv(file_list[j][1],usecols=feature_list[j]) \n",
    "    X_test = df.iloc[:,0:-1]\n",
    "    df['Label'] = df['Label'].astype('category')\n",
    "    y_test=df['Label'].cat.codes  \n",
    "    \n",
    "    X= np.concatenate([X_train, X_test])\n",
    "    test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "    y = np.concatenate([y_train, y_test])\n",
    "    ps = PredefinedSplit(test_fold)\n",
    "    \n",
    "\n",
    "    param_grid =  {\n",
    "        'hidden_layer_sizes': layers,\n",
    "        'solver': ['sgd', 'adam', 'lbfgs'],\n",
    "        'activation': ['relu', 'tanh'],\n",
    "        'alpha': [0.0001,0.001,0.01,0.1, 0.05],\n",
    "        'learning_rate': ['constant','adaptive']\n",
    "    }\n",
    "\n",
    "\n",
    "\n",
    "    second=time()\n",
    "    f1=[]\n",
    "    clf=MLPClassifier()\n",
    "    for ii in range(1):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-35s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "\n",
    "    \n",
    "    ######################################################################################################################\n",
    "    for i in tqdm(range(1)):\n",
    "        second=time()\n",
    "        a,b,clf=run_random_search(MLPClassifier(),param_grid,X,y)\n",
    "        f1=[]\n",
    "        for ii in range(1):\n",
    "            clf.fit(X_train, y_train)\n",
    "            predict =clf.predict(X_test)\n",
    "            f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "        f1_result=sum(f1)/len(f1)   \n",
    "        f1=np.array(f1)\n",
    "        stndtd=f1.std()\n",
    "        temp=list(a.values())\n",
    "        print('%-90s %-20s %-8s %-8s' % (a,f1_result,round(time()-second,3),i))\n",
    "        temp=temp+[f1_result,stndtd,round(time()-second,3),i,j]\n",
    "        lines.append(temp)\n",
    "\n",
    "        #if f1>0.76:\n",
    "\n",
    "results = pd.DataFrame (lines[1:], columns = lines[0])\n",
    "results.to_csv(\"ann_HPO.csv\",index=False)\n",
    "\n",
    "\n",
    "print (tabulate(results, headers=list(results.columns)))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    solver    learning_rate    hidden_layer_sizes      alpha  activation          F1    Std    Time    No  Attack\n",
      "--  --------  ---------------  --------------------  -------  ------------  --------  -----  ------  ----  --------\n",
      " 0  adam      constant         (1220, 1965)              0.1  relu          0.300807      0   27710     0  AaDevID\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "print (tabulate(results, headers=list(results.columns)))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  IoTSense- IoTsentinel "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from scipy.stats import randint as sp_randint\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import uniform as sp_randFloat\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from time import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import randint as sp_randInt\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, PredefinedSplit\n",
    "from sklearn.metrics import make_scorer\n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoTSentinel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARP</th>\n",
       "      <th>LLC</th>\n",
       "      <th>EAPOL</th>\n",
       "      <th>IP</th>\n",
       "      <th>ICMP</th>\n",
       "      <th>ICMP6</th>\n",
       "      <th>TCP</th>\n",
       "      <th>UDP</th>\n",
       "      <th>HTTP</th>\n",
       "      <th>HTTPS</th>\n",
       "      <th>...</th>\n",
       "      <th>NTP</th>\n",
       "      <th>IP_padding</th>\n",
       "      <th>IP_add_count</th>\n",
       "      <th>IP_ralert</th>\n",
       "      <th>Portcl_src</th>\n",
       "      <th>Portcl_dst</th>\n",
       "      <th>Pck_size</th>\n",
       "      <th>Pck_rawdata</th>\n",
       "      <th>MAC</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>117</td>\n",
       "      <td>1</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>1</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>328</td>\n",
       "      <td>0</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>328</td>\n",
       "      <td>0</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86079</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>313</td>\n",
       "      <td>1</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86080</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>262</td>\n",
       "      <td>1</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86081</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86082</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86083</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>86084 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ARP  LLC  EAPOL  IP  ICMP  ICMP6  TCP  UDP  HTTP  HTTPS  ...  NTP  \\\n",
       "0        0    0      1   0     0      0    0    0     0      0  ...    0   \n",
       "1        0    0      1   0     0      0    0    0     0      0  ...    0   \n",
       "2        0    0      0   1     0      0    0    1     0      0  ...    0   \n",
       "3        0    0      0   1     0      0    0    1     0      0  ...    0   \n",
       "4        1    0      0   0     0      0    0    0     0      0  ...    0   \n",
       "...    ...  ...    ...  ..   ...    ...  ...  ...   ...    ...  ...  ...   \n",
       "86079    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "86080    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "86081    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "86082    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "86083    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "\n",
       "       IP_padding  IP_add_count  IP_ralert  Portcl_src  Portcl_dst  Pck_size  \\\n",
       "0               0             0          0           0           0       117   \n",
       "1               0             0          0           0           0        95   \n",
       "2               0             1          0           1           1       328   \n",
       "3               0             1          0           1           1       328   \n",
       "4               0             0          0           0           0         0   \n",
       "...           ...           ...        ...         ...         ...       ...   \n",
       "86079           0             3          0           3           1       313   \n",
       "86080           0             3          0           3           1       262   \n",
       "86081           0             3          0           3           1        40   \n",
       "86082           0             3          0           3           1        40   \n",
       "86083           0             3          0           3           1        40   \n",
       "\n",
       "       Pck_rawdata                MAC     Label  \n",
       "0                1  20:f8:5e:ca:91:52      Aria  \n",
       "1                1  20:f8:5e:ca:91:52      Aria  \n",
       "2                0  20:f8:5e:ca:91:52      Aria  \n",
       "3                0  20:f8:5e:ca:91:52      Aria  \n",
       "4                0  20:f8:5e:ca:91:52      Aria  \n",
       "...            ...                ...       ...  \n",
       "86079            1  00:24:e4:24:80:2a  Withings  \n",
       "86080            1  00:24:e4:24:80:2a  Withings  \n",
       "86081            0  00:24:e4:24:80:2a  Withings  \n",
       "86082            0  00:24:e4:24:80:2a  Withings  \n",
       "86083            0  00:24:e4:24:80:2a  Withings  \n",
       "\n",
       "[86084 rows x 25 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"Aalto_train_IoTSentinel.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ARP', 'LLC', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'HTTP',\n",
       "       'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP', 'IP_padding',\n",
       "       'IP_add_count', 'IP_ralert', 'Portcl_src', 'Portcl_dst', 'Pck_size',\n",
       "       'Pck_rawdata', 'MAC', 'Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['ARP', 'LLC', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'HTTP',\n",
    "       'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP', 'IP_padding',\n",
    "       'IP_add_count', 'IP_ralert', 'Portcl_src', 'Portcl_dst', 'Pck_size',\n",
    "       'Pck_rawdata',  'Label']\n",
    "df=pd.read_csv(\"Aalto_train_IoTSentinel.csv\",usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "df=pd.read_csv(\"Aalto_validation_IoTSentinel.csv\",usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(86084, 23) (21723, 23)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                                                                            F1 Score             Time     No      \n",
      "default                                                                                    0.6009580335884174   3.297    24      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 19, 'min_samples_split': 6}    0.6030508357696247   3.882    0       \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 11, 'min_samples_split': 2}    0.6019134446933325   2.7      1       \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 7, 'min_samples_split': 4}     0.6008899861597053   2.675    2       \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 12, 'min_samples_split': 2}       0.6011518449043427   3.048    3       \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 14, 'min_samples_split': 4}       0.6019271172312137   3.166    4       \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 8, 'min_samples_split': 5}     0.600776034294108    2.699    5       \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 13, 'min_samples_split': 4}    0.6012155830985823   3.5      6       \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 12, 'min_samples_split': 4}    0.6007169103536598   3.149    7       \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 20, 'min_samples_split': 4}       0.6011493068164223   3.807    8       \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 6, 'min_samples_split': 5}        0.6013879377666946   2.508    9       \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 13, 'min_samples_split': 8}       0.6001132367734511   3.339    10      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 8, 'min_samples_split': 5}     0.6011763041325195   2.934    11      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 12, 'min_samples_split': 6}    0.6021024566596548   3.493    12      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 19, 'min_samples_split': 9}    0.6016383576743459   3.964    13      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 20, 'min_samples_split': 2}       0.6020742363039984   3.852    14      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 11, 'min_samples_split': 6}    0.6020211030396712   3.283    15      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 14, 'min_samples_split': 8}    0.6018172305951058   3.374    16      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 17, 'min_samples_split': 6}       0.6014710326768737   3.63     17      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 4, 'min_samples_split': 3}        0.5968674903457019   2.483    18      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 20, 'min_samples_split': 8}    0.6022104439491639   4.286    19      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 7, 'min_samples_split': 4}        0.6015693137161562   2.792    20      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 14, 'min_samples_split': 6}       0.6017949216348957   3.074    21      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 19, 'min_samples_split': 6}    0.6024430151083207   3.692    22      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 5, 'min_samples_split': 5}        0.6022739364798936   2.498    23      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 21, 'min_samples_split': 7}       0.6016857134025669   3.667    24      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 3}    0.6015151763417111   3.33     25      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 3, 'min_samples_split': 5}        0.5998416932351841   2.64     26      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 9, 'min_samples_split': 7}     0.603140806980107    3.021    27      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 17, 'min_samples_split': 7}    0.6029479447390311   3.835    28      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 11, 'min_samples_split': 5}       0.6013014797072703   3.124    29      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 21, 'min_samples_split': 9}    0.6019580675893905   3.98     30      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 12, 'min_samples_split': 4}       0.6010475693032766   3.181    31      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 9, 'min_samples_split': 3}     0.6018394030497072   2.939    32      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 12, 'min_samples_split': 3}    0.6012430819654145   3.022    33      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 9, 'min_samples_split': 8}        0.5985461479436998   2.964    34      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 8, 'min_samples_split': 7}     0.601039893545091    3.005    35      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 20, 'min_samples_split': 8}       0.6007605099676454   3.967    36      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 8, 'min_samples_split': 5}        0.6008745629513287   2.611    37      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 7, 'min_samples_split': 2}        0.6024562181865079   2.861    38      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 4, 'min_samples_split': 3}     0.6023181079153617   2.604    39      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 21, 'min_samples_split': 7}    0.6029429057703155   4.212    40      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 18, 'min_samples_split': 7}    0.60174529230521     3.86     41      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 13, 'min_samples_split': 3}       0.6016370279207668   3.332    42      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 5, 'min_samples_split': 4}        0.6011680084683833   2.629    43      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 7, 'min_samples_split': 3}     0.6021200172252105   2.691    44      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 12, 'min_samples_split': 2}       0.6013955748810014   3.242    45      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 22, 'min_samples_split': 8}    0.6025412911227193   3.791    46      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 6, 'min_samples_split': 5}        0.6019437360503138   2.675    47      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 12, 'min_samples_split': 9}    0.5991803543717217   2.868    48      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 21, 'min_samples_split': 6}       0.6014563832507143   3.87     49      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 21, 'min_samples_split': 3}    0.602723854653127    3.975    50      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 16, 'min_samples_split': 5}       0.6019333950583351   3.25     51      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 6, 'min_samples_split': 7}        0.6009520232475655   2.597    52      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 9, 'min_samples_split': 2}        0.6015812662827617   3.174    53      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 5, 'min_samples_split': 2}        0.5997503129299953   2.637    54      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 3, 'min_samples_split': 3}        0.6025699515987437   2.563    55      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 5, 'min_samples_split': 6}     0.5997699179427607   2.413    56      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 9, 'min_samples_split': 5}        0.6011193171473207   2.811    57      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 17, 'min_samples_split': 2}    0.6011052786908276   3.656    58      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 14, 'min_samples_split': 9}       0.5996343767546538   3.469    59      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 3, 'min_samples_split': 4}        0.6013376013918397   2.458    60      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 19, 'min_samples_split': 8}       0.6007069584492748   3.748    61      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 16, 'min_samples_split': 8}    0.6014832428943402   3.58     62      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 4, 'min_samples_split': 6}     0.6023492070265482   2.658    63      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 14, 'min_samples_split': 6}    0.6031510585593872   3.392    64      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 6, 'min_samples_split': 3}     0.601555484339515    2.832    65      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 2, 'min_samples_split': 4}     0.601422434493549    2.776    66      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 16, 'min_samples_split': 5}       0.6020133204779852   3.343    67      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 14, 'min_samples_split': 6}    0.599879166738513    3.176    68      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 16, 'min_samples_split': 6}       0.600009685023504    3.384    69      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 1, 'min_samples_split': 7}     0.6015932702508229   2.51     70      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 8, 'min_samples_split': 7}     0.6013977186893489   2.701    71      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 5, 'min_samples_split': 7}     0.6012880339892824   2.474    72      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 20, 'min_samples_split': 8}    0.6018019629778957   3.688    73      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 1, 'min_samples_split': 7}        0.601566431526619    2.366    74      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 5, 'min_samples_split': 3}     0.5988656614889367   2.487    75      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 22, 'min_samples_split': 8}    0.6026870216420587   4.066    76      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 5, 'min_samples_split': 2}        0.60217108847782     2.608    77      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 9}    0.6019685574050225   3.841    78      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 18, 'min_samples_split': 6}       0.6017340423797859   3.579    79      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 3, 'min_samples_split': 2}     0.6018775381511804   2.459    80      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 6, 'min_samples_split': 2}        0.6019806341906919   2.542    81      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 4, 'min_samples_split': 8}     0.6017744350049079   2.295    82      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 1, 'min_samples_split': 5}        0.6014333482602545   2.132    83      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 5, 'min_samples_split': 4}     0.6011016775724602   2.485    84      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 2, 'min_samples_split': 3}     0.601882698769493    2.509    85      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 9, 'min_samples_split': 8}     0.6016977769018671   2.808    86      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 10, 'min_samples_split': 6}    0.6021734909698996   2.736    87      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 3, 'min_samples_split': 8}     0.6016550794937849   2.407    88      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 6, 'min_samples_split': 2}     0.6013843126579413   2.569    89      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 16, 'min_samples_split': 5}    0.601390319765352    3.392    90      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 2, 'min_samples_split': 7}     0.6018528190587332   2.284    91      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 5, 'min_samples_split': 9}     0.6012402498723529   2.573    92      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 19, 'min_samples_split': 3}       0.6015526953577036   3.524    93      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 19, 'min_samples_split': 9}       0.6000532471721961   3.468    94      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 1, 'min_samples_split': 5}        0.6009605802372405   2.268    95      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 12, 'min_samples_split': 9}    0.6001490274508822   3.048    96      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 19, 'min_samples_split': 9}       0.6002758142874377   3.604    97      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 5, 'min_samples_split': 6}     0.6021446462975866   2.577    98      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 22, 'min_samples_split': 6}    0.6030107813067344   3.999    99      \n"
     ]
    }
   ],
   "source": [
    "print ('%-90s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "  \n",
    "\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(25):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoT Sense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARP</th>\n",
       "      <th>EAPOL</th>\n",
       "      <th>IP</th>\n",
       "      <th>ICMP</th>\n",
       "      <th>ICMP6</th>\n",
       "      <th>TCP</th>\n",
       "      <th>UDP</th>\n",
       "      <th>TCP_w_size</th>\n",
       "      <th>HTTP</th>\n",
       "      <th>HTTPS</th>\n",
       "      <th>...</th>\n",
       "      <th>SSDP</th>\n",
       "      <th>DNS</th>\n",
       "      <th>MDNS</th>\n",
       "      <th>NTP</th>\n",
       "      <th>IP_padding</th>\n",
       "      <th>IP_ralert</th>\n",
       "      <th>payload_l</th>\n",
       "      <th>Entropy</th>\n",
       "      <th>MAC</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.175388</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.610262</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>20:f8:5e:ca:91:52</td>\n",
       "      <td>Aria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86079</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8400</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>273</td>\n",
       "      <td>5.144670</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86080</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7991</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>222</td>\n",
       "      <td>5.123301</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86081</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7614</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86082</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7614</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86083</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7613</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:24:80:2a</td>\n",
       "      <td>Withings</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>86084 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ARP  EAPOL  IP  ICMP  ICMP6  TCP  UDP  TCP_w_size  HTTP  HTTPS  ...  \\\n",
       "0        0      1   0     0      0    0    0           0     0      0  ...   \n",
       "1        0      1   0     0      0    0    0           0     0      0  ...   \n",
       "2        0      0   1     0      0    0    1           0     0      0  ...   \n",
       "3        0      0   1     0      0    0    1           0     0      0  ...   \n",
       "4        1      0   0     0      0    0    0           0     0      0  ...   \n",
       "...    ...    ...  ..   ...    ...  ...  ...         ...   ...    ...  ...   \n",
       "86079    0      0   1     0      0    1    0        8400     1      0  ...   \n",
       "86080    0      0   1     0      0    1    0        7991     1      0  ...   \n",
       "86081    0      0   1     0      0    1    0        7614     1      0  ...   \n",
       "86082    0      0   1     0      0    1    0        7614     1      0  ...   \n",
       "86083    0      0   1     0      0    1    0        7613     1      0  ...   \n",
       "\n",
       "       SSDP  DNS  MDNS  NTP  IP_padding  IP_ralert  payload_l   Entropy  \\\n",
       "0         0    0     0    0           0          0          0  4.175388   \n",
       "1         0    0     0    0           0          0          0  1.610262   \n",
       "2         0    0     0    0           0          0        300  0.000000   \n",
       "3         0    0     0    0           0          0        300  0.000000   \n",
       "4         0    0     0    0           0          0          0  0.000000   \n",
       "...     ...  ...   ...  ...         ...        ...        ...       ...   \n",
       "86079     0    0     0    0           0          0        273  5.144670   \n",
       "86080     0    0     0    0           0          0        222  5.123301   \n",
       "86081     0    0     0    0           0          0          0  0.000000   \n",
       "86082     0    0     0    0           0          0          0  0.000000   \n",
       "86083     0    0     0    0           0          0          0  0.000000   \n",
       "\n",
       "                     MAC     Label  \n",
       "0      20:f8:5e:ca:91:52      Aria  \n",
       "1      20:f8:5e:ca:91:52      Aria  \n",
       "2      20:f8:5e:ca:91:52      Aria  \n",
       "3      20:f8:5e:ca:91:52      Aria  \n",
       "4      20:f8:5e:ca:91:52      Aria  \n",
       "...                  ...       ...  \n",
       "86079  00:24:e4:24:80:2a  Withings  \n",
       "86080  00:24:e4:24:80:2a  Withings  \n",
       "86081  00:24:e4:24:80:2a  Withings  \n",
       "86082  00:24:e4:24:80:2a  Withings  \n",
       "86083  00:24:e4:24:80:2a  Withings  \n",
       "\n",
       "[86084 rows x 22 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"Aalto_train_IoTSense.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ARP', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'TCP_w_size',\n",
       "       'HTTP', 'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP',\n",
       "       'IP_padding', 'IP_ralert', 'payload_l', 'Entropy', 'MAC', 'Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['ARP', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'TCP_w_size',\n",
    "       'HTTP', 'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP',\n",
    "       'IP_padding', 'IP_ralert', 'payload_l', 'Entropy', 'Label']\n",
    "df=pd.read_csv(\"Aalto_train_IoTSense.csv\",usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "df=pd.read_csv(\"Aalto_validation_IoTSense.csv\",usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(86084, 20) (21723, 20)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                                                                            F1 Score             Time     No      \n",
      "default                                                                                    0.558691645008419    3.763    24      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 15, 'min_samples_split': 2}    0.5595939405707852   5.056    0       \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 17, 'min_samples_split': 5}       0.5598395436693392   4.321    1       \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 11, 'min_samples_split': 5}       0.5588452549829972   3.867    2       \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 9, 'min_samples_split': 2}        0.558784869621737    4.126    3       \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 10, 'min_samples_split': 7}       0.5584838920603935   3.171    4       \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 17, 'min_samples_split': 3}    0.5594077404103128   5.856    5       \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 6, 'min_samples_split': 2}     0.5578979723451639   4.352    6       \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 5}    0.560104098992348    5.024    7       \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 15, 'min_samples_split': 7}    0.5589826281197542   4.628    8       \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 10, 'min_samples_split': 3}    0.5578691798198041   3.807    9       \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 4}    0.5599425244566223   5.04     10      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 8, 'min_samples_split': 2}        0.5588144386309883   3.277    11      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 10, 'min_samples_split': 5}       0.5591740766338967   3.24     12      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 12, 'min_samples_split': 8}    0.5584870456857225   4.284    13      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 12, 'min_samples_split': 2}    0.557735271771046    4.3      14      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 11, 'min_samples_split': 9}       0.5566510739652359   3.45     15      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 19, 'min_samples_split': 3}       0.5594735232503032   4.289    16      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 16, 'min_samples_split': 4}       0.5589298593903004   4.128    17      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 19, 'min_samples_split': 4}    0.5602022199981592   5.156    18      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 18, 'min_samples_split': 4}    0.5590846382513375   4.714    19      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 17, 'min_samples_split': 3}    0.5597057425436107   5.141    20      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 10, 'min_samples_split': 2}       0.5592884775615491   3.736    21      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 14, 'min_samples_split': 4}       0.5581566203445458   3.984    22      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 6, 'min_samples_split': 2}        0.557453022529816    3.228    23      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 19, 'min_samples_split': 7}    0.5608599075029683   5.074    24      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 7, 'min_samples_split': 6}        0.5586587463390914   3.311    25      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 17, 'min_samples_split': 7}       0.5588660384128833   3.979    26      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 14, 'min_samples_split': 6}    0.559312407927916    4.668    27      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 14, 'min_samples_split': 3}       0.5586897649017216   3.761    28      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 11, 'min_samples_split': 3}    0.5579665105908079   4.155    29      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 8, 'min_samples_split': 6}        0.5593412054537021   3.308    30      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 12, 'min_samples_split': 6}       0.5596784723687197   3.674    31      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 10, 'min_samples_split': 2}    0.5593189676546773   4.357    32      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 14, 'min_samples_split': 8}    0.55838562534971     4.617    33      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 1, 'min_samples_split': 2}        0.5577650446097624   3.211    34      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 18, 'min_samples_split': 6}    0.5600714036121163   5.242    35      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 7, 'min_samples_split': 6}        0.5585427420728055   3.198    36      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 12, 'min_samples_split': 2}       0.5590093924223245   3.926    37      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 17, 'min_samples_split': 6}       0.5601038879511053   4.011    38      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 19, 'min_samples_split': 9}    0.5599754145942392   5.115    39      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 16, 'min_samples_split': 8}       0.5583923741615665   4.255    40      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 10, 'min_samples_split': 3}    0.5575411779984907   4.314    41      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 15, 'min_samples_split': 6}       0.5599309334421654   4.362    42      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 16, 'min_samples_split': 9}       0.5582932766894707   4.177    43      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 10, 'min_samples_split': 8}       0.5572775116595065   3.099    44      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 16, 'min_samples_split': 7}       0.5587037658238144   4.133    45      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 10, 'min_samples_split': 3}    0.5582054018609128   4.351    46      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 15, 'min_samples_split': 3}       0.5585986325141291   3.965    47      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 15, 'min_samples_split': 3}    0.558407787064271    5.62     48      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 19, 'min_samples_split': 6}    0.5601370285703191   6.537    49      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 4}       0.5600779171190392   4.643    50      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 15, 'min_samples_split': 3}    0.5588715078396503   4.686    51      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 15, 'min_samples_split': 6}       0.559884329460211    3.843    52      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 10, 'min_samples_split': 2}       0.559055485102379    3.753    53      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 14, 'min_samples_split': 2}    0.5582864722741846   4.686    54      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 17, 'min_samples_split': 2}    0.560359852182515    5.06     55      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 15, 'min_samples_split': 3}    0.5584946497657707   4.855    56      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 19, 'min_samples_split': 6}       0.5601168896783831   4.718    57      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 18, 'min_samples_split': 9}    0.5596546064605892   4.992    58      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 15, 'min_samples_split': 2}       0.5593275073070032   3.931    59      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 16, 'min_samples_split': 2}    0.560227409908011    5.131    60      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 17, 'min_samples_split': 7}    0.55894843824888     4.95     61      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 19, 'min_samples_split': 6}    0.5607282578231187   5.315    62      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 5, 'min_samples_split': 2}     0.5580912023897762   3.76     63      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 17, 'min_samples_split': 4}       0.5587925475051779   4.098    64      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 11, 'min_samples_split': 6}    0.5588781392163361   4.092    65      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 18, 'min_samples_split': 7}    0.5602472665899518   4.968    66      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 15, 'min_samples_split': 2}       0.55939776875985     3.675    67      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 13, 'min_samples_split': 6}       0.5596147153375949   3.511    68      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 17, 'min_samples_split': 2}       0.5604574769640314   4.224    69      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 19, 'min_samples_split': 3}       0.5589656953041975   4.28     70      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 3, 'min_samples_split': 6}        0.5580918580858953   3.189    71      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 17, 'min_samples_split': 9}    0.5597958118026125   4.936    72      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 16, 'min_samples_split': 5}       0.5595723728690746   4.469    73      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 12, 'min_samples_split': 5}       0.5597731618449867   3.551    74      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 14, 'min_samples_split': 5}       0.5594763513557893   3.745    75      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 12, 'min_samples_split': 4}       0.558105718632929    3.819    76      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 1, 'min_samples_split': 7}        0.5570228142778209   2.813    77      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 17, 'min_samples_split': 8}    0.5598303562551654   4.902    78      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 18, 'min_samples_split': 6}    0.5599906559603507   6.582    79      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 12, 'min_samples_split': 2}    0.5593997775593452   5.163    80      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 12, 'min_samples_split': 8}    0.5586985721701444   4.703    81      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 12, 'min_samples_split': 5}    0.556801146898135    5.05     82      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 12, 'min_samples_split': 7}       0.5582009526342308   4.358    83      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 18, 'min_samples_split': 7}    0.5599311466004808   6.346    84      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 16, 'min_samples_split': 2}    0.5599188390672722   6.076    85      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 18, 'min_samples_split': 8}    0.5597801346037977   6.009    86      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 3, 'min_samples_split': 2}     0.5580562622735399   4.528    87      \n",
      "{'criterion': 'entropy', 'max_depth': 14.0, 'max_features': 16, 'min_samples_split': 8}    0.5524122882268142   5.718    88      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 19, 'min_samples_split': 2}    0.5608715533999528   6.559    89      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 10, 'min_samples_split': 2}       0.5593037990566836   3.536    90      \n",
      "{'criterion': 'entropy', 'max_depth': 14.0, 'max_features': 13, 'min_samples_split': 6}    0.5508370678922492   4.083    91      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 10, 'min_samples_split': 7}    0.5583932688356819   4.165    92      \n",
      "{'criterion': 'gini', 'max_depth': 18.0, 'max_features': 19, 'min_samples_split': 3}       0.5498665624201955   4.979    93      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 9}       0.5598487175963873   5.208    94      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 19, 'min_samples_split': 4}    0.5594798814967475   6.362    95      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 10, 'min_samples_split': 8}    0.5583394313548576   5.366    96      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 5}       0.5589600071932559   5.078    97      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 19, 'min_samples_split': 3}       0.5575940720565131   5.073    98      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 14, 'min_samples_split': 2}    0.5591307646108168   5.53     99      \n"
     ]
    }
   ],
   "source": [
    "print ('%-90s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "  \n",
    "\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(25):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "________________"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UNSW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from scipy.stats import randint as sp_randint\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import uniform as sp_randFloat\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from time import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import randint as sp_randInt\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, PredefinedSplit\n",
    "from sklearn.metrics import make_scorer\n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoTSentinel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARP</th>\n",
       "      <th>LLC</th>\n",
       "      <th>EAPOL</th>\n",
       "      <th>IP</th>\n",
       "      <th>ICMP</th>\n",
       "      <th>ICMP6</th>\n",
       "      <th>TCP</th>\n",
       "      <th>UDP</th>\n",
       "      <th>HTTP</th>\n",
       "      <th>HTTPS</th>\n",
       "      <th>...</th>\n",
       "      <th>NTP</th>\n",
       "      <th>IP_padding</th>\n",
       "      <th>IP_add_count</th>\n",
       "      <th>IP_ralert</th>\n",
       "      <th>Portcl_src</th>\n",
       "      <th>Portcl_dst</th>\n",
       "      <th>Pck_size</th>\n",
       "      <th>Pck_rawdata</th>\n",
       "      <th>MAC</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>1</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>177</td>\n",
       "      <td>1</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>187</td>\n",
       "      <td>1</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282806</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282807</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>262</td>\n",
       "      <td>1</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282808</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282809</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282810</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>282811 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ARP  LLC  EAPOL  IP  ICMP  ICMP6  TCP  UDP  HTTP  HTTPS  ...  NTP  \\\n",
       "0         0    0      0   1     0      0    0    1     0      0  ...    1   \n",
       "1         0    0      1   0     0      0    0    0     0      0  ...    0   \n",
       "2         0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "3         0    0      0   1     0      0    0    1     0      0  ...    0   \n",
       "4         0    0      0   1     0      0    0    1     0      0  ...    0   \n",
       "...     ...  ...    ...  ..   ...    ...  ...  ...   ...    ...  ...  ...   \n",
       "282806    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "282807    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "282808    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "282809    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "282810    0    0      0   1     0      0    1    0     1      0  ...    0   \n",
       "\n",
       "        IP_padding  IP_add_count  IP_ralert  Portcl_src  Portcl_dst  Pck_size  \\\n",
       "0                0            37          0           2           1        76   \n",
       "1                0             0          0           0           0        95   \n",
       "2                0            31          0           3           1        60   \n",
       "3                0            37          0           2           2       177   \n",
       "4                0            36          0           3           2       187   \n",
       "...            ...           ...        ...         ...         ...       ...   \n",
       "282806           0             4          0           3           1        40   \n",
       "282807           0             4          0           3           1       262   \n",
       "282808           0             4          0           3           1        40   \n",
       "282809           0             4          0           3           1        40   \n",
       "282810           0             4          0           3           1        40   \n",
       "\n",
       "        Pck_rawdata                MAC                 Label  \n",
       "0                 0  44:65:0d:56:cc:d3           Amazon Echo  \n",
       "1                 1  44:65:0d:56:cc:d3           Amazon Echo  \n",
       "2                 0  44:65:0d:56:cc:d3           Amazon Echo  \n",
       "3                 1  44:65:0d:56:cc:d3           Amazon Echo  \n",
       "4                 1  44:65:0d:56:cc:d3           Amazon Echo  \n",
       "...             ...                ...                   ...  \n",
       "282806            0  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282807            1  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282808            0  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282809            0  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282810            0  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "\n",
       "[282811 rows x 25 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"UNSW_train_IoTSentinel.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ARP', 'LLC', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'HTTP',\n",
       "       'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP', 'IP_padding',\n",
       "       'IP_add_count', 'IP_ralert', 'Portcl_src', 'Portcl_dst', 'Pck_size',\n",
       "       'Pck_rawdata', 'MAC', 'Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['ARP', 'LLC', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'HTTP',\n",
    "       'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP', 'IP_padding',\n",
    "       'IP_add_count', 'IP_ralert', 'Portcl_src', 'Portcl_dst', 'Pck_size',\n",
    "       'Pck_rawdata',  'Label']\n",
    "df=pd.read_csv(\"UNSW_train_IoTSentinel.csv\",usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "df=pd.read_csv(\"UNSW_validation_IoTSentinel.csv\",usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(282811, 23) (70449, 23)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                                                                            F1 Score             Time     No      \n",
      "default                                                                                    0.504726029658401    13.599   24      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 6, 'min_samples_split': 5}     0.5011076161175921   10.733   0       \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 1, 'min_samples_split': 4}        0.5099407835954021   10.322   1       \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 20, 'min_samples_split': 7}       0.5034832751111291   17.245   2       \n",
      "{'criterion': 'gini', 'max_depth': 14.0, 'max_features': 8, 'min_samples_split': 8}        0.4840597514969355   11.02    3       \n",
      "{'criterion': 'gini', 'max_depth': 16.0, 'max_features': 8, 'min_samples_split': 9}        0.4902067490058035   12.682   4       \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 12, 'min_samples_split': 2}    0.5016726208159694   12.441   5       \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 7, 'min_samples_split': 9}        0.5034736913310647   10.913   6       \n",
      "{'criterion': 'gini', 'max_depth': 16.0, 'max_features': 8, 'min_samples_split': 6}        0.5008376244569992   12.164   7       \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 7, 'min_samples_split': 3}     0.5039092700355589   10.641   8       \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 15, 'min_samples_split': 6}    0.4996327590271571   16.35    9       \n",
      "{'criterion': 'gini', 'max_depth': 18.0, 'max_features': 11, 'min_samples_split': 4}       0.5061354111005771   12.233   10      \n",
      "{'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 4, 'min_samples_split': 5}     0.4986113240344758   9.395    11      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 20, 'min_samples_split': 7}       0.5050065394085064   18.013   12      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 12, 'min_samples_split': 7}    0.4960164778500743   12.823   13      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 7, 'min_samples_split': 3}        0.5075217944059709   12.056   14      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 1, 'min_samples_split': 4}        0.5077250843318716   10.546   15      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 11, 'min_samples_split': 5}    0.4986665069490616   12.909   16      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 1, 'min_samples_split': 2}        0.5100791293821751   9.719    17      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 7, 'min_samples_split': 5}     0.5006937243444661   12.388   18      \n",
      "{'criterion': 'gini', 'max_depth': 18.0, 'max_features': 10, 'min_samples_split': 2}       0.5038110574780196   13.683   19      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 4, 'min_samples_split': 3}     0.5081433291552055   10.394   20      \n",
      "{'criterion': 'gini', 'max_depth': 16.0, 'max_features': 17, 'min_samples_split': 2}       0.5029991392838162   14.67    21      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 2}    0.4986205079473723   12.995   22      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 15, 'min_samples_split': 6}       0.5062511840151633   18.285   23      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 6, 'min_samples_split': 7}        0.5090669921246591   12.73    24      \n",
      "{'criterion': 'gini', 'max_depth': 15.0, 'max_features': 8, 'min_samples_split': 5}        0.4906767680208231   12.137   25      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 7, 'min_samples_split': 5}        0.5070183295947354   11.813   26      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 3, 'min_samples_split': 2}     0.5085798825005508   10.277   27      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 2, 'min_samples_split': 2}        0.5119003823868639   10.773   28      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 22, 'min_samples_split': 3}       0.5044494289399762   20.797   29      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 8, 'min_samples_split': 6}        0.5054691586463822   11.663   30      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 6, 'min_samples_split': 5}        0.5091749882257619   11.075   31      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 20, 'min_samples_split': 5}    0.5012757402327029   16.771   32      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 7, 'min_samples_split': 3}     0.5031779087491085   10.291   33      \n",
      "{'criterion': 'entropy', 'max_depth': 11.0, 'max_features': 19, 'min_samples_split': 4}    0.4911663108778608   13.677   34      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 5, 'min_samples_split': 6}     0.5015989686107943   9.756    35      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 10, 'min_samples_split': 7}    0.4965797407924477   12.52    36      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 21, 'min_samples_split': 7}       0.5033311331482704   18.193   37      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 6, 'min_samples_split': 5}     0.504662660944637    10.959   38      \n",
      "{'criterion': 'gini', 'max_depth': 15.0, 'max_features': 6, 'min_samples_split': 6}        0.4872157537399422   10.647   39      \n",
      "{'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 3, 'min_samples_split': 4}     0.4982487639434857   9.373    40      \n",
      "{'criterion': 'gini', 'max_depth': 17.0, 'max_features': 10, 'min_samples_split': 6}       0.5050965370624528   11.397   41      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 5, 'min_samples_split': 2}     0.5026617658618661   10.701   42      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 20, 'min_samples_split': 4}    0.500004790575644    16.968   43      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 18, 'min_samples_split': 6}    0.49891426702570124  16.496   44      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 15, 'min_samples_split': 8}    0.4944616282131263   17.819   45      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 13, 'min_samples_split': 6}    0.503500611071228    14.531   46      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 2, 'min_samples_split': 9}        0.5071728953550535   12.655   47      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 14, 'min_samples_split': 8}       0.5078407122747342   18.85    48      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 18, 'min_samples_split': 3}    0.5013723903832534   18.582   49      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 4, 'min_samples_split': 5}     0.5047723891239894   14.321   50      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 13, 'min_samples_split': 9}    0.5004311190785782   18.093   51      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 9, 'min_samples_split': 6}     0.5011535971703264   14.24    52      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 8, 'min_samples_split': 4}        0.5092393261112336   11.935   53      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 1, 'min_samples_split': 3}        0.5122428509301188   10.181   54      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 15, 'min_samples_split': 7}    0.5019485168697276   17.64    55      \n",
      "{'criterion': 'gini', 'max_depth': 16.0, 'max_features': 13, 'min_samples_split': 8}       0.49506843259753497  14.223   56      \n",
      "{'criterion': 'entropy', 'max_depth': 14.0, 'max_features': 2, 'min_samples_split': 6}     0.4904174980600612   10.34    57      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 1, 'min_samples_split': 4}        0.5079040871241363   12.143   58      \n",
      "{'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 22, 'min_samples_split': 7}    0.49945671873638003  21.553   59      \n",
      "{'criterion': 'gini', 'max_depth': 17.0, 'max_features': 16, 'min_samples_split': 5}       0.5043649900323741   18.325   60      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 18, 'min_samples_split': 3}    0.5006713367493819   19.049   61      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 18, 'min_samples_split': 5}    0.5001553785231405   17.376   62      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 11, 'min_samples_split': 6}       0.5048528597366588   15.338   63      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 7, 'min_samples_split': 8}     0.5031902178663394   14.358   64      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 15, 'min_samples_split': 6}       0.5068745596054003   17.978   65      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 17, 'min_samples_split': 2}       0.5071040567497408   20.052   66      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 3, 'min_samples_split': 6}     0.502716077553813    13.01    67      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 22, 'min_samples_split': 6}       0.49997792082799825  23.418   68      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 5, 'min_samples_split': 7}        0.5096458826834419   12.484   69      \n",
      "{'criterion': 'gini', 'max_depth': 18.0, 'max_features': 2, 'min_samples_split': 7}        0.49659956126445826  12.039   70      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 2, 'min_samples_split': 5}        0.5099309512267065   12.101   71      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 12, 'min_samples_split': 9}       0.5063904521566465   16.185   72      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 8, 'min_samples_split': 7}        0.5118625572364861   14.977   73      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 19, 'min_samples_split': 7}    0.49795256079982947  20.013   74      \n",
      "{'criterion': 'gini', 'max_depth': 17.0, 'max_features': 15, 'min_samples_split': 7}       0.5026367955168828   18.117   75      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 14, 'min_samples_split': 7}       0.5077700723822438   20.633   76      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 5, 'min_samples_split': 9}     0.5035560127069834   12.458   77      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 21, 'min_samples_split': 9}    0.4968526702523654   21.048   78      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 11, 'min_samples_split': 6}       0.5065265917544086   15.711   79      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 19, 'min_samples_split': 8}    0.5005613412380567   17.986   80      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 6, 'min_samples_split': 7}        0.5045096101503401   12.857   81      \n",
      "{'criterion': 'gini', 'max_depth': 16.0, 'max_features': 5, 'min_samples_split': 2}        0.49421511220560727  13.151   82      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 10, 'min_samples_split': 7}       0.5068203934279417   17.213   83      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 3, 'min_samples_split': 8}        0.507005281426433    13.029   84      \n",
      "{'criterion': 'gini', 'max_depth': 16.0, 'max_features': 15, 'min_samples_split': 3}       0.5033434101711056   17.28    85      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 11, 'min_samples_split': 9}       0.5017773368647607   12.825   86      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 9, 'min_samples_split': 4}        0.5090498916161378   14.851   87      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 2, 'min_samples_split': 7}        0.5061024492241123   13.76    88      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 1, 'min_samples_split': 2}     0.5085771956701629   12.947   89      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 5, 'min_samples_split': 7}        0.510062955924215    13.323   90      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 2, 'min_samples_split': 6}        0.5083278988030088   11.858   91      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 10, 'min_samples_split': 4}    0.5025879483088694   16.41    92      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 4, 'min_samples_split': 8}        0.5080662422847475   12.863   93      \n",
      "{'criterion': 'entropy', 'max_depth': 11.0, 'max_features': 6, 'min_samples_split': 6}     0.4743027829974632   11.877   94      \n",
      "{'criterion': 'gini', 'max_depth': 15.0, 'max_features': 6, 'min_samples_split': 9}        0.48835584837989865  17.79    95      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 2, 'min_samples_split': 5}     0.506027121639033    12.968   96      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 2, 'min_samples_split': 7}     0.5073068351019956   8.709    97      \n",
      "{'criterion': 'entropy', 'max_depth': 14.0, 'max_features': 1, 'min_samples_split': 9}     0.4657560423624059   8.486    98      \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 13, 'min_samples_split': 2}       0.508157148214933    12.795   99      \n"
     ]
    }
   ],
   "source": [
    "print ('%-90s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "  \n",
    "\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(25):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoT Sense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARP</th>\n",
       "      <th>EAPOL</th>\n",
       "      <th>IP</th>\n",
       "      <th>ICMP</th>\n",
       "      <th>ICMP6</th>\n",
       "      <th>TCP</th>\n",
       "      <th>UDP</th>\n",
       "      <th>TCP_w_size</th>\n",
       "      <th>HTTP</th>\n",
       "      <th>HTTPS</th>\n",
       "      <th>...</th>\n",
       "      <th>SSDP</th>\n",
       "      <th>DNS</th>\n",
       "      <th>MDNS</th>\n",
       "      <th>NTP</th>\n",
       "      <th>IP_padding</th>\n",
       "      <th>IP_ralert</th>\n",
       "      <th>payload_l</th>\n",
       "      <th>Entropy</th>\n",
       "      <th>MAC</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>1.290573</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.711966</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4380</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>149</td>\n",
       "      <td>5.710742</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>159</td>\n",
       "      <td>5.696487</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282806</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8014</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282807</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8014</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>222</td>\n",
       "      <td>5.120422</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282808</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7637</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282809</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7637</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282810</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7636</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>282811 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ARP  EAPOL  IP  ICMP  ICMP6  TCP  UDP  TCP_w_size  HTTP  HTTPS  ...  \\\n",
       "0         0      0   1     0      0    0    1           0     0      0  ...   \n",
       "1         0      1   0     0      0    0    0           0     0      0  ...   \n",
       "2         0      0   1     0      0    1    0        4380     1      0  ...   \n",
       "3         0      0   1     0      0    0    1           0     0      0  ...   \n",
       "4         0      0   1     0      0    0    1           0     0      0  ...   \n",
       "...     ...    ...  ..   ...    ...  ...  ...         ...   ...    ...  ...   \n",
       "282806    0      0   1     0      0    1    0        8014     1      0  ...   \n",
       "282807    0      0   1     0      0    1    0        8014     1      0  ...   \n",
       "282808    0      0   1     0      0    1    0        7637     1      0  ...   \n",
       "282809    0      0   1     0      0    1    0        7637     1      0  ...   \n",
       "282810    0      0   1     0      0    1    0        7636     1      0  ...   \n",
       "\n",
       "        SSDP  DNS  MDNS  NTP  IP_padding  IP_ralert  payload_l   Entropy  \\\n",
       "0          0    0     0    1           0          0         48  1.290573   \n",
       "1          0    0     0    0           0          0          0  1.711966   \n",
       "2          0    0     0    0           0          0          0  0.000000   \n",
       "3          0    0     0    0           0          0        149  5.710742   \n",
       "4          0    0     0    0           0          0        159  5.696487   \n",
       "...      ...  ...   ...  ...         ...        ...        ...       ...   \n",
       "282806     0    0     0    0           0          0          0  0.000000   \n",
       "282807     0    0     0    0           0          0        222  5.120422   \n",
       "282808     0    0     0    0           0          0          0  0.000000   \n",
       "282809     0    0     0    0           0          0          0  0.000000   \n",
       "282810     0    0     0    0           0          0          0  0.000000   \n",
       "\n",
       "                      MAC                 Label  \n",
       "0       44:65:0d:56:cc:d3           Amazon Echo  \n",
       "1       44:65:0d:56:cc:d3           Amazon Echo  \n",
       "2       44:65:0d:56:cc:d3           Amazon Echo  \n",
       "3       44:65:0d:56:cc:d3           Amazon Echo  \n",
       "4       44:65:0d:56:cc:d3           Amazon Echo  \n",
       "...                   ...                   ...  \n",
       "282806  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282807  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282808  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282809  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "282810  00:24:e4:1b:6f:96  Withings Smart scale  \n",
       "\n",
       "[282811 rows x 22 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"UNSW_train_IoTSense.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ARP', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'TCP_w_size',\n",
       "       'HTTP', 'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP',\n",
       "       'IP_padding', 'IP_ralert', 'payload_l', 'Entropy', 'MAC', 'Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['ARP', 'EAPOL', 'IP', 'ICMP', 'ICMP6', 'TCP', 'UDP', 'TCP_w_size',\n",
    "       'HTTP', 'HTTPS', 'DHCP', 'BOOTP', 'SSDP', 'DNS', 'MDNS', 'NTP',\n",
    "       'IP_padding', 'IP_ralert', 'payload_l', 'Entropy', 'Label']\n",
    "df=pd.read_csv(\"UNSW_train_IoTSense.csv\",usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "df=pd.read_csv(\"UNSW_validation_IoTSense.csv\",usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(282811, 20) (70449, 20)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                                                                            F1 Score             Time     No      \n",
      "default                                                                                    0.7001257412876442   12.571   24      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 10, 'min_samples_split': 2}    0.6962463666427438   16.541   0       \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 16, 'min_samples_split': 6}       0.6838823911295954   18.952   1       \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 13, 'min_samples_split': 4}    0.6936496038475042   18.467   2       \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 6, 'min_samples_split': 4}     0.6912024571500703   14.505   3       \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 5, 'min_samples_split': 3}        0.6893947984204619   12.652   4       \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 6, 'min_samples_split': 2}        0.6899510406043099   14.627   5       \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 9, 'min_samples_split': 6}     0.6928022946125325   20.951   6       \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 9, 'min_samples_split': 3}        0.6910147378351368   19.288   7       \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 7, 'min_samples_split': 2}        0.6918778120243164   16.435   8       \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 9, 'min_samples_split': 8}     0.688732867102963    18.739   9       \n",
      "{'criterion': 'gini', 'max_depth': 23.0, 'max_features': 12, 'min_samples_split': 2}       0.690491354887647    15.243   10      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 11, 'min_samples_split': 5}    0.6957719438700803   16.739   11      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 9, 'min_samples_split': 9}     0.6863490179307066   15.766   12      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 9, 'min_samples_split': 5}        0.6839057256556476   14.682   13      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 12, 'min_samples_split': 4}    0.6951327129964894   18.031   14      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 9, 'min_samples_split': 2}        0.6912623877924392   13.39    15      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 16, 'min_samples_split': 4}    0.6911555500792397   18.095   16      \n",
      "{'criterion': 'gini', 'max_depth': 25.0, 'max_features': 19, 'min_samples_split': 6}       0.691319250520437    21.042   17      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 5, 'min_samples_split': 2}     0.6932637029791313   17.78    18      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 3, 'min_samples_split': 9}     0.6813813756446596   17.994   19      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 12, 'min_samples_split': 8}    0.6905132076847235   21.092   20      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 12, 'min_samples_split': 7}    0.6922000969803984   19.078   21      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 11, 'min_samples_split': 4}    0.6985094029104743   18.59    22      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 2, 'min_samples_split': 5}     0.688680060771191    13.692   23      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 14, 'min_samples_split': 7}       0.6729053258232088   19.847   24      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 14, 'min_samples_split': 9}    0.690791475801948    19.001   25      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 9, 'min_samples_split': 3}        0.689423557315838    14.935   26      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 14, 'min_samples_split': 2}       0.6929113635081885   17.307   27      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 12, 'min_samples_split': 9}    0.6906661681031472   18.024   28      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 14, 'min_samples_split': 3}    0.6954950672962711   17.919   29      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 9, 'min_samples_split': 4}        0.6752863725180661   14.823   30      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 5}    0.6939793078044088   23.807   31      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 13, 'min_samples_split': 6}    0.6960335610638793   16.733   32      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 17, 'min_samples_split': 5}       0.68859082912847     16.22    33      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 8, 'min_samples_split': 4}     0.6939052438445559   16.382   34      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 11, 'min_samples_split': 7}    0.6907153997235231   17.37    35      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 11, 'min_samples_split': 5}    0.6957428893615905   17.498   36      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 14, 'min_samples_split': 4}       0.6879947391126389   16.527   37      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 10, 'min_samples_split': 8}    0.6890183156304949   15.672   38      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 3}       0.6860991474660654   14.942   39      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 14, 'min_samples_split': 8}    0.6924603073747808   18.62    40      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 19, 'min_samples_split': 9}       0.6916950973740211   17.531   41      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 6, 'min_samples_split': 8}     0.6912850163871483   14.64    42      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 13, 'min_samples_split': 9}    0.6915718682190396   17.658   43      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 9, 'min_samples_split': 9}     0.6837315616684481   15.4     44      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 13, 'min_samples_split': 9}    0.6847142639138403   18.055   45      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 13, 'min_samples_split': 5}    0.6948649111091717   19.62    46      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 8, 'min_samples_split': 6}     0.6872834908850775   16.565   47      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 4, 'min_samples_split': 3}     0.6896827700556419   15.197   48      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 7, 'min_samples_split': 8}     0.6883226495321716   15.687   49      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 14, 'min_samples_split': 3}       0.6941825750005505   16.143   50      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 2, 'min_samples_split': 3}        0.6871883610876159   12.064   51      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 17, 'min_samples_split': 3}    0.6910708862009356   18.878   52      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 8, 'min_samples_split': 5}        0.6890964566116566   13.411   53      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 17, 'min_samples_split': 2}    0.697008296565034    19.799   54      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 13, 'min_samples_split': 5}    0.6876337004224824   17.405   55      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 12, 'min_samples_split': 9}    0.6880851815352907   16.976   56      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 11, 'min_samples_split': 3}       0.6874227213264116   15.943   57      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 12, 'min_samples_split': 2}       0.6895681016627089   15.402   58      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 13, 'min_samples_split': 2}    0.7011795266098307   20.808   59      \n",
      "{'criterion': 'entropy', 'max_depth': 15.0, 'max_features': 13, 'min_samples_split': 2}    0.68570676479959     19.183   60      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 8, 'min_samples_split': 5}     0.6920184736238427   19.815   61      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 1, 'min_samples_split': 8}     0.6845832506103366   12.351   62      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 17, 'min_samples_split': 2}    0.6979122231588134   19.791   63      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 11, 'min_samples_split': 2}    0.7020353530806149   17.137   64      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 4, 'min_samples_split': 2}     0.6868099548358466   13.787   65      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 13, 'min_samples_split': 3}       0.688253723725793    15.54    66      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 13, 'min_samples_split': 8}       0.6806916351085426   15.052   67      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 5, 'min_samples_split': 8}     0.6890773947162215   13.896   68      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 10, 'min_samples_split': 2}    0.6967636461415959   16.49    69      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 8, 'min_samples_split': 7}        0.6812277628747292   14.678   70      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 14, 'min_samples_split': 2}       0.6884250410584416   15.182   71      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 15, 'min_samples_split': 2}    0.7004214185593153   18.017   72      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 13, 'min_samples_split': 4}    0.6922988544803507   18.252   73      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 9, 'min_samples_split': 3}     0.6940627907550798   20.817   74      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 17, 'min_samples_split': 6}       0.6893125037038383   17.115   75      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 16, 'min_samples_split': 9}    0.6896878424446047   18.531   76      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 14, 'min_samples_split': 4}       0.6846989921880179   16.228   77      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 14, 'min_samples_split': 5}    0.6924721742061712   17.811   78      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 14, 'min_samples_split': 5}    0.6914194529082218   17.509   79      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 12, 'min_samples_split': 4}    0.6897462129585402   18.612   80      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 1, 'min_samples_split': 9}     0.6839947433220651   14.133   81      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 9, 'min_samples_split': 4}     0.6939577365374173   18.133   82      \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 15, 'min_samples_split': 6}       0.6838624918285072   17.124   83      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 8, 'min_samples_split': 4}        0.6950351299140831   16.795   84      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 9, 'min_samples_split': 2}        0.6921802353338171   15.179   85      \n",
      "{'criterion': 'gini', 'max_depth': 21.0, 'max_features': 4, 'min_samples_split': 9}        0.6788470154664276   11.689   86      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 15, 'min_samples_split': 6}       0.6855807525577119   16.068   87      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 18, 'min_samples_split': 2}    0.6955946134142823   22.037   88      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 15, 'min_samples_split': 2}    0.6974422511743148   18.0     89      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 11, 'min_samples_split': 6}    0.6960609344462718   15.764   90      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 5, 'min_samples_split': 5}     0.6864748030967359   14.769   91      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 4, 'min_samples_split': 2}     0.6926984636534825   14.347   92      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 2}    0.6982015818519289   17.625   93      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 19, 'min_samples_split': 3}    0.6941474900255398   21.093   94      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 3, 'min_samples_split': 2}        0.6901584383910708   11.939   95      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 15, 'min_samples_split': 6}    0.6965631117482737   19.254   96      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 2, 'min_samples_split': 2}     0.6836667333284371   12.766   97      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 5, 'min_samples_split': 2}     0.695810475036335    14.534   98      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 18, 'min_samples_split': 7}    0.6943752708886707   21.701   99      \n"
     ]
    }
   ],
   "source": [
    "print ('%-90s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "  \n",
    "\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(25):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoTDevID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pck_size</th>\n",
       "      <th>Ether_type</th>\n",
       "      <th>LLC_dsap</th>\n",
       "      <th>LLC_ssap</th>\n",
       "      <th>LLC_ctrl</th>\n",
       "      <th>EAPOL_version</th>\n",
       "      <th>EAPOL_type</th>\n",
       "      <th>EAPOL_len</th>\n",
       "      <th>IP_version</th>\n",
       "      <th>IP_ihl</th>\n",
       "      <th>...</th>\n",
       "      <th>dport_bare</th>\n",
       "      <th>TCP_sport</th>\n",
       "      <th>TCP_dport</th>\n",
       "      <th>UDP_sport</th>\n",
       "      <th>UDP_dport</th>\n",
       "      <th>payload_bytes</th>\n",
       "      <th>payload_entropy</th>\n",
       "      <th>MAC</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>76</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>123</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>33247</td>\n",
       "      <td>123</td>\n",
       "      <td>48</td>\n",
       "      <td>1.290573</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>10</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>95</td>\n",
       "      <td>34958</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>1.711966</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>1</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>53487</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>4</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>177</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>33434</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>47532</td>\n",
       "      <td>33434</td>\n",
       "      <td>149</td>\n",
       "      <td>5.710742</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>12</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>187</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>33434</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>49280</td>\n",
       "      <td>33434</td>\n",
       "      <td>159</td>\n",
       "      <td>5.696487</td>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>12</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282806</th>\n",
       "      <td>40</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>49153</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>4</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282807</th>\n",
       "      <td>262</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>49153</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>222</td>\n",
       "      <td>5.120422</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>5</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282808</th>\n",
       "      <td>40</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>49153</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>4</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282809</th>\n",
       "      <td>40</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>49153</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>4</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282810</th>\n",
       "      <td>40</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>49153</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>4</td>\n",
       "      <td>Withings Smart scale</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>282811 rows × 97 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pck_size  Ether_type  LLC_dsap  LLC_ssap  LLC_ctrl  EAPOL_version  \\\n",
       "0             76        2048         0         0         0              0   \n",
       "1             95       34958         0         0         0              1   \n",
       "2             60        2048         0         0         0              0   \n",
       "3            177        2048         0         0         0              0   \n",
       "4            187        2048         0         0         0              0   \n",
       "...          ...         ...       ...       ...       ...            ...   \n",
       "282806        40        2048         0         0         0              0   \n",
       "282807       262        2048         0         0         0              0   \n",
       "282808        40        2048         0         0         0              0   \n",
       "282809        40        2048         0         0         0              0   \n",
       "282810        40        2048         0         0         0              0   \n",
       "\n",
       "        EAPOL_type  EAPOL_len  IP_version  IP_ihl  ...  dport_bare  TCP_sport  \\\n",
       "0                0          0           4       5  ...         123          0   \n",
       "1                3         95           0       0  ...           0          0   \n",
       "2                0          0           4       5  ...          80      53487   \n",
       "3                0          0           4       5  ...       33434          0   \n",
       "4                0          0           4       5  ...       33434          0   \n",
       "...            ...        ...         ...     ...  ...         ...        ...   \n",
       "282806           0          0           4       5  ...          80      49153   \n",
       "282807           0          0           4       5  ...          80      49153   \n",
       "282808           0          0           4       5  ...          80      49153   \n",
       "282809           0          0           4       5  ...          80      49153   \n",
       "282810           0          0           4       5  ...          80      49153   \n",
       "\n",
       "        TCP_dport  UDP_sport  UDP_dport  payload_bytes  payload_entropy  \\\n",
       "0               0      33247        123             48         1.290573   \n",
       "1               0          0          0             95         1.711966   \n",
       "2              80          0          0              0         0.000000   \n",
       "3               0      47532      33434            149         5.710742   \n",
       "4               0      49280      33434            159         5.696487   \n",
       "...           ...        ...        ...            ...              ...   \n",
       "282806         80          0          0              0         0.000000   \n",
       "282807         80          0          0            222         5.120422   \n",
       "282808         80          0          0              0         0.000000   \n",
       "282809         80          0          0              0         0.000000   \n",
       "282810         80          0          0              0         0.000000   \n",
       "\n",
       "                      MAC  Protocol                 Label  \n",
       "0       44:65:0d:56:cc:d3        10           Amazon Echo  \n",
       "1       44:65:0d:56:cc:d3         1           Amazon Echo  \n",
       "2       44:65:0d:56:cc:d3         4           Amazon Echo  \n",
       "3       44:65:0d:56:cc:d3        12           Amazon Echo  \n",
       "4       44:65:0d:56:cc:d3        12           Amazon Echo  \n",
       "...                   ...       ...                   ...  \n",
       "282806  00:24:e4:1b:6f:96         4  Withings Smart scale  \n",
       "282807  00:24:e4:1b:6f:96         5  Withings Smart scale  \n",
       "282808  00:24:e4:1b:6f:96         4  Withings Smart scale  \n",
       "282809  00:24:e4:1b:6f:96         4  Withings Smart scale  \n",
       "282810  00:24:e4:1b:6f:96         4  Withings Smart scale  \n",
       "\n",
       "[282811 rows x 97 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"UNSW_train_IoTDevID.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['pck_size', 'Ether_type', 'LLC_dsap', 'LLC_ssap', 'LLC_ctrl',\n",
       "       'EAPOL_version', 'EAPOL_type', 'EAPOL_len', 'IP_version', 'IP_ihl',\n",
       "       'IP_tos', 'IP_len', 'IP_flags', 'IP_Z', 'IP_MF', 'IP_id', 'IP_chksum',\n",
       "       'IP_DF', 'IP_frag', 'IP_ttl', 'IP_proto', 'IP_options', 'IP_add_count',\n",
       "       'ICMP_type', 'ICMP_code', 'ICMP_chksum', 'ICMP_id', 'ICMP_seq',\n",
       "       'ICMP_ts_ori', 'ICMP_ts_rx', 'ICMP_ts_tx', 'ICMP_ptr', 'ICMP_reserved',\n",
       "       'ICMP_length', 'ICMP_nexthopmtu', 'ICMP_unused', 'TCP_seq', 'TCP_ack',\n",
       "       'TCP_dataofs', 'TCP_reserved', 'TCP_flags', 'TCP_FIN', 'TCP_SYN',\n",
       "       'TCP_RST', 'TCP_PSH', 'TCP_ACK', 'TCP_URG', 'TCP_ECE', 'TCP_CWR',\n",
       "       'TCP_window', 'TCP_chksum', 'TCP_urgptr', 'TCP_options', 'UDP_len',\n",
       "       'UDP_chksum', 'DHCP_options', 'BOOTP_op', 'BOOTP_htype', 'BOOTP_hlen',\n",
       "       'BOOTP_hops', 'BOOTP_xid', 'BOOTP_secs', 'BOOTP_flags', 'BOOTP_sname',\n",
       "       'BOOTP_file', 'BOOTP_options', 'DNS_length', 'DNS_id', 'DNS_qr',\n",
       "       'DNS_opcode', 'DNS_aa', 'DNS_tc', 'DNS_rd', 'DNS_ra', 'DNS_z', 'DNS_ad',\n",
       "       'DNS_cd', 'DNS_rcode', 'DNS_qdcount', 'DNS_ancount', 'DNS_nscount',\n",
       "       'DNS_arcount', 'sport_class', 'dport_class', 'sport23', 'dport23',\n",
       "       'sport_bare', 'dport_bare', 'TCP_sport', 'TCP_dport', 'UDP_sport',\n",
       "       'UDP_dport', 'payload_bytes', 'payload_entropy', 'MAC', 'Protocol',\n",
       "       'Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['pck_size', 'Ether_type', 'LLC_ctrl', 'EAPOL_version', 'EAPOL_type', 'IP_ihl', 'IP_tos', 'IP_len', 'IP_flags', 'IP_DF', 'IP_ttl', 'IP_options', 'ICMP_code', 'TCP_dataofs', 'TCP_FIN', 'TCP_ACK', 'TCP_window', 'UDP_len', 'DHCP_options', 'BOOTP_hlen', 'BOOTP_flags', 'BOOTP_sname', 'BOOTP_file', 'BOOTP_options', 'DNS_qr', 'DNS_rd', 'DNS_qdcount', 'dport_class', 'payload_bytes', 'entropy',\n",
    "'Label']\n",
    "df=pd.read_csv(\"UNSW_train_IoTDevID.csv\",usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "\n",
    "df=pd.read_csv(\"UNSW_validation_IoTDevID.csv\",usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(282811, 30) (70449, 30)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomizedSearchCV  DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                                                                            F1 Score             Time     No      \n",
      "default                                                                                    0.8195978420312032   21.566   24      \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 5, 'min_samples_split': 7}     0.8323455036302395   14.317   0       \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 9, 'min_samples_split': 4}        0.8280182896840249   16.834   1       \n",
      "{'criterion': 'gini', 'max_depth': 22.0, 'max_features': 22, 'min_samples_split': 7}       0.8234451217809853   21.915   2       \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 17, 'min_samples_split': 8}    0.8401239596610508   22.14    3       \n",
      "{'criterion': 'entropy', 'max_depth': 30.0, 'max_features': 17, 'min_samples_split': 3}    0.8389146492593934   21.593   4       \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 24, 'min_samples_split': 3}    0.8298105842136465   24.507   5       \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 29, 'min_samples_split': 5}    0.8266814904437577   27.172   6       \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 6, 'min_samples_split': 4}     0.8346399252015262   13.768   7       \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 19, 'min_samples_split': 4}       0.8353015991591904   22.487   8       \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 5, 'min_samples_split': 2}     0.8356332249673563   13.116   9       \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 18, 'min_samples_split': 3}    0.8367648201378618   20.709   10      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 21, 'min_samples_split': 8}    0.8288974575127522   22.085   11      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 10, 'min_samples_split': 5}    0.835427634371817    16.389   12      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 24, 'min_samples_split': 6}    0.8283315455395912   24.892   13      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 9, 'min_samples_split': 7}     0.8326494183633478   16.294   14      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 28, 'min_samples_split': 5}    0.8281997425077925   27.357   15      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 4, 'min_samples_split': 3}     0.8361018737158247   14.633   16      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 5, 'min_samples_split': 4}        0.8296858067121526   14.078   17      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 13, 'min_samples_split': 3}       0.8317979710762653   19.518   18      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 11, 'min_samples_split': 6}       0.827274386170492    18.25    19      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 21, 'min_samples_split': 3}    0.8303547862220337   22.429   20      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 15, 'min_samples_split': 3}    0.8363697294745656   22.048   21      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 16, 'min_samples_split': 7}    0.8227403753292174   22.993   22      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 13, 'min_samples_split': 6}       0.8351418271651093   17.502   23      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 10, 'min_samples_split': 2}       0.8292249495394688   15.769   24      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 20, 'min_samples_split': 5}    0.8411410044532194   25.58    25      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 19, 'min_samples_split': 3}    0.8338041860639126   21.014   26      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 29, 'min_samples_split': 2}       0.8221013484287332   28.225   27      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 21, 'min_samples_split': 7}       0.8286730688563122   21.69    28      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 21, 'min_samples_split': 2}    0.8376517597728984   25.465   29      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 5}    0.8267613648787466   21.379   30      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 16, 'min_samples_split': 8}    0.8378037767957747   21.963   31      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 12, 'min_samples_split': 4}    0.8341055809299021   22.311   32      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 23, 'min_samples_split': 7}    0.8325632711164136   22.559   33      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 17, 'min_samples_split': 9}    0.8359108124141309   19.842   34      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 4, 'min_samples_split': 6}     0.833650714187229    15.562   35      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 5, 'min_samples_split': 6}     0.8269793763911965   16.518   36      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 22, 'min_samples_split': 4}    0.826203696033005    24.615   37      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 26, 'min_samples_split': 4}       0.8256713663177898   25.719   38      \n",
      "{'criterion': 'entropy', 'max_depth': 28.0, 'max_features': 15, 'min_samples_split': 2}    0.8373348279853662   20.841   39      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 17, 'min_samples_split': 3}       0.8352639855001773   20.756   40      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 26, 'min_samples_split': 7}    0.8262450735090955   24.209   41      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 22, 'min_samples_split': 9}       0.8223447037776687   23.171   42      \n",
      "{'criterion': 'entropy', 'max_depth': 18.0, 'max_features': 26, 'min_samples_split': 4}    0.8330573117260226   26.9     43      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 5, 'min_samples_split': 9}        0.8267224483547463   14.151   44      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 20, 'min_samples_split': 5}       0.8310907364605014   28.152   45      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 22, 'min_samples_split': 5}       0.8262053912756336   22.675   46      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 11, 'min_samples_split': 4}    0.8371924331341448   19.724   47      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 22, 'min_samples_split': 2}       0.8282200594539677   27.26    48      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 10, 'min_samples_split': 6}    0.8311058103028012   18.823   49      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 1, 'min_samples_split': 5}     0.8268240505977196   14.623   50      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 4, 'min_samples_split': 5}     0.8311645150991157   15.399   51      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 19, 'min_samples_split': 7}    0.8348119469335796   20.102   52      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 16, 'min_samples_split': 9}       0.8192823410531247   19.806   53      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 14, 'min_samples_split': 8}    0.834122383357036    17.543   54      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 1, 'min_samples_split': 5}     0.8217907553009752   13.364   55      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 16, 'min_samples_split': 6}       0.8298804542236533   18.353   56      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 10, 'min_samples_split': 6}       0.8288904647120535   14.997   57      \n",
      "{'criterion': 'entropy', 'max_depth': 27.0, 'max_features': 13, 'min_samples_split': 3}    0.8388303013983308   19.271   58      \n",
      "{'criterion': 'gini', 'max_depth': 28.0, 'max_features': 27, 'min_samples_split': 2}       0.8217666179953043   25.057   59      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 19, 'min_samples_split': 8}    0.8362543423910034   21.237   60      \n",
      "{'criterion': 'gini', 'max_depth': 19.0, 'max_features': 10, 'min_samples_split': 5}       0.8110013361357734   13.993   61      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 10, 'min_samples_split': 3}    0.8309932006942402   16.628   62      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 24, 'min_samples_split': 2}    0.8293261843483514   21.899   63      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 5, 'min_samples_split': 7}        0.8262085018823836   12.568   64      \n",
      "{'criterion': 'gini', 'max_depth': 30.0, 'max_features': 7, 'min_samples_split': 3}        0.8271759636324952   14.4     65      \n",
      "{'criterion': 'entropy', 'max_depth': 29.0, 'max_features': 7, 'min_samples_split': 5}     0.8347431224601489   15.675   66      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 21, 'min_samples_split': 2}       0.8286538140201337   21.689   67      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 16, 'min_samples_split': 6}    0.8287498776191907   21.561   68      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 17, 'min_samples_split': 5}    0.8385685639162324   21.551   69      \n",
      "{'criterion': 'gini', 'max_depth': 32.0, 'max_features': 7, 'min_samples_split': 2}        0.8311587896853675   14.223   70      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 7, 'min_samples_split': 6}     0.8211184268927546   14.834   71      \n",
      "{'criterion': 'entropy', 'max_depth': 25.0, 'max_features': 5, 'min_samples_split': 9}     0.8314286524439974   13.936   72      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 17, 'min_samples_split': 4}    0.8364013697663165   22.837   73      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 15, 'min_samples_split': 2}    0.8385102792083129   21.154   74      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 13, 'min_samples_split': 9}    0.83562847989224     18.956   75      \n",
      "{'criterion': 'entropy', 'max_depth': 21.0, 'max_features': 10, 'min_samples_split': 8}    0.8244454723852099   17.156   76      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 22, 'min_samples_split': 2}    0.8318236933844912   21.955   77      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 21, 'min_samples_split': 3}    0.830694284584746    25.712   78      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 17, 'min_samples_split': 9}    0.8245123383970594   23.962   79      \n",
      "{'criterion': 'entropy', 'max_depth': 31.0, 'max_features': 5, 'min_samples_split': 5}     0.8285767941870726   15.956   80      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 19, 'min_samples_split': 3}       0.8296996787134554   20.246   81      \n",
      "{'criterion': 'entropy', 'max_depth': 17.0, 'max_features': 24, 'min_samples_split': 3}    0.827794537951374    23.021   82      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 20, 'min_samples_split': 4}    0.8296262749936502   21.778   83      \n",
      "{'criterion': 'entropy', 'max_depth': 32.0, 'max_features': 12, 'min_samples_split': 2}    0.837095431789412    19.956   84      \n",
      "{'criterion': 'entropy', 'max_depth': 20.0, 'max_features': 18, 'min_samples_split': 8}    0.8359073522586872   20.334   85      \n",
      "{'criterion': 'entropy', 'max_depth': 23.0, 'max_features': 20, 'min_samples_split': 7}    0.8348039733606515   22.293   86      \n",
      "{'criterion': 'entropy', 'max_depth': 24.0, 'max_features': 19, 'min_samples_split': 7}    0.8344447775256143   21.363   87      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 20, 'min_samples_split': 8}       0.8213717923664405   22.811   88      \n",
      "{'criterion': 'gini', 'max_depth': 20.0, 'max_features': 15, 'min_samples_split': 9}       0.8164977105996312   20.045   89      \n",
      "{'criterion': 'gini', 'max_depth': 29.0, 'max_features': 15, 'min_samples_split': 2}       0.8335143371801785   21.067   90      \n",
      "{'criterion': 'gini', 'max_depth': 24.0, 'max_features': 11, 'min_samples_split': 3}       0.8278172860330545   19.605   91      \n",
      "{'criterion': 'gini', 'max_depth': 27.0, 'max_features': 16, 'min_samples_split': 9}       0.83335073347038     21.272   92      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 10, 'min_samples_split': 4}       0.8340420473370536   14.677   93      \n",
      "{'criterion': 'gini', 'max_depth': 31.0, 'max_features': 7, 'min_samples_split': 2}        0.8288722255360627   16.915   94      \n",
      "{'criterion': 'entropy', 'max_depth': 26.0, 'max_features': 24, 'min_samples_split': 3}    0.8311287890776246   25.391   95      \n",
      "{'criterion': 'entropy', 'max_depth': 22.0, 'max_features': 14, 'min_samples_split': 5}    0.8375227247238051   18.634   96      \n",
      "{'criterion': 'entropy', 'max_depth': 16.0, 'max_features': 24, 'min_samples_split': 6}    0.8264832687274264   24.077   97      \n",
      "{'criterion': 'entropy', 'max_depth': 19.0, 'max_features': 14, 'min_samples_split': 7}    0.830432854409312    19.309   98      \n",
      "{'criterion': 'gini', 'max_depth': 26.0, 'max_features': 8, 'min_samples_split': 2}        0.8300389230950793   14.834   99      \n"
     ]
    }
   ],
   "source": [
    "print ('%-90s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "  \n",
    "\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(25):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-90s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "\n",
    "for i in range(100):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(25):\n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1=sum(f1)/len(f1)   \n",
    "    #if f1>0.76:\n",
    "    print('%-90s %-20s %-8s %-8s' % (a,f1,round(time()-second,3),i))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}