feature_engineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import pearsonr\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.feature_selection import chi2, mutual_info_classif, RFE, SelectFromModel, SelectKBest, VarianceThreshold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.options.mode.use_inf_as_na = True\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('G:\\IDS2018\\datasets\\Thursday-15-02-2018_TrafficForML_CICFlowMeter.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_fea = list(data.select_dtypes(exclude=['object']).columns)\n",
    "category_fea = list(filter(lambda x: x not in numerical_fea, list(data.columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_numerical_serial_fea(data, feas):\n",
    "    numerical_serial_fea = []\n",
    "    numerical_noserial_fea = []\n",
    "    for fea in feas:\n",
    "        temp = data[fea].nunique()\n",
    "        if temp <= 10:\n",
    "            numerical_noserial_fea.append(fea)\n",
    "        else:\n",
    "            numerical_serial_fea.append(fea)\n",
    "    return numerical_serial_fea, numerical_noserial_fea\n",
    "numerical_serial_fea, numerical_noserial_fea = get_numerical_serial_fea(data, numerical_fea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Dst Port': 0,\n",
       " 'Protocol': 0,\n",
       " 'Timestamp': 0,\n",
       " 'Flow Duration': 0,\n",
       " 'Tot Fwd Pkts': 0,\n",
       " 'Tot Bwd Pkts': 0,\n",
       " 'TotLen Fwd Pkts': 0,\n",
       " 'TotLen Bwd Pkts': 0,\n",
       " 'Fwd Pkt Len Max': 0,\n",
       " 'Fwd Pkt Len Min': 0,\n",
       " 'Fwd Pkt Len Mean': 0,\n",
       " 'Fwd Pkt Len Std': 0,\n",
       " 'Bwd Pkt Len Max': 0,\n",
       " 'Bwd Pkt Len Min': 0,\n",
       " 'Bwd Pkt Len Mean': 0,\n",
       " 'Bwd Pkt Len Std': 0,\n",
       " 'Flow Byts/s': 8027,\n",
       " 'Flow Pkts/s': 8027,\n",
       " 'Flow IAT Mean': 0,\n",
       " 'Flow IAT Std': 0,\n",
       " 'Flow IAT Max': 0,\n",
       " 'Flow IAT Min': 0,\n",
       " 'Fwd IAT Tot': 0,\n",
       " 'Fwd IAT Mean': 0,\n",
       " 'Fwd IAT Std': 0,\n",
       " 'Fwd IAT Max': 0,\n",
       " 'Fwd IAT Min': 0,\n",
       " 'Bwd IAT Tot': 0,\n",
       " 'Bwd IAT Mean': 0,\n",
       " 'Bwd IAT Std': 0,\n",
       " 'Bwd IAT Max': 0,\n",
       " 'Bwd IAT Min': 0,\n",
       " 'Fwd PSH Flags': 0,\n",
       " 'Bwd PSH Flags': 0,\n",
       " 'Fwd URG Flags': 0,\n",
       " 'Bwd URG Flags': 0,\n",
       " 'Fwd Header Len': 0,\n",
       " 'Bwd Header Len': 0,\n",
       " 'Fwd Pkts/s': 0,\n",
       " 'Bwd Pkts/s': 0,\n",
       " 'Pkt Len Min': 0,\n",
       " 'Pkt Len Max': 0,\n",
       " 'Pkt Len Mean': 0,\n",
       " 'Pkt Len Std': 0,\n",
       " 'Pkt Len Var': 0,\n",
       " 'FIN Flag Cnt': 0,\n",
       " 'SYN Flag Cnt': 0,\n",
       " 'RST Flag Cnt': 0,\n",
       " 'PSH Flag Cnt': 0,\n",
       " 'ACK Flag Cnt': 0,\n",
       " 'URG Flag Cnt': 0,\n",
       " 'CWE Flag Count': 0,\n",
       " 'ECE Flag Cnt': 0,\n",
       " 'Down/Up Ratio': 0,\n",
       " 'Pkt Size Avg': 0,\n",
       " 'Fwd Seg Size Avg': 0,\n",
       " 'Bwd Seg Size Avg': 0,\n",
       " 'Fwd Byts/b Avg': 0,\n",
       " 'Fwd Pkts/b Avg': 0,\n",
       " 'Fwd Blk Rate Avg': 0,\n",
       " 'Bwd Byts/b Avg': 0,\n",
       " 'Bwd Pkts/b Avg': 0,\n",
       " 'Bwd Blk Rate Avg': 0,\n",
       " 'Subflow Fwd Pkts': 0,\n",
       " 'Subflow Fwd Byts': 0,\n",
       " 'Subflow Bwd Pkts': 0,\n",
       " 'Subflow Bwd Byts': 0,\n",
       " 'Init Fwd Win Byts': 0,\n",
       " 'Init Bwd Win Byts': 0,\n",
       " 'Fwd Act Data Pkts': 0,\n",
       " 'Fwd Seg Size Min': 0,\n",
       " 'Active Mean': 0,\n",
       " 'Active Std': 0,\n",
       " 'Active Max': 0,\n",
       " 'Active Min': 0,\n",
       " 'Idle Mean': 0,\n",
       " 'Idle Std': 0,\n",
       " 'Idle Max': 0,\n",
       " 'Idle Min': 0,\n",
       " 'Label': 0}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[numerical_serial_fea] = data[numerical_serial_fea].fillna(data[numerical_serial_fea].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Dst Port': 0,\n",
       " 'Protocol': 0,\n",
       " 'Timestamp': 0,\n",
       " 'Flow Duration': 0,\n",
       " 'Tot Fwd Pkts': 0,\n",
       " 'Tot Bwd Pkts': 0,\n",
       " 'TotLen Fwd Pkts': 0,\n",
       " 'TotLen Bwd Pkts': 0,\n",
       " 'Fwd Pkt Len Max': 0,\n",
       " 'Fwd Pkt Len Min': 0,\n",
       " 'Fwd Pkt Len Mean': 0,\n",
       " 'Fwd Pkt Len Std': 0,\n",
       " 'Bwd Pkt Len Max': 0,\n",
       " 'Bwd Pkt Len Min': 0,\n",
       " 'Bwd Pkt Len Mean': 0,\n",
       " 'Bwd Pkt Len Std': 0,\n",
       " 'Flow Byts/s': 0,\n",
       " 'Flow Pkts/s': 0,\n",
       " 'Flow IAT Mean': 0,\n",
       " 'Flow IAT Std': 0,\n",
       " 'Flow IAT Max': 0,\n",
       " 'Flow IAT Min': 0,\n",
       " 'Fwd IAT Tot': 0,\n",
       " 'Fwd IAT Mean': 0,\n",
       " 'Fwd IAT Std': 0,\n",
       " 'Fwd IAT Max': 0,\n",
       " 'Fwd IAT Min': 0,\n",
       " 'Bwd IAT Tot': 0,\n",
       " 'Bwd IAT Mean': 0,\n",
       " 'Bwd IAT Std': 0,\n",
       " 'Bwd IAT Max': 0,\n",
       " 'Bwd IAT Min': 0,\n",
       " 'Fwd PSH Flags': 0,\n",
       " 'Bwd PSH Flags': 0,\n",
       " 'Fwd URG Flags': 0,\n",
       " 'Bwd URG Flags': 0,\n",
       " 'Fwd Header Len': 0,\n",
       " 'Bwd Header Len': 0,\n",
       " 'Fwd Pkts/s': 0,\n",
       " 'Bwd Pkts/s': 0,\n",
       " 'Pkt Len Min': 0,\n",
       " 'Pkt Len Max': 0,\n",
       " 'Pkt Len Mean': 0,\n",
       " 'Pkt Len Std': 0,\n",
       " 'Pkt Len Var': 0,\n",
       " 'FIN Flag Cnt': 0,\n",
       " 'SYN Flag Cnt': 0,\n",
       " 'RST Flag Cnt': 0,\n",
       " 'PSH Flag Cnt': 0,\n",
       " 'ACK Flag Cnt': 0,\n",
       " 'URG Flag Cnt': 0,\n",
       " 'CWE Flag Count': 0,\n",
       " 'ECE Flag Cnt': 0,\n",
       " 'Down/Up Ratio': 0,\n",
       " 'Pkt Size Avg': 0,\n",
       " 'Fwd Seg Size Avg': 0,\n",
       " 'Bwd Seg Size Avg': 0,\n",
       " 'Fwd Byts/b Avg': 0,\n",
       " 'Fwd Pkts/b Avg': 0,\n",
       " 'Fwd Blk Rate Avg': 0,\n",
       " 'Bwd Byts/b Avg': 0,\n",
       " 'Bwd Pkts/b Avg': 0,\n",
       " 'Bwd Blk Rate Avg': 0,\n",
       " 'Subflow Fwd Pkts': 0,\n",
       " 'Subflow Fwd Byts': 0,\n",
       " 'Subflow Bwd Pkts': 0,\n",
       " 'Subflow Bwd Byts': 0,\n",
       " 'Init Fwd Win Byts': 0,\n",
       " 'Init Bwd Win Byts': 0,\n",
       " 'Fwd Act Data Pkts': 0,\n",
       " 'Fwd Seg Size Min': 0,\n",
       " 'Active Mean': 0,\n",
       " 'Active Std': 0,\n",
       " 'Active Max': 0,\n",
       " 'Active Min': 0,\n",
       " 'Idle Mean': 0,\n",
       " 'Idle Std': 0,\n",
       " 'Idle Max': 0,\n",
       " 'Idle Min': 0,\n",
       " 'Label': 0}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Timestamp'] = pd.to_datetime(data['Timestamp'],format='%d/%m/%Y %H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Label'].replace(to_replace='Benign', value=0, inplace=True)\n",
    "data['Label'].replace(to_replace='DoS attacks-GoldenEye', value=1, inplace=True)\n",
    "data['Label'].replace(to_replace='DoS attacks-Slowloris', value=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 异常值处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_outliers_by_3segama(data, fea):\n",
    "    data_std = np.std(data[fea])\n",
    "    data_mean = np.mean(data[fea])\n",
    "    outliers_cut_off = 3 * data_std\n",
    "    lower_rule = data_mean - outliers_cut_off\n",
    "    upper_rule = data_mean + outliers_cut_off\n",
    "    data[fea+'_outliers'] = data[fea].apply(lambda x: str('异常值') if x > upper_rule or x < lower_rule else '正常值')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正常值    1039789\n",
      "异常值       8786\n",
      "Name: Dst Port_outliers, dtype: int64\n",
      "Dst Port_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    991764\n",
      "异常值     56811\n",
      "Name: Flow Duration_outliers, dtype: int64\n",
      "Flow Duration_outliers\n",
      "异常值      280\n",
      "正常值    52218\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046745\n",
      "异常值       1830\n",
      "Name: Tot Fwd Pkts_outliers, dtype: int64\n",
      "Tot Fwd Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046952\n",
      "异常值       1623\n",
      "Name: Tot Bwd Pkts_outliers, dtype: int64\n",
      "Tot Bwd Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1048549\n",
      "异常值         26\n",
      "Name: TotLen Fwd Pkts_outliers, dtype: int64\n",
      "TotLen Fwd Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1047070\n",
      "异常值       1505\n",
      "Name: TotLen Bwd Pkts_outliers, dtype: int64\n",
      "TotLen Bwd Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1038021\n",
      "异常值      10554\n",
      "Name: Fwd Pkt Len Max_outliers, dtype: int64\n",
      "Fwd Pkt Len Max_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1045594\n",
      "异常值       2981\n",
      "Name: Fwd Pkt Len Min_outliers, dtype: int64\n",
      "Fwd Pkt Len Min_outliers\n",
      "异常值       98\n",
      "正常值    52400\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1039579\n",
      "异常值       8996\n",
      "Name: Fwd Pkt Len Mean_outliers, dtype: int64\n",
      "Fwd Pkt Len Mean_outliers\n",
      "异常值     1178\n",
      "正常值    51320\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1037266\n",
      "异常值      11309\n",
      "Name: Fwd Pkt Len Std_outliers, dtype: int64\n",
      "Fwd Pkt Len Std_outliers\n",
      "异常值      634\n",
      "正常值    51864\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1048568\n",
      "异常值          7\n",
      "Name: Bwd Pkt Len Max_outliers, dtype: int64\n",
      "Bwd Pkt Len Max_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1033705\n",
      "异常值      14870\n",
      "Name: Bwd Pkt Len Min_outliers, dtype: int64\n",
      "Bwd Pkt Len Min_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1022827\n",
      "异常值      25748\n",
      "Name: Bwd Pkt Len Mean_outliers, dtype: int64\n",
      "Bwd Pkt Len Mean_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1047631\n",
      "异常值        944\n",
      "Name: Bwd Pkt Len Std_outliers, dtype: int64\n",
      "Bwd Pkt Len Std_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1043268\n",
      "异常值       5307\n",
      "Name: Flow Byts/s_outliers, dtype: int64\n",
      "Flow Byts/s_outliers\n",
      "异常值        4\n",
      "正常值    52494\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1031972\n",
      "异常值      16603\n",
      "Name: Flow Pkts/s_outliers, dtype: int64\n",
      "Flow Pkts/s_outliers\n",
      "异常值      458\n",
      "正常值    52040\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1004984\n",
      "异常值      43591\n",
      "Name: Flow IAT Mean_outliers, dtype: int64\n",
      "Flow IAT Mean_outliers\n",
      "异常值     3115\n",
      "正常值    49383\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1033696\n",
      "异常值      14879\n",
      "Name: Flow IAT Std_outliers, dtype: int64\n",
      "Flow IAT Std_outliers\n",
      "异常值     3698\n",
      "正常值    48800\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1021705\n",
      "异常值      26870\n",
      "Name: Flow IAT Max_outliers, dtype: int64\n",
      "Flow IAT Max_outliers\n",
      "异常值     3778\n",
      "正常值    48720\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1006778\n",
      "异常值      41797\n",
      "Name: Flow IAT Min_outliers, dtype: int64\n",
      "Flow IAT Min_outliers\n",
      "异常值     1862\n",
      "正常值    50636\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    991938\n",
      "异常值     56637\n",
      "Name: Fwd IAT Tot_outliers, dtype: int64\n",
      "Fwd IAT Tot_outliers\n",
      "异常值      281\n",
      "正常值    52217\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1002872\n",
      "异常值      45703\n",
      "Name: Fwd IAT Mean_outliers, dtype: int64\n",
      "Fwd IAT Mean_outliers\n",
      "异常值     4781\n",
      "正常值    47717\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1010889\n",
      "异常值      37686\n",
      "Name: Fwd IAT Std_outliers, dtype: int64\n",
      "Fwd IAT Std_outliers\n",
      "异常值      239\n",
      "正常值    52259\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1004804\n",
      "异常值      43771\n",
      "Name: Fwd IAT Max_outliers, dtype: int64\n",
      "Fwd IAT Max_outliers\n",
      "异常值     3778\n",
      "正常值    48720\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1002826\n",
      "异常值      45749\n",
      "Name: Fwd IAT Min_outliers, dtype: int64\n",
      "Fwd IAT Min_outliers\n",
      "异常值     4902\n",
      "正常值    47596\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    979526\n",
      "异常值     69049\n",
      "Name: Bwd IAT Tot_outliers, dtype: int64\n",
      "Bwd IAT Tot_outliers\n",
      "异常值     7142\n",
      "正常值    45356\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1035909\n",
      "异常值      12666\n",
      "Name: Bwd IAT Mean_outliers, dtype: int64\n",
      "Bwd IAT Mean_outliers\n",
      "异常值     7353\n",
      "正常值    45145\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1019297\n",
      "异常值      29278\n",
      "Name: Bwd IAT Std_outliers, dtype: int64\n",
      "Bwd IAT Std_outliers\n",
      "异常值     5001\n",
      "正常值    47497\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1007151\n",
      "异常值      41424\n",
      "Name: Bwd IAT Max_outliers, dtype: int64\n",
      "Bwd IAT Max_outliers\n",
      "异常值     7196\n",
      "正常值    45302\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1037488\n",
      "异常值      11087\n",
      "Name: Bwd IAT Min_outliers, dtype: int64\n",
      "Bwd IAT Min_outliers\n",
      "异常值     7150\n",
      "正常值    45348\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046973\n",
      "异常值       1602\n",
      "Name: Fwd Header Len_outliers, dtype: int64\n",
      "Fwd Header Len_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046933\n",
      "异常值       1642\n",
      "Name: Bwd Header Len_outliers, dtype: int64\n",
      "Bwd Header Len_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1032449\n",
      "异常值      16126\n",
      "Name: Fwd Pkts/s_outliers, dtype: int64\n",
      "Fwd Pkts/s_outliers\n",
      "异常值       55\n",
      "正常值    52443\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046807\n",
      "异常值       1768\n",
      "Name: Bwd Pkts/s_outliers, dtype: int64\n",
      "Bwd Pkts/s_outliers\n",
      "异常值      754\n",
      "正常值    51744\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1044348\n",
      "异常值       4227\n",
      "Name: Pkt Len Min_outliers, dtype: int64\n",
      "Pkt Len Min_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1048557\n",
      "异常值         18\n",
      "Name: Pkt Len Max_outliers, dtype: int64\n",
      "Pkt Len Max_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1029538\n",
      "异常值      19037\n",
      "Name: Pkt Len Mean_outliers, dtype: int64\n",
      "Pkt Len Mean_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1029158\n",
      "异常值      19417\n",
      "Name: Pkt Len Std_outliers, dtype: int64\n",
      "Pkt Len Std_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1048554\n",
      "异常值         21\n",
      "Name: Pkt Len Var_outliers, dtype: int64\n",
      "Pkt Len Var_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1044334\n",
      "异常值       4241\n",
      "Name: Down/Up Ratio_outliers, dtype: int64\n",
      "Down/Up Ratio_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1028615\n",
      "异常值      19960\n",
      "Name: Pkt Size Avg_outliers, dtype: int64\n",
      "Pkt Size Avg_outliers\n",
      "异常值        2\n",
      "正常值    52496\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1039579\n",
      "异常值       8996\n",
      "Name: Fwd Seg Size Avg_outliers, dtype: int64\n",
      "Fwd Seg Size Avg_outliers\n",
      "异常值     1178\n",
      "正常值    51320\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1022827\n",
      "异常值      25748\n",
      "Name: Bwd Seg Size Avg_outliers, dtype: int64\n",
      "Bwd Seg Size Avg_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046745\n",
      "异常值       1830\n",
      "Name: Subflow Fwd Pkts_outliers, dtype: int64\n",
      "Subflow Fwd Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1048549\n",
      "异常值         26\n",
      "Name: Subflow Fwd Byts_outliers, dtype: int64\n",
      "Subflow Fwd Byts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1046952\n",
      "异常值       1623\n",
      "Name: Subflow Bwd Pkts_outliers, dtype: int64\n",
      "Subflow Bwd Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1047070\n",
      "异常值       1505\n",
      "Name: Subflow Bwd Byts_outliers, dtype: int64\n",
      "Subflow Bwd Byts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1025928\n",
      "异常值      22647\n",
      "Name: Init Fwd Win Byts_outliers, dtype: int64\n",
      "Init Fwd Win Byts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    966856\n",
      "异常值     81719\n",
      "Name: Init Bwd Win Byts_outliers, dtype: int64\n",
      "Init Bwd Win Byts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1044073\n",
      "异常值       4502\n",
      "Name: Fwd Act Data Pkts_outliers, dtype: int64\n",
      "Fwd Act Data Pkts_outliers\n",
      "异常值        0\n",
      "正常值    52498\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1033732\n",
      "异常值      14843\n",
      "Name: Fwd Seg Size Min_outliers, dtype: int64\n",
      "Fwd Seg Size Min_outliers\n",
      "异常值     2232\n",
      "正常值    50266\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1034611\n",
      "异常值      13964\n",
      "Name: Active Mean_outliers, dtype: int64\n",
      "Active Mean_outliers\n",
      "异常值     5598\n",
      "正常值    46900\n",
      "Name: Label, dtype: int64\n",
      "**********\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正常值    1034351\n",
      "异常值      14224\n",
      "Name: Active Std_outliers, dtype: int64\n",
      "Active Std_outliers\n",
      "异常值     4935\n",
      "正常值    47563\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1032734\n",
      "异常值      15841\n",
      "Name: Active Max_outliers, dtype: int64\n",
      "Active Max_outliers\n",
      "异常值     5557\n",
      "正常值    46941\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1035242\n",
      "异常值      13333\n",
      "Name: Active Min_outliers, dtype: int64\n",
      "Active Min_outliers\n",
      "异常值     5463\n",
      "正常值    47035\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    996157\n",
      "异常值     52418\n",
      "Name: Idle Mean_outliers, dtype: int64\n",
      "Idle Mean_outliers\n",
      "异常值     4229\n",
      "正常值    48269\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    1037445\n",
      "异常值      11130\n",
      "Name: Idle Std_outliers, dtype: int64\n",
      "Idle Std_outliers\n",
      "异常值     4788\n",
      "正常值    47710\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    988121\n",
      "异常值     60454\n",
      "Name: Idle Max_outliers, dtype: int64\n",
      "Idle Max_outliers\n",
      "异常值     8686\n",
      "正常值    43812\n",
      "Name: Label, dtype: int64\n",
      "**********\n",
      "正常值    996490\n",
      "异常值     52085\n",
      "Name: Idle Min_outliers, dtype: int64\n",
      "Idle Min_outliers\n",
      "异常值     4277\n",
      "正常值    48221\n",
      "Name: Label, dtype: int64\n",
      "**********\n"
     ]
    }
   ],
   "source": [
    "for fea in numerical_serial_fea:\n",
    "    data = find_outliers_by_3segama(data, fea)\n",
    "    print(data[fea+'_outliers'].value_counts())\n",
    "    print(data.groupby(fea+'_outliers')['Label'].sum())\n",
    "    print('*'*10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected: ['Dst Port' 'Flow Duration' 'Tot Fwd Pkts' 'Tot Bwd Pkts'\n",
      " 'TotLen Fwd Pkts' 'TotLen Bwd Pkts' 'Fwd Pkt Len Max' 'Fwd Pkt Len Min'\n",
      " 'Fwd Pkt Len Mean' 'Fwd Pkt Len Std' 'Bwd Pkt Len Max' 'Bwd Pkt Len Min'\n",
      " 'Bwd Pkt Len Mean' 'Bwd Pkt Len Std' 'Flow Byts/s' 'Flow Pkts/s'\n",
      " 'Flow IAT Mean' 'Flow IAT Std' 'Flow IAT Max' 'Flow IAT Min'\n",
      " 'Fwd IAT Tot' 'Fwd IAT Mean' 'Fwd IAT Std' 'Fwd IAT Max' 'Fwd IAT Min'\n",
      " 'Bwd IAT Tot' 'Bwd IAT Mean' 'Bwd IAT Std' 'Bwd IAT Max' 'Bwd IAT Min'\n",
      " 'Fwd Header Len' 'Bwd Header Len' 'Fwd Pkts/s' 'Bwd Pkts/s' 'Pkt Len Min'\n",
      " 'Pkt Len Max' 'Pkt Len Mean' 'Pkt Len Std' 'Pkt Len Var' 'Pkt Size Avg'\n",
      " 'Fwd Seg Size Avg' 'Bwd Seg Size Avg' 'Subflow Fwd Pkts'\n",
      " 'Subflow Fwd Byts' 'Subflow Bwd Pkts' 'Subflow Bwd Byts'\n",
      " 'Init Fwd Win Byts' 'Init Bwd Win Byts' 'Fwd Act Data Pkts'\n",
      " 'Fwd Seg Size Min' 'Active Mean' 'Active Std' 'Active Max' 'Active Min'\n",
      " 'Idle Mean' 'Idle Std' 'Idle Max' 'Idle Min']\n",
      "Deleted:  ['Down/Up Ratio']\n"
     ]
    }
   ],
   "source": [
    "selector = VarianceThreshold(threshold=3)\n",
    "selector = selector.fit(data[numerical_serial_fea])\n",
    "features_mask = selector.get_support(indices=True)\n",
    "selected_features = np.array(numerical_serial_fea)[features_mask]\n",
    "print('Selected:', selected_features)\n",
    "print('Deleted: ', [fea for fea in numerical_serial_fea if fea not in selected_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Fwd Seg Size Min', (0.4827216248545034, 0.0)),\n",
       " ('Bwd IAT Mean', (0.30192251129739955, 0.0)),\n",
       " ('Init Fwd Win Byts', (0.2684559690822708, 0.0)),\n",
       " ('Bwd IAT Min', (0.2500177147030524, 0.0)),\n",
       " ('Flow IAT Std', (0.21858340497356774, 0.0)),\n",
       " ('Bwd IAT Max', (0.19606408350148533, 0.0)),\n",
       " ('Bwd IAT Std', (0.18719441138249784, 0.0)),\n",
       " ('Idle Max', (0.18100280839192964, 0.0)),\n",
       " ('Fwd Pkt Len Std', (0.16886516326958462, 0.0)),\n",
       " ('Idle Std', (0.15481787043716266, 0.0))]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pearsonr_result = []\n",
    "for fea in numerical_serial_fea:\n",
    "    pearsonr_result.append((fea, pearsonr(data[fea], data['Label'])))\n",
    "sorted(pearsonr_result, key=lambda x: abs(x[1][0]), reverse=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dst Port\n",
      "(array([3.99751145e+08]), array([0.]))\n",
      "Flow Duration\n",
      "(array([3.59172003e+11]), array([0.]))\n",
      "Tot Fwd Pkts\n",
      "(array([3778.26711379]), array([0.]))\n",
      "Tot Bwd Pkts\n",
      "(array([147047.64832782]), array([0.]))\n",
      "TotLen Fwd Pkts\n",
      "(array([634718.39574114]), array([0.]))\n",
      "TotLen Bwd Pkts\n",
      "(array([2.24665013e+08]), array([0.]))\n",
      "Fwd Pkt Len Max\n",
      "(array([3126399.65719183]), array([0.]))\n",
      "Fwd Pkt Len Min\n",
      "(array([672290.30465466]), array([0.]))\n",
      "Fwd Pkt Len Mean\n",
      "(array([1008178.14621648]), array([0.]))\n",
      "Fwd Pkt Len Std\n",
      "(array([4910010.25013736]), array([0.]))\n",
      "Bwd Pkt Len Max\n",
      "(array([2426276.39917122]), array([0.]))\n",
      "Bwd Pkt Len Min\n",
      "(array([1859688.16916162]), array([0.]))\n",
      "Bwd Pkt Len Mean\n",
      "(array([14682.06813255]), array([0.]))\n",
      "Bwd Pkt Len Std\n",
      "(array([6324182.09067248]), array([0.]))\n",
      "Flow Byts/s\n",
      "(array([1.5768512e+10]), array([0.]))\n",
      "Flow Pkts/s\n",
      "(array([1.00582909e+09]), array([0.]))\n",
      "Flow IAT Mean\n",
      "(array([2.96727475e+11]), array([0.]))\n",
      "Flow IAT Std\n",
      "(array([9.98169939e+11]), array([0.]))\n",
      "Flow IAT Max\n",
      "(array([1.02174605e+12]), array([0.]))\n",
      "Flow IAT Min\n",
      "(array([2.28890909e+10]), array([0.]))\n",
      "Fwd IAT Tot\n",
      "(array([2.86848641e+11]), array([0.]))\n",
      "Fwd IAT Mean\n",
      "(array([9.49019226e+11]), array([0.]))\n",
      "Fwd IAT Std\n",
      "(array([4.93833027e+10]), array([0.]))\n",
      "Fwd IAT Max\n",
      "(array([9.7532279e+11]), array([0.]))\n",
      "Fwd IAT Min\n",
      "(array([1.06993311e+12]), array([0.]))\n",
      "Bwd IAT Tot\n",
      "(array([3.67104733e+11]), array([0.]))\n",
      "Bwd IAT Mean\n",
      "(array([3.57822439e+12]), array([0.]))\n",
      "Bwd IAT Std\n",
      "(array([5.31186971e+11]), array([0.]))\n",
      "Bwd IAT Max\n",
      "(array([1.79577127e+12]), array([0.]))\n",
      "Bwd IAT Min\n",
      "(array([4.16347887e+12]), array([0.]))\n",
      "Fwd Header Len\n",
      "(array([1134722.69775687]), array([0.]))\n",
      "Bwd Header Len\n",
      "(array([1334952.53878541]), array([0.]))\n",
      "Fwd Pkts/s\n",
      "(array([1.3340166e+09]), array([0.]))\n",
      "Bwd Pkts/s\n",
      "(array([1.27108414e+08]), array([0.]))\n",
      "Pkt Len Min\n",
      "(array([763088.03979813]), array([0.]))\n",
      "Pkt Len Max\n",
      "(array([3235866.47097282]), array([0.]))\n",
      "Pkt Len Mean\n",
      "(array([79343.84160227]), array([0.]))\n",
      "Pkt Len Std\n",
      "(array([2061704.30901782]), array([0.]))\n",
      "Pkt Len Var\n",
      "(array([2.48367375e+08]), array([0.]))\n",
      "Down/Up Ratio\n",
      "(array([3734.48793754]), array([0.]))\n",
      "Pkt Size Avg\n",
      "(array([40190.97203183]), array([0.]))\n",
      "Fwd Seg Size Avg\n",
      "(array([1008178.14621648]), array([0.]))\n",
      "Bwd Seg Size Avg\n",
      "(array([14682.06813255]), array([0.]))\n",
      "Subflow Fwd Pkts\n",
      "(array([3778.26711379]), array([0.]))\n",
      "Subflow Fwd Byts\n",
      "(array([634718.39574114]), array([0.]))\n",
      "Subflow Bwd Pkts\n",
      "(array([147047.64832782]), array([0.]))\n",
      "Subflow Bwd Byts\n",
      "(array([2.24665013e+08]), array([0.]))\n",
      "Fwd Act Data Pkts\n",
      "(array([870.91484755]), array([2.06394786e-191]))\n",
      "Fwd Seg Size Min\n",
      "(array([823283.21674079]), array([0.]))\n",
      "Active Mean\n",
      "(array([1.42494306e+11]), array([0.]))\n",
      "Active Std\n",
      "(array([3.00722196e+10]), array([0.]))\n",
      "Active Max\n",
      "(array([1.15913032e+11]), array([0.]))\n",
      "Active Min\n",
      "(array([1.15899376e+11]), array([0.]))\n",
      "Idle Mean\n",
      "(array([1.11834743e+12]), array([0.]))\n",
      "Idle Std\n",
      "(array([5.2196894e+11]), array([0.]))\n",
      "Idle Max\n",
      "(array([1.64438117e+12]), array([0.]))\n",
      "Idle Min\n",
      "(array([8.75959539e+11]), array([0.]))\n"
     ]
    }
   ],
   "source": [
    "# find out the features can not be applied by chi2\n",
    "for fea in [fea for fea in numerical_serial_fea if fea not in ['Init Fwd Win Byts', 'Init Bwd Win Byts']]:\n",
    "    print(fea)\n",
    "    print(chi2(np.array(data[fea]).reshape(-1, 1), np.array(data['Label']).reshape(-1, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected: ['Bwd IAT Mean' 'Bwd IAT Max' 'Bwd IAT Min' 'Idle Mean' 'Idle Max']\n"
     ]
    }
   ],
   "source": [
    "chi2_test_fea = [fea for fea in numerical_serial_fea if fea not in ['Init Fwd Win Byts', 'Init Bwd Win Byts']]\n",
    "selector = SelectKBest(chi2, k=5)\n",
    "selector = selector.fit(data[chi2_test_fea], data['Label'])\n",
    "features_mask = selector.get_support(indices=True)\n",
    "selected_features = np.array(chi2_test_fea)[features_mask]\n",
    "print('Selected:', selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected: ['Flow IAT Max' 'Fwd Header Len' 'Pkt Len Max' 'Init Fwd Win Byts'\n",
      " 'Fwd Seg Size Min']\n"
     ]
    }
   ],
   "source": [
    "selector = SelectKBest(mutual_info_classif, k=5)\n",
    "selector = selector.fit(data[numerical_serial_fea], data['Label'])\n",
    "features_mask = selector.get_support(indices=True)\n",
    "selected_features = np.array(numerical_serial_fea)[features_mask]\n",
    "print('Selected:', selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected: ['Dst Port' 'Fwd Pkts/s' 'Down/Up Ratio' 'Init Fwd Win Byts'\n",
      " 'Fwd Seg Size Min']\n"
     ]
    }
   ],
   "source": [
    "features = [fea for fea in data.columns if fea not in ['Timestamp', 'Label']]\n",
    "selector = RFE(DecisionTreeClassifier(), n_features_to_select=5, step=1)\n",
    "selector = selector.fit(data[features], data['Label'])\n",
    "features_mask = selector.get_support(indices=True)\n",
    "selected_features = np.array(features)[features_mask]\n",
    "print('Selected:', selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected: ['Dst Port' 'Flow Duration' 'Flow Byts/s' 'Flow Pkts/s' 'Flow IAT Max'\n",
      " 'Fwd IAT Tot' 'Fwd IAT Mean' 'Fwd IAT Std' 'Fwd IAT Max' 'Fwd IAT Min'\n",
      " 'Bwd IAT Tot' 'Bwd IAT Mean' 'Bwd IAT Std' 'Bwd IAT Max' 'Bwd IAT Min'\n",
      " 'Fwd Pkts/s' 'Pkt Len Var' 'Idle Mean' 'Idle Std' 'Idle Max' 'Idle Min']\n"
     ]
    }
   ],
   "source": [
    "selector = SelectFromModel(LogisticRegression(penalty='l2', C=10))\n",
    "selector = selector.fit(data[features], data['Label'])\n",
    "features_mask = selector.get_support(indices=True)\n",
    "selected_features = np.array(features)[features_mask]\n",
    "print('Selected:', selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected: ['Dst Port' 'Fwd Seg Size Min']\n"
     ]
    }
   ],
   "source": [
    "selector = SelectFromModel(DecisionTreeClassifier())\n",
    "selector = selector.fit(data[features], data['Label'])\n",
    "features_mask = selector.get_support(indices=True)\n",
    "selected_features = np.array(features)[features_mask]\n",
    "print('Selected:', selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}