nsl_kdd_binary_calassification_with_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "52GSZ0J_zaYj"
   },
   "source": [
    "# NSL KDD binary classification with Transformer\n",
    "\n",
    "I used it to classify the NSL-KDD dataset by making a slight change on the code I got from the [keras documentation page.](https://keras.io/examples/nlp/text_classification_with_transformer/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "i7MlsDyNzaYr"
   },
   "source": [
    "## importing of required libraries \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y8gX00uEzaYt"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RVj4msuOzaYz"
   },
   "source": [
    "## Implement multi head self attention as a Keras layer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p09RV47lzaY3"
   },
   "outputs": [],
   "source": [
    "\n",
    "class MultiHeadSelfAttention(layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads=8):\n",
    "        super(MultiHeadSelfAttention, self).__init__()\n",
    "        self.embed_dim = embed_dim\n",
    "        self.num_heads = num_heads\n",
    "        if embed_dim % num_heads != 0:\n",
    "            raise ValueError(\n",
    "                f\"embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}\"\n",
    "            )\n",
    "        self.projection_dim = embed_dim // num_heads\n",
    "        self.query_dense = layers.Dense(embed_dim)\n",
    "        self.key_dense = layers.Dense(embed_dim)\n",
    "        self.value_dense = layers.Dense(embed_dim)\n",
    "        self.combine_heads = layers.Dense(embed_dim)\n",
    "\n",
    "    def attention(self, query, key, value):\n",
    "        score = tf.matmul(query, key, transpose_b=True)\n",
    "        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n",
    "        scaled_score = score / tf.math.sqrt(dim_key)\n",
    "        weights = tf.nn.softmax(scaled_score, axis=-1)\n",
    "        output = tf.matmul(weights, value)\n",
    "        return output, weights\n",
    "\n",
    "    def separate_heads(self, x, batch_size):\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "\n",
    "    def call(self, inputs):\n",
    "        # x.shape = [batch_size, seq_len, embedding_dim]\n",
    "        batch_size = tf.shape(inputs)[0]\n",
    "        query = self.query_dense(inputs)  # (batch_size, seq_len, embed_dim)\n",
    "        key = self.key_dense(inputs)  # (batch_size, seq_len, embed_dim)\n",
    "        value = self.value_dense(inputs)  # (batch_size, seq_len, embed_dim)\n",
    "        query = self.separate_heads(\n",
    "            query, batch_size\n",
    "        )  # (batch_size, num_heads, seq_len, projection_dim)\n",
    "        key = self.separate_heads(\n",
    "            key, batch_size\n",
    "        )  # (batch_size, num_heads, seq_len, projection_dim)\n",
    "        value = self.separate_heads(\n",
    "            value, batch_size\n",
    "        )  # (batch_size, num_heads, seq_len, projection_dim)\n",
    "        attention, weights = self.attention(query, key, value)\n",
    "        attention = tf.transpose(\n",
    "            attention, perm=[0, 2, 1, 3]\n",
    "        )  # (batch_size, seq_len, num_heads, projection_dim)\n",
    "        concat_attention = tf.reshape(\n",
    "            attention, (batch_size, -1, self.embed_dim)\n",
    "        )  # (batch_size, seq_len, embed_dim)\n",
    "        output = self.combine_heads(\n",
    "            concat_attention\n",
    "        )  # (batch_size, seq_len, embed_dim)\n",
    "        return output\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GANkWrJ7zaY6"
   },
   "source": [
    "## Implement a Transformer block as a layer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Cj879AGizaY7"
   },
   "outputs": [],
   "source": [
    "\n",
    "class TransformerBlock(layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n",
    "        super(TransformerBlock, self).__init__()\n",
    "        self.att = MultiHeadSelfAttention(embed_dim, num_heads)\n",
    "        self.ffn = keras.Sequential(\n",
    "            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n",
    "        )\n",
    "        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = layers.Dropout(rate)\n",
    "        self.dropout2 = layers.Dropout(rate)\n",
    "\n",
    "    def call(self, inputs, training):\n",
    "        attn_output = self.att(inputs)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(inputs + attn_output)\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        return self.layernorm2(out1 + ffn_output)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fZ5U2V7RzaZC"
   },
   "source": [
    "## Implement embedding layer\n",
    "\n",
    "Two seperate embedding layers, one for tokens, one for token index (positions).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SADnKmhRzaZE"
   },
   "outputs": [],
   "source": [
    "class TokenAndPositionEmbedding(layers.Layer):\n",
    "    def __init__(self, maxlen, vocab_size, embed_dim):\n",
    "        super(TokenAndPositionEmbedding, self).__init__()\n",
    "        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n",
    "        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n",
    "\n",
    "    def call(self, x):\n",
    "        maxlen = tf.shape(x)[-1]\n",
    "        positions = tf.range(start=0, limit=maxlen, delta=1)\n",
    "        positions = self.pos_emb(positions)\n",
    "        x = self.token_emb(x)\n",
    "        return x + positions\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7yVOPPwSzaZO"
   },
   "source": [
    "# prepare NSL KDD dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## reading CSV files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# c_names --->  column names\n",
    "c_names = [\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\n",
    "    \"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "    \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "    \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "    \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "    \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "    \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "    \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "    \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "    \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"labels\",\"difficulty_degree\"]\n",
    "\n",
    "train = pd.read_csv( \"data/KDDTrain+.csv\", names=c_names) # train file\n",
    "test = pd.read_csv(\"data/KDDTest+.csv\", names=c_names) # test file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## deletion of unnecessary feature (difficulty_degree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train[\"difficulty_degree\"] \n",
    "del test[\"difficulty_degree\"] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting object features to categories first and then to dummy tables (except \"labels\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "int64\n",
      "object\n",
      "object\n",
      "object\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "int64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "int64\n",
      "int64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "float64\n",
      "object\n"
     ]
    }
   ],
   "source": [
    "for i in c_names:\n",
    "    print((train[i].dtypes))\n",
    "    if train[i].dtypes==object:\n",
    "        train[i] = train[i].astype('category')\n",
    "        test[i] = test[i].astype('category')\n",
    "        if i==\"labels\":\n",
    "            break\n",
    "        train=pd.get_dummies(train, columns=[i])\n",
    "        test=pd.get_dummies(test, columns=[i])   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## labels feature  converts to binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TRAIN\n",
    "attack_or_not=[]\n",
    "for i in train[\"labels\"]:#it changes the normal label to \"1\" and the attack tag to \"0\" for use in the machine learning algorithm\n",
    "    if i ==\"normal\":\n",
    "        attack_or_not.append(1)\n",
    "    else:\n",
    "        attack_or_not.append(0)           \n",
    "train[\"labels\"]=attack_or_not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST\n",
    "attack_or_not=[]\n",
    "for i in test[\"labels\"]:#it changes the normal label to \"1\" and the attack tag to \"0\" for use in the machine learning algorithm\n",
    "    if i ==\"normal\":\n",
    "        attack_or_not.append(1)\n",
    "    else:\n",
    "        attack_or_not.append(0)           \n",
    "test[\"labels\"]=attack_or_not"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Synchronizing Test and Train datasets.\n",
    "### Add \"0\" for the feature that does not exist in one of these two datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22543\n",
      "service_aol\n",
      "22543\n",
      "service_harvest\n",
      "22543\n",
      "service_http_2784\n",
      "22543\n",
      "service_http_8001\n",
      "22543\n",
      "service_red_i\n",
      "22543\n",
      "service_tftp_u\n",
      "22543\n",
      "service_urh_i\n"
     ]
    }
   ],
   "source": [
    "f=list(train.columns)\n",
    "e=list(test.columns)\n",
    "\n",
    "for i in f:\n",
    "    if i not in e:\n",
    "        zero_data =pd.array(np.zeros(len(test[\"labels\"]))) \n",
    "        print(len(zero_data))\n",
    "        test[i] = zero_data\n",
    "        print(i)\n",
    "for i in e:\n",
    "    if i not in f:\n",
    "        zero_data = np.zeros(len(train[\"labels\"]))\n",
    "        train[i] = zero_data\n",
    "        print(i)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## separation of features (data) and Label (target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train[\"labels\"] #this section separates the label and the data into two separate pieces, as Label=y Data=X \n",
    "del train[\"labels\"] \n",
    "X = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = test[\"labels\"] #this section separates the label and the data into two separate pieces, as Label=y Data=X \n",
    "del test[\"labels\"] \n",
    "x_test=test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalization and Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = preprocessing.scale(X)\n",
    "X = preprocessing.normalize(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = preprocessing.scale(x_test)\n",
    "x_test = preprocessing.normalize(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separating Train data into two parts as train and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100778 Training sequences (100778, 122)\n",
      "25195 Validation sequences (25195, 122)\n",
      "22543 Test sequences (22543, 122)\n"
     ]
    }
   ],
   "source": [
    "x_train, x_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, random_state = 0, stratify=y)\n",
    "print(len(x_train), \"Training sequences\",x_train.shape)\n",
    "print(len(x_val), \"Validation sequences\",x_val.shape)\n",
    "print(len(x_test), \"Test sequences\",x_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZGkJpYWEzaZU"
   },
   "source": [
    "## Create classifier model using transformer layer\n",
    "\n",
    "Transformer layer outputs one vector for each time step of our input sequence.\n",
    "Here, we take the mean across all time steps and\n",
    "use a feed forward network on top of it to classify text.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bum8sPyKzaZV"
   },
   "outputs": [],
   "source": [
    "maxlen=122\n",
    "vocab_size = 100000  # Only consider the top 20k words\n",
    "\n",
    "\n",
    "\n",
    "embed_dim = 32  # Embedding size for each token\n",
    "num_heads = 2  # Number of attention heads\n",
    "ff_dim = 32  # Hidden layer size in feed forward network inside transformer\n",
    "\n",
    "inputs = layers.Input(shape=(maxlen,))\n",
    "embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\n",
    "x = embedding_layer(inputs)\n",
    "transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\n",
    "x = transformer_block(x)\n",
    "x = layers.GlobalAveragePooling1D()(x)\n",
    "x = layers.Dropout(0.1)(x)\n",
    "x = layers.Dense(20, activation=\"relu\")(x)\n",
    "x = layers.Dropout(0.1)(x)\n",
    "outputs = layers.Dense(2, activation=\"softmax\")(x)\n",
    "\n",
    "model = keras.Model(inputs=inputs, outputs=outputs)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "x_val = keras.preprocessing.sequence.pad_sequences(x_val, maxlen=maxlen)\n",
    "x_test = keras.preprocessing.sequence.pad_sequences(x_test , maxlen=maxlen)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "id": "98T8ht1ezaZk",
    "outputId": "030b4f93-ccf8-496a-f416-6848021638e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 100778 samples, validate on 25195 samples\n",
      "Epoch 1/2\n",
      "100778/100778 [==============================] - 240s 2ms/sample - loss: 0.6915 - accuracy: 0.5331 - val_loss: 0.6908 - val_accuracy: 0.5346\n",
      "Epoch 2/2\n",
      "100778/100778 [==============================] - 221s 2ms/sample - loss: 0.6908 - accuracy: 0.5345 - val_loss: 0.6910 - val_accuracy: 0.5346\n"
     ]
    }
   ],
   "source": [
    "model.compile(\"adam\", \"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "history = model.fit(\n",
    "    x_train, y_train, batch_size=32, epochs=2, validation_data=(x_val, y_val)\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.7010403732089466\n",
      "Test accuracy: 0.43073237\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print(\"Test loss:\", score[0])\n",
    "print(\"Test accuracy:\", score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "Zm1_60H5z0fp",
    "outputId": "09bc4104-5d63-4826-c0a6-535cc9f271f5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.690967743196618\n",
      "Test accuracy: 0.5345902\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_val, y_val, verbose=0)\n",
    "print(\"Test loss:\", score[0])\n",
    "print(\"Test accuracy:\", score[1])"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "text_classification_with_transformer",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}