model_building.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from keras.models import Sequential\n",
    "from keras.models import Model as KerasModel\n",
    "from keras.layers import Input, Dense, Activation, Reshape, Merge, Conv1D, MaxPooling1D, Flatten\n",
    "from keras.layers import Concatenate\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total samples:401399\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Event Name</th>\n",
       "      <th>Bus no.</th>\n",
       "      <th>Route Name</th>\n",
       "      <th>Stop Name</th>\n",
       "      <th>weekday</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>83</td>\n",
       "      <td>176</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>86</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>83</td>\n",
       "      <td>176</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>86</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>83</td>\n",
       "      <td>176</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Event Name  Bus no.  Route Name  Stop Name  weekday  hour\n",
       "0           2       31          83        176        1     3\n",
       "1           2       34          86        105        1     3\n",
       "2           2       31          83        176        1     3\n",
       "3           2       34          86        105        1     3\n",
       "4           2       31          83        176        1     3"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ahmd_traffic = pd.read_csv('data/ahmd_traffic_clean_df.csv')\n",
    "print(\"Total samples:\" + str(len(ahmd_traffic)))\n",
    "ahmd_traffic.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ahmd_traffic = ahmd_traffic.sample(frac=1).reset_index(drop=True) #shuffle\n",
    "Y_labels = ahmd_traffic['Event Name']\n",
    "ahmd_traffic = ahmd_traffic.drop(\"Event Name\", 1)\n",
    "all_cols = ahmd_traffic.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Displaying number of unique vals in each categorical column:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bus no. 182\n",
      "Route Name 119\n",
      "Stop Name 182\n",
      "weekday 7\n",
      "hour 24\n"
     ]
    }
   ],
   "source": [
    "for cat_var in all_cols:\n",
    "    print (cat_var, ahmd_traffic[cat_var].nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training and validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ahmd_traffic_train = ahmd_traffic[:int(0.8*len(ahmd_traffic))]\n",
    "ahmd_traffic_val = ahmd_traffic[int(0.8*len(ahmd_traffic)):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_train  = Y_labels[:int(0.8*len(ahmd_traffic))]\n",
    "Y_val  = Y_labels[int(0.8*len(ahmd_traffic)): ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preproc(X_train ) : \n",
    "\n",
    "    input_list_train = []\n",
    "    \n",
    "    \n",
    "    #the cols to be embedded: rescaling to range [0, # values)\n",
    "    for c in all_cols :\n",
    "        \n",
    "        \"\"\"\n",
    "        vals = np.asarray(X_train[c].tolist() )\n",
    "        input_list_train.append( np.asarray( vals ))\n",
    "        this fails as keras Expect 0,1,2,3.. as cat and not 1,2,3,5 if there are 4 categories.\n",
    "        Using below method instead from https://stackoverflow.com/a/45988584 \n",
    "        \n",
    "        \"\"\"\n",
    "        vals = np.asarray(X_train[c].tolist())\n",
    "        vals = pd.factorize( vals )[0]\n",
    "        input_list_train.append( np.asarray(vals)  )\n",
    "        \"\"\"\n",
    "        This below was the original method used in the code by INSERT NAME HERE. But I found the above implemntation much simpler to understand.\n",
    "        raw_vals = np.unique(X_train[c])\n",
    "        val_map = {}\n",
    "        for i in range(len(raw_vals)):\n",
    "            val_map[raw_vals[i]] = i       \n",
    "        input_list_train.append(X_train[c].map(val_map).values)\n",
    "        \"\"\"\n",
    "    return input_list_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ahmd_traffic_train_modified = preproc( ahmd_traffic_train )\n",
    "ahmd_traffic_val = preproc( ahmd_traffic_val )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### As expected, the proceesed training DataFrame has 5 elements becasue we had 5 categorical columns:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ahmd_traffic_train_modified)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The lenght of each list in the processed dataframe is same as the number of training rows we have:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "321119\n",
      "321119\n",
      "321119\n"
     ]
    }
   ],
   "source": [
    "print (len(ahmd_traffic_train_modified[0]))\n",
    "print (len(ahmd_traffic_train_modified[1]))\n",
    "print (len(ahmd_traffic_train_modified[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  The below code adds a embedding network for each of the catgeoriacal variable. The embedding size is set as according to the rule (as mentioned in the paper): \n",
    "\n",
    "Embedding size  = min( no-of-unique-cat/2 , 50 )\n",
    "\n",
    "##### Each  model is appending to a list named models defined intialised in below cell. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------\n",
      "for categoical column  Bus no.\n",
      "number of unique cat 182\n",
      "embedding_size set as  50\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 1, 50)             9150      \n",
      "_________________________________________________________________\n",
      "reshape_1 (Reshape)          (None, 50)                0         \n",
      "=================================================================\n",
      "Total params: 9,150\n",
      "Trainable params: 9,150\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "------------------------------------------------------------------\n",
      "for categoical column  Route Name\n",
      "number of unique cat 119\n",
      "embedding_size set as  50\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 1, 50)             6000      \n",
      "_________________________________________________________________\n",
      "reshape_2 (Reshape)          (None, 50)                0         \n",
      "=================================================================\n",
      "Total params: 6,000\n",
      "Trainable params: 6,000\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "------------------------------------------------------------------\n",
      "for categoical column  Stop Name\n",
      "number of unique cat 182\n",
      "embedding_size set as  50\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 1, 50)             9150      \n",
      "_________________________________________________________________\n",
      "reshape_3 (Reshape)          (None, 50)                0         \n",
      "=================================================================\n",
      "Total params: 9,150\n",
      "Trainable params: 9,150\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "------------------------------------------------------------------\n",
      "for categoical column  weekday\n",
      "number of unique cat 7\n",
      "embedding_size set as  4\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_4 (Embedding)      (None, 1, 4)              32        \n",
      "_________________________________________________________________\n",
      "reshape_4 (Reshape)          (None, 4)                 0         \n",
      "=================================================================\n",
      "Total params: 32\n",
      "Trainable params: 32\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "------------------------------------------------------------------\n",
      "for categoical column  hour\n",
      "number of unique cat 24\n",
      "embedding_size set as  12\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_5 (Embedding)      (None, 1, 12)             300       \n",
      "_________________________________________________________________\n",
      "reshape_5 (Reshape)          (None, 12)                0         \n",
      "=================================================================\n",
      "Total params: 300\n",
      "Trainable params: 300\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "for categoical_var in all_cols :\n",
    "    print (\"------------------------------------------------------------------\")\n",
    "    print (\"for categoical column \", categoical_var     )\n",
    "    model = Sequential()\n",
    "    no_of_unique_cat  = ahmd_traffic[categoical_var].nunique()\n",
    "    print (\"number of unique cat\",no_of_unique_cat)\n",
    "    embedding_size = min(np.ceil((no_of_unique_cat)/2), 50 )\n",
    "    embedding_size = int(embedding_size)\n",
    "    print (\"embedding_size set as \", embedding_size)\n",
    "    model.add(  Embedding( no_of_unique_cat+1, embedding_size, input_length = 1 ) )\n",
    "    \n",
    "    model.add(Reshape(target_shape=( [embedding_size] )))\n",
    "\n",
    "    \n",
    "    print (model.summary() )\n",
    "    \n",
    "    models.append( model )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ajinkya/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "full_model = Sequential()\n",
    "full_model.add(Merge(models, mode='concat'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "merge_1 (Merge)              (None, 166)               0         \n",
      "=================================================================\n",
      "Total params: 24,632\n",
      "Trainable params: 24,632\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "full_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now are input layers and embedding layers are done. We can build on those as any other keras sequential model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ajinkya/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:1210: calling reduce_prod (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "WARNING:tensorflow:From /home/ajinkya/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:2745: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "WARNING:tensorflow:From /home/ajinkya/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:1299: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "full_model.add(Reshape((166,1)))\n",
    "full_model.add(Conv1D(128, 5))\n",
    "full_model.add(Activation('relu'))\n",
    "full_model.add(MaxPooling1D(5))\n",
    "full_model.add(Conv1D(128, 5))\n",
    "full_model.add(Activation('relu'))\n",
    "full_model.add(MaxPooling1D(5))\n",
    "full_model.add(Conv1D(128, 5))\n",
    "full_model.add(Activation('relu'))\n",
    "full_model.add(MaxPooling1D(1))\n",
    "full_model.add(Flatten())\n",
    "\n",
    "full_model.add(Dense(Y_labels.nunique()))\n",
    "full_model.add(Activation('softmax'))\n",
    "full_model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "merge_1 (Merge)              (None, 166)               0         \n",
      "_________________________________________________________________\n",
      "reshape_6 (Reshape)          (None, 166, 1)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 162, 128)          768       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 162, 128)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 32, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 28, 128)           82048     \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 28, 128)           0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 5, 128)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 1, 128)            82048     \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 1, 128)            0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 1, 128)            0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 5)                 645       \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 5)                 0         \n",
      "=================================================================\n",
      "Total params: 190,141\n",
      "Trainable params: 190,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "full_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from  keras.utils.np_utils  import to_categorical\n",
    "Y_train_cat = to_categorical(Y_train.tolist() )\n",
    "Y_val_cat = to_categorical(Y_val.tolist()  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 321119 samples, validate on 80280 samples\n",
      "Epoch 1/1\n",
      "321119/321119 [==============================] - 526s - loss: 0.4651 - acc: 0.8338 - val_loss: 2.4058 - val_acc: 0.4170\n"
     ]
    }
   ],
   "source": [
    "history = full_model.fit(x = ahmd_traffic_train_modified, y = Y_train_cat, batch_size = 64,\n",
    "                 epochs = 1, validation_data = (ahmd_traffic_val, Y_val_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Bus no.', 'Route Name', 'Stop Name', 'weekday', 'hour'], dtype='object')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving trained embeddings weights for clustering and visualizing later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bus_embedding = models[0].layers[0].get_weights()[0]\n",
    "route_embedding = models[1].layers[0].get_weights()[0]\n",
    "stop_embedding = models[2].layers[0].get_weights()[0]\n",
    "weekday_embedding = models[3].layers[0].get_weights()[0]\n",
    "hour_embedding = models[4].layers[0].get_weights()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bus_embedding.dump(open('data/embeddings/bus_embedding.npy', 'wb'))\n",
    "route_embedding.dump(open('data/embeddings/route_embedding.npy', 'wb'))\n",
    "stop_embedding.dump(open('data/embeddings/stop_embedding.npy', 'wb'))\n",
    "weekday_embedding.dump(open('data/embeddings/weekday_embedding.npy', 'wb'))\n",
    "hour_embedding.dump(open('data/embeddings/hour_embedding.npy', 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}