TrafficSignClassifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Traffic Sign Classifier\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load The Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Loaded\n"
     ]
    }
   ],
   "source": [
    "# Load pickled data\n",
    "import pickle\n",
    "\n",
    "training_file = 'train.p'\n",
    "validation_file= 'valid.p'\n",
    "testing_file =  'test.p'\n",
    "\n",
    "with open(training_file, mode='rb') as f:\n",
    "    train = pickle.load(f)\n",
    "with open(validation_file, mode='rb') as f:\n",
    "    valid = pickle.load(f)\n",
    "with open(testing_file, mode='rb') as f:\n",
    "    test = pickle.load(f)\n",
    "    \n",
    "X_train, y_train = train['features'], train['labels']\n",
    "X_valid, y_valid = valid['features'], valid['labels']\n",
    "X_test, y_test = test['features'], test['labels']\n",
    "\n",
    "print('Data Loaded')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary of Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples = 34799\n",
      "Number of testing examples = 12630\n",
      "Number of testing samples = 12630\n",
      "Image data shape = (32, 32)\n",
      "Number of classes = 43\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_train = X_train.shape[0]\n",
    "n_validation = X_valid.shape[0]\n",
    "n_test = X_test.shape[0]\n",
    "image_shape = X_train.shape[1:3]\n",
    "n_classes = len(np.unique(y_test))\n",
    "\n",
    "print(\"Number of training examples =\", n_train)\n",
    "print(\"Number of testing examples =\", n_test)\n",
    "print(\"Number of testing samples =\",n_test)\n",
    "print(\"Image data shape =\", image_shape)\n",
    "print(\"Number of classes =\", n_classes)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Random Images From Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#Plot images from the features set\n",
    "image_num= random.randint(0,n_train)\n",
    "print(X_train.shape)\n",
    "image = X_train[image_num]\n",
    "\n",
    "print(image.shape)\n",
    "plt.imshow(image)\n",
    "\n",
    "#imgplot = plt.imshow(image)\n",
    "\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import imutils\n",
    "from random import randint\n",
    "import cv2\n",
    "def rgb2gray(images):\n",
    "    \n",
    "    r, g, b = images[:,:,:,0], images[:,:,:,1], images[:,:,:,2]\n",
    "    gray = 0.3 * r + 0.11* g + 0.59 * b\n",
    "    gray = image_histogram_equalization(gray)\n",
    "    return gray\n",
    "\n",
    "\"\"\"\n",
    "    Balances misrepresented classes by adding rotated versions of them to compensate\n",
    "\"\"\"\n",
    "def BalanceImageSet(X_set,y_set):\n",
    "    \n",
    "    unique, counts = np.unique(y_set, return_counts=True)\n",
    "    count_dict = dict(zip(unique, counts))\n",
    "    \n",
    "    average_count = sum(count_dict[key] for key in count_dict) / len(count_dict)\n",
    "    max_count = max(counts)\n",
    "    \n",
    "    new_images = []\n",
    "    continue_balance = True\n",
    "    \n",
    "    while(continue_balance):\n",
    "        new_values = []\n",
    "        unique, counts = np.unique(y_set, return_counts=True)\n",
    "        count_dict = dict(zip(unique, counts))\n",
    "        \n",
    "        continue_balance = False;\n",
    "        for i in range(len(X_set)):\n",
    "            if(count_dict[y_set[i]] < max_count/2):\n",
    "                continue_balance = True\n",
    "                \n",
    "                #Rotate Image and Add\n",
    "                rot_angle = randint(-10,10)\n",
    "                rotated_image = imutils.rotate(X_set[i],rot_angle)\n",
    "                new_images.append(rotated_image)\n",
    "                new_values.append(y_set[i])\n",
    "                \n",
    "                #Translate Image and add\n",
    "                translate_x,translate_y = randint(-5,5),randint(-5,5)\n",
    "                translated_image = imutils.translate(X_set[i],translate_x,translate_y)\n",
    "                new_images.append(translated_image)\n",
    "                new_values.append(y_set[i])\n",
    "                \n",
    "        #Updated Y set so that we know how many we need to keep doing.         \n",
    "        y_set = np.concatenate((y_set,np.array(new_values)))\n",
    "                \n",
    "    X_set = np.concatenate((X_set,np.array(new_images)))\n",
    "        \n",
    "    \n",
    "    print(X_set.shape,y_set.shape)\n",
    "   \n",
    "    return X_set,y_set\n",
    "\n",
    "def AugmentData(X_set, y_set):\n",
    "    \n",
    "    new_images = []\n",
    "    new_values = []\n",
    "    \n",
    "    for i in range(len(X_set)):\n",
    "\n",
    "        #Rotate Image and Add\n",
    "        rot_angle = randint(-10,10)\n",
    "        rotated_image = imutils.rotate(X_set[i],rot_angle)\n",
    "        new_images.append(rotated_image)\n",
    "        new_values.append(y_set[i])\n",
    "\n",
    "        #Translate Image and add\n",
    "        translate_x,translate_y = randint(-4,4),randint(-4,4)\n",
    "        translated_image = imutils.translate(X_set[i],translate_x,translate_y)\n",
    "        new_images.append(translated_image)\n",
    "        new_values.append(y_set[i]) \n",
    "\n",
    "        #Change Luminescene and add\n",
    "        new_image = X_set[i] - 10\n",
    "        new_images.append(new_image)\n",
    "        new_values.append(y_set[i])\n",
    "        \n",
    "        #Change Luminescene and add\n",
    "        new_image = X_set[i] + 10\n",
    "        new_images.append(new_image)\n",
    "        new_values.append(y_set[i])\n",
    "\n",
    "            \n",
    "    augmented_y = np.array(new_values)\n",
    "    augmented_X = np.array(new_images)\n",
    "    \n",
    "    return augmented_X, augmented_y\n",
    "\n",
    "def hist_equal(image):\n",
    "    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(4,4))\n",
    "    cl1 = clahe.apply(image)\n",
    "    return cl1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Balance Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(63417, 32, 32, 3) (63417,)\n",
      "Image balance complete\n",
      "Data Augmented\n",
      "(317085, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "# Balance Data Types\n",
    "X_train,y_train = BalanceImageSet(X_train,y_train)\n",
    "print(\"Image balance complete\")\n",
    "\n",
    "X_train_extra,y_train_extra = AugmentData(X_train,y_train)\n",
    "print(\"Data Augmented\")\n",
    "\n",
    "X_train = np.concatenate((X_train,X_train_extra))\n",
    "y_train = np.concatenate((y_train,y_train_extra))\n",
    "\n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Example architecture\n",
    "from tensorflow.contrib.layers import flatten\n",
    "def LeNet(x,keep_prob):    \n",
    "    # Arguments used for tf.truncated_normal, randomly defines variables for the weights and biases for each layer\n",
    "    mu = 0\n",
    "    sigma = 0.01\n",
    "    x_gray = tf.image.rgb_to_grayscale(x)\n",
    "    \n",
    "    # Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x16.\n",
    "    conv1_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 1, 16), mean = mu, stddev = sigma))\n",
    "    conv1_b = tf.Variable(tf.zeros(16))\n",
    "    conv1   = tf.nn.conv2d(x_gray, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b\n",
    "\n",
    "    # Activation.\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "\n",
    "    # Pooling. Input = 28x28x16. Output = 14x14x16.\n",
    "    conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "    \n",
    "    #Dropout\n",
    "    conv1 = tf.nn.dropout(conv1,keep_prob)\n",
    "    \n",
    "    # Layer 2: Convolutional Input = 14x14x16 Output = 10x10x16.\n",
    "    conv2_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 16, 24), mean = mu, stddev = sigma))\n",
    "    conv2_b = tf.Variable(tf.zeros(24))\n",
    "    conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b\n",
    "    \n",
    "    # Activation.\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "\n",
    "    # Pooling. Input = 10x10x24. Output = 5x5x24.\n",
    "    conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "\n",
    "    #Dropout\n",
    "    conv2 = tf.nn.dropout(conv2,keep_prob)\n",
    "    \n",
    "    # Flatten. Input = 5x5x16. Output = 600.\n",
    "    fc0   = flatten(conv2)\n",
    "    \n",
    "    # Layer 3: Fully Connected. Input = 600. Output = 240.\n",
    "    fc1_W = tf.Variable(tf.truncated_normal(shape=(600, 240), mean = mu, stddev = sigma))\n",
    "    fc1_b = tf.Variable(tf.zeros(240))\n",
    "    fc1   = tf.matmul(fc0, fc1_W) + fc1_b\n",
    "    \n",
    "    # Activation.\n",
    "    fc1    = tf.nn.relu(fc1)\n",
    "    \n",
    "    #Dropout\n",
    "    fc1 = tf.nn.dropout(fc1,keep_prob)\n",
    "\n",
    "    # Layer 4: Fully Connected. Input = 240. Output = 120.\n",
    "    fc2_W  = tf.Variable(tf.truncated_normal(shape=(240, 120), mean = mu, stddev = sigma))\n",
    "    fc2_b  = tf.Variable(tf.zeros(120))\n",
    "    fc2    = tf.matmul(fc1, fc2_W) + fc2_b\n",
    "    \n",
    "    # Activation.\n",
    "    fc2    = tf.nn.relu(fc2)\n",
    "    \n",
    "    #Dropout\n",
    "    fc1 = tf.nn.dropout(fc1,keep_prob)\n",
    "\n",
    "    # Layer 5: Fully Connected. Input = 120. Output = 43.\n",
    "    fc3_W  = tf.Variable(tf.truncated_normal(shape=(120, 43), mean = mu, stddev = sigma))\n",
    "    fc3_b  = tf.Variable(tf.zeros(43))\n",
    "    logits = tf.matmul(fc2, fc3_W) + fc3_b\n",
    "    \n",
    "    return logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Built\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "#Training Parameters\n",
    "learning_rate = 0.0002\n",
    "EPOCHS = 30\n",
    "BATCH_SIZE = 128\n",
    "KEEP_PROB = 0.6\n",
    "\n",
    "#Parameters\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "#Model\n",
    "x = tf.placeholder(tf.float32,shape=(None,32,32,3))\n",
    "logits=LeNet(x,keep_prob)\n",
    "\n",
    "y = tf.placeholder(tf.int32,(None))\n",
    "one_hot_y = tf.one_hot(y,43)\n",
    "\n",
    "\n",
    "# Calculate the loss\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=one_hot_y,logits=logits)\n",
    "loss = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "#Set up the optimizer\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "print(\"Model Built\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))\n",
    "prediction = tf.argmax(logits,1)\n",
    "accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "softmax_logits = tf.nn.softmax(logits)\n",
    "top_k = tf.nn.top_k(softmax_logits,k=5)\n",
    "\n",
    "def evaluate(X_data, y_data):\n",
    "    num_examples = len(X_data)\n",
    "    total_accuracy = 0\n",
    "    sess = tf.get_default_session()\n",
    "    for offset in range(0, num_examples, BATCH_SIZE):\n",
    "        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]\n",
    "        \n",
    "        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y,keep_prob: 1.0})\n",
    "        total_accuracy += (accuracy * len(batch_x))\n",
    "    return total_accuracy / num_examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./lenet\n",
      "Training...\n",
      "\n",
      "EPOCH 1 ...\n",
      "Training Accuracy = 0.985\n",
      "Validation Accuracy = 0.975\n",
      "\n",
      "EPOCH 2 ...\n",
      "Training Accuracy = 0.986\n",
      "Validation Accuracy = 0.975\n",
      "\n",
      "EPOCH 3 ...\n",
      "Training Accuracy = 0.985\n",
      "Validation Accuracy = 0.975\n",
      "\n",
      "EPOCH 4 ...\n",
      "Training Accuracy = 0.985\n",
      "Validation Accuracy = 0.973\n",
      "\n",
      "EPOCH 5 ...\n",
      "Training Accuracy = 0.986\n",
      "Validation Accuracy = 0.974\n",
      "\n",
      "EPOCH 6 ...\n",
      "Training Accuracy = 0.986\n",
      "Validation Accuracy = 0.976\n",
      "\n",
      "Model saved\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    #Initiate variables\n",
    "    #init = tf.global_variables_initializer()\n",
    "    #sess.run(init)\n",
    "    saver.restore(sess,'./lenet')\n",
    "    num_examples = len(X_train)\n",
    "    \n",
    "    print(\"Training...\")\n",
    "    print()\n",
    "    for i in range(EPOCHS):\n",
    "        X_train, y_train = shuffle(X_train, y_train)\n",
    "        for offset in range(0, num_examples, BATCH_SIZE):\n",
    "            end = offset + BATCH_SIZE\n",
    "            batch_x = X_train[offset:end]\n",
    "            batch_y = y_train[offset:end]\n",
    "\n",
    "            sess.run(training_op, feed_dict={x: batch_x, y: batch_y,keep_prob:KEEP_PROB})\n",
    "            \n",
    "        train_accuracy = evaluate(X_train,y_train)\n",
    "        validation_accuracy = evaluate(X_valid, y_valid)\n",
    "        print(\"EPOCH {} ...\".format(i+1))\n",
    "        print(\"Training Accuracy = {:.3f}\".format(train_accuracy))\n",
    "        print(\"Validation Accuracy = {:.3f}\".format(validation_accuracy))\n",
    "        print()\n",
    "        \n",
    "    saver.save(sess, './lenet')\n",
    "    print(\"Model saved\")\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the Model on the Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./lenet\n",
      "Test Accuracy = 0.960\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, './lenet')\n",
    "\n",
    "    test_accuracy = evaluate(X_test, y_test)\n",
    "    print(\"Test Accuracy = {:.3f}\".format(test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Test Model on Images from the Internet\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Output the Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 32, 32, 3)\n",
      "(1, 32, 32, 3)\n",
      "(1, 32, 32, 3)\n",
      "(1, 32, 32, 3)\n",
      "(1, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.image as mpimg\n",
    "from PIL import Image\n",
    "from scipy import ndimage\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "size = [32,32]\n",
    "images = []\n",
    "y_internet = np.array([23,14,1,13,25])\n",
    "\n",
    "for i in range(1,6):\n",
    "    img_loc = 'germansigns/sign' + str(i) + '.jpg'\n",
    "    img = Image.open(img_loc)\n",
    "    img = img.resize(size,Image.ANTIALIAS)\n",
    "    img = np.array(img)\n",
    "    img = np.resize(img,(1,32,32,3))\n",
    "    \n",
    "    images.append(img)\n",
    "    print(img.shape)\n",
    "\n",
    "X_internet = np.vstack(images)\n",
    "\n",
    "#Plot image\n",
    "image = X_internet[4]\n",
    "imgplot = plt.imshow(image)\n",
    "\n",
    "#Convert to grayscale\n",
    "#X_internet = np.resize(rgb2gray(X_internet),(X_internet.shape[0],32,32,1))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict the Sign Type for Each Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./lenet\n",
      "Prediction is [23 14 23 13 22]\n",
      "Accuracy is 0.600000023842\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, './lenet')\n",
    "    \n",
    "    prediction = sess.run(prediction, feed_dict={x: X_internet, y: y_internet,keep_prob: 1.0})\n",
    "    print(\"Prediction is \" + str(prediction))\n",
    "    \n",
    "    validation_accuracy = evaluate(X_internet, y_internet)\n",
    "    print(\"Accuracy is \" + str(validation_accuracy))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output Top 5 Softmax Probabilities For Each Image Found on the Web"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./lenet\n",
      "Top Softmax are: \n",
      "[[ 0.99849  0.00074  0.00052  0.00021  0.00002]\n",
      " [ 0.99999  0.       0.       0.       0.     ]\n",
      " [ 0.20639  0.13894  0.11292  0.10266  0.04195]\n",
      " [ 0.99121  0.00194  0.00074  0.00061  0.00053]\n",
      " [ 0.59453  0.12667  0.08019  0.04463  0.04068]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, './lenet')\n",
    "    softmax = sess.run(top_k,feed_dict={x: X_internet, y: y_internet,keep_prob: 1.0})\n",
    "    print(\"Top Softmax are: \")\n",
    "    np.set_printoptions(precision=5,suppress=True)\n",
    "    print(softmax[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}