CNN on Foosball sounds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import IPython.display\n",
    "from IPython.display import Audio\n",
    "import pandas\n",
    "from pandas import DataFrame\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "ds = pandas.read_pickle(\"full_dataset_44100.pickle\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert target column to 1-hot encoding\n",
    "Our neural net will output a 2d vector. If the first element is 1 then its a noise, otherwise its a goal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def to1hot(row):\n",
    "    one_hot = np.zeros(2)\n",
    "    one_hot[row]=1.0\n",
    "    return one_hot\n",
    "\n",
    "ds[\"one_hot_encoding\"] = ds.target.apply(to1hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Establishing the baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is the error rate if we always guess the majority: 0.49\n"
     ]
    }
   ],
   "source": [
    "print \"This is the error rate if we always guess the majority: %.2f\" % \\\n",
    "(min(ds[ds[\"target\"] == 0].index.size, ds[ds[\"target\"] == 1].index.size) / (float)(ds.index.size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Form the training, testing, and validation data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds[\"mels_flatten\"] = ds.mels.apply(lambda mels: mels.flatten())\n",
    "train_data = ds[0:160]\n",
    "validation_data = ds[160:190]\n",
    "test_data = ds[189:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_x = np.vstack(train_data.mels_flatten).reshape(train_data.shape[0],128, 87,1).astype(np.float32)\n",
    "train_y = np.vstack(train_data[\"one_hot_encoding\"])\n",
    "train_size = train_y.shape[0]\n",
    "validation_x = np.vstack(validation_data.mels_flatten).reshape(validation_data.shape[0],128, 87,1).astype(np.float32)\n",
    "validation_y = np.vstack(validation_data[\"one_hot_encoding\"])\n",
    "test_x = np.vstack(test_data.mels_flatten).reshape(test_data.shape[0],128, 87,1).astype(np.float32)\n",
    "test_y = np.vstack(test_data[\"one_hot_encoding\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "BATCH_SIZE = 160 # we have so little data, just set the batch size to the entire training set\n",
    "NUM_CHANNELS = 1 \n",
    "NUM_LABELS = 2\n",
    "INPUT_SHAPE = (128,87)\n",
    "SEED = 42\n",
    "\n",
    "# This node is where we feed a batch of the training data and labels at each training step\n",
    "train_data_node = tf.placeholder(tf.float32,shape=(BATCH_SIZE, INPUT_SHAPE[0], INPUT_SHAPE[1], 1))\n",
    "train_labels_node = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_LABELS))\n",
    "\n",
    "# constants for validation and tests\n",
    "validation_data_node = tf.constant(validation_x)\n",
    "test_data_node = tf.constant(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "conv1_weights = tf.Variable(\n",
    "    # The first 3 elements defines the shape of the filter, the last one is the number of feature maps it outputs\n",
    "    # This 1d filter only looks at a small contiguous chunk of audio signal (550 samples, ~550ms) (\n",
    "    # if the data was an image then one would probably use a 2d (greyscale) or even 3d (color) filter\n",
    "    # The size of the filter can be anything, as long as it is smaller than the input\n",
    "    tf.truncated_normal([2, 8, 1, 32], # Creating 32 feature maps.\n",
    "    stddev=0.1, \n",
    "    seed=SEED))\n",
    "conv1_biases = tf.Variable(tf.zeros([32])) #Each feature needs a bias for ReLU\n",
    "\n",
    "conv2_weights = tf.Variable(\n",
    "    tf.truncated_normal([30, 8, 32, 64], # Creating 64 feature maps. \n",
    "    stddev=0.1, \n",
    "    seed=SEED))\n",
    "conv2_biases = tf.Variable(tf.constant(0.1, shape=[64]))\n",
    "\n",
    "# 56 is the ceiling of 11025/50/2/2. \n",
    "# See comments below for explaination on the effect of stride size on the size of hidden layers\n",
    "fc1_weights = tf.Variable(\n",
    "    tf.truncated_normal([48 * 64, 512], stddev=0.1, seed=SEED))\n",
    "fc1_biases = tf.Variable(tf.constant(0.1, shape=[512]))\n",
    "\n",
    "fc2_weights = tf.Variable(\n",
    "    tf.truncated_normal([512, NUM_LABELS], stddev=0.1, seed=SEED))\n",
    "fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]))\n",
    "print 'done'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#wire the variables together\n",
    "\n",
    "def model(data, train=False):\n",
    "    \"\"\"The Model definition.\"\"\"\n",
    "    # 2D convolution, with 'SAME' padding (i.e. the output feature map has\n",
    "    # the same size as the input). Note that {strides} is a 4D array whose\n",
    "    # shape matches the data layout: [image index, y, x, depth].\n",
    "    \n",
    "    print data.get_shape()\n",
    "    conv = tf.nn.conv2d(data,\n",
    "                      conv1_weights,\n",
    "                      strides=[1, 2, 2, 1], # Since stride is 50, the filters moves 50 frames each time. Therefore the shape becomes ceiling(11025/50)\n",
    "                      padding='SAME')\n",
    "    print conv.get_shape()\n",
    "    \n",
    "    # Bias and rectified linear non-linearity.\n",
    "    relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))\n",
    "\n",
    "    print relu.get_shape()\n",
    "    # Max pooling. The kernel size spec ksize also follows the layout of\n",
    "    # the data.\n",
    "    pool = tf.nn.max_pool(relu,\n",
    "                        ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1],\n",
    "                        padding='SAME')\n",
    "\n",
    "    print \"pool_shape: %s\" % pool.get_shape()\n",
    "    \n",
    "    conv = tf.nn.conv2d(pool,\n",
    "                      conv2_weights,\n",
    "                      strides=[1, 2, 2, 1],\n",
    "                      padding='SAME')\n",
    "\n",
    "    print \"conv: %s\" % conv.get_shape()\n",
    "    \n",
    "    relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))\n",
    "    pool = tf.nn.max_pool(relu,\n",
    "                        ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], # max pool aggregates 2 units into 1, therefore the shape is halved again.\n",
    "                        padding='SAME')\n",
    "\n",
    "    print \"pool: %s\" % pool.get_shape()\n",
    "    \n",
    "    # Reshape the feature map cuboid into a 2D matrix to feed it to the\n",
    "    # fully connected layers.\n",
    "\n",
    "    pool_shape = pool.get_shape().as_list()\n",
    "    \n",
    "    reshape = tf.reshape(\n",
    "      pool,\n",
    "      [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])\n",
    "\n",
    "    # Fully connected layer. Note that the '+' operation automatically\n",
    "    # broadcasts the biases.\n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)\n",
    "\n",
    "    # Add a 50% dropout during training only. Dropout also scales\n",
    "    # activations such that no rescaling is needed at evaluation time.\n",
    "    if train:\n",
    "        hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)\n",
    "    return tf.matmul(hidden, fc2_weights) + fc2_biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(160, 128, 87, 1)\n",
      "(160, 64, 44, 32)\n",
      "(160, 64, 44, 32)\n",
      "pool_shape: (160, 32, 22, 32)\n",
      "conv: (160, 16, 11, 64)\n",
      "pool: (160, 8, 6, 64)\n",
      "(30, 128, 87, 1)\n",
      "(30, 64, 44, 32)\n",
      "(30, 64, 44, 32)\n",
      "pool_shape: (30, 32, 22, 32)\n",
      "conv: (30, 16, 11, 64)\n",
      "pool: (30, 8, 6, 64)\n",
      "(20, 128, 87, 1)\n",
      "(20, 64, 44, 32)\n",
      "(20, 64, 44, 32)\n",
      "pool_shape: (20, 32, 22, 32)\n",
      "conv: (20, 16, 11, 64)\n",
      "pool: (20, 8, 6, 64)\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Training computation: logits + cross-entropy loss.\n",
    "logits = model(train_data_node, True)\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
    "  logits, train_labels_node))\n",
    "\n",
    "# L2 regularization for the fully connected parameters.\n",
    "regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +\n",
    "                tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))\n",
    "# Add the regularization term to the loss.\n",
    "loss += 5e-4 * regularizers\n",
    "\n",
    "# Optimizer: set up a variable that's incremented once per batch and\n",
    "# controls the learning rate decay.\n",
    "batch = tf.Variable(0)\n",
    "# Decay once per epoch, using an exponential schedule starting at 0.01.\n",
    "learning_rate = tf.train.exponential_decay(\n",
    "  0.01,                # Base learning rate.\n",
    "  batch * BATCH_SIZE,  # Current index into the dataset.\n",
    "  train_size,          # Decay step.\n",
    "  0.99,                # Decay rate.\n",
    "  staircase=True)\n",
    "# Use simple momentum for the optimization.\n",
    "optimizer = tf.train.MomentumOptimizer(learning_rate,\n",
    "                                       0.9).minimize(loss,\n",
    "                                                     global_step=batch)\n",
    "\n",
    "# Predictions for the minibatch, validation set and test set.\n",
    "train_prediction = tf.nn.softmax(logits)\n",
    "# We'll compute them only once in a while by calling their {eval()} method.\n",
    "validation_prediction = tf.nn.softmax(model(validation_data_node))\n",
    "test_prediction = tf.nn.softmax(model(test_data_node))\n",
    "\n",
    "print 'Done'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create a new interactive session that we'll use in\n",
    "# subsequent code cells.\n",
    "s = tf.InteractiveSession()\n",
    "\n",
    "# Use our newly created session as the default for \n",
    "# subsequent operations.\n",
    "s.as_default()\n",
    "\n",
    "# Initialize all the variables we defined above.\n",
    "tf.initialize_all_variables().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "def error_rate(predictions, labels):\n",
    "    # We use argmax to convert prediction probabilities into 1-hot encoding and compare it against the labels\n",
    "    correct = np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
    "    total = predictions.shape[0]\n",
    "\n",
    "    error = 100.0 - (100 * float(correct) / float(total))\n",
    "\n",
    "    confusions = np.zeros([NUM_LABELS, NUM_LABELS], np.float32)\n",
    "\n",
    "    bundled = zip(np.argmax(predictions, 1), np.argmax(labels, 1))\n",
    "    for predicted, actual in bundled:\n",
    "        confusions[predicted, actual] += 1\n",
    "    return error, confusions\n",
    "\n",
    "print 'Done'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mini-batch loss: 4.12113 Error: 56.87500 Learning rate: 0.01000\n",
      "Validation error: 60.00000\n"
     ]
    }
   ],
   "source": [
    "# I modified the original code to use the entire training set instead of mini batches\n",
    "for i in range(200):\n",
    "    # Train over the first 1/4th of our training set.\n",
    "#     steps = int(train_size / BATCH_SIZE)\n",
    "    #for step in xrange(steps):\n",
    "    # Compute the offset of the current minibatch in the data.\n",
    "    # Note that we could use better randomization across epochs.\n",
    "    offset = 0 #(step * BATCH_SIZE) % (train_size - BATCH_SIZE)\n",
    "    batch_data = train_x[offset:(offset + BATCH_SIZE), :, :, :]\n",
    "    batch_labels = train_y[offset:(offset + BATCH_SIZE)]\n",
    "    # This dictionary maps the batch data (as a numpy array) to the\n",
    "    # node in the graph it should be fed to.\n",
    "    feed_dict = {train_data_node: batch_data,\n",
    "               train_labels_node: batch_labels}\n",
    "    # Run the graph and fetch some of the nodes.\n",
    "    _, l, lr, predictions = s.run(\n",
    "    [optimizer, loss, learning_rate, train_prediction],\n",
    "    feed_dict=feed_dict)\n",
    "\n",
    "      # Print out the loss periodically.\n",
    "    if i % 20 == 0:\n",
    "        error, _ = error_rate(predictions, batch_labels)\n",
    "        print 'Mini-batch loss: %.5f Error: %.5f Learning rate: %.5f' % (l, error, lr)\n",
    "        print 'Validation error: %.5f' % error_rate(\n",
    "            validation_prediction.eval(), validation_y)[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_error, confusions = error_rate(test_prediction.eval(), test_y)\n",
    "print 'Test error: %.5f' % test_error\n",
    "\n",
    "plt.xlabel('Actual')\n",
    "plt.ylabel('Predicted')\n",
    "plt.grid(False)\n",
    "plt.xticks(np.arange(1))\n",
    "plt.yticks(np.arange(1))\n",
    "plt.imshow(confusions, cmap=plt.cm.jet, interpolation='nearest');\n",
    "\n",
    "for i, cas in enumerate(confusions):\n",
    "  for j, count in enumerate(cas):\n",
    "    if count > 0:\n",
    "      xoff = .07 * len(str(count))\n",
    "      plt.text(j-xoff, i+.2, int(count), fontsize=9, color='white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def show_sample(i):\n",
    "    print i, test_data.iloc[i][\"target\"]\n",
    "    IPython.display.display(IPython.display.Audio(data=test_data.iloc[i][\"data\"], rate=44100))\n",
    "res = np.argmax(test_prediction.eval(),1) == np.argmax(test_y, 1)\n",
    "right = []\n",
    "wrong = []\n",
    "for i, v in enumerate(res.tolist()):\n",
    "    if v:\n",
    "        right.append(i)\n",
    "    else:\n",
    "        wrong.append(i)\n",
    "\n",
    "#These are the samples our model got incorrect\n",
    "for w in wrong:\n",
    "    show_sample(w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# saver = tf.train.Saver()\n",
    "# save_path = saver.save(s, \"model.ckpt\")\n",
    "# print(\"Model saved in file: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}