model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import librosa\n",
    "from sequential_model.layers import *\n",
    "from sequential_model.convnet import * \n",
    "from sequential_model.solver import *\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Skip: these sections are for testing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Naive forward pass \n",
    "# Dimension checked\n",
    "# JODO: check correctedness\n",
    "x = np.random.randn(10, 1, 128, 128)\n",
    "w = np.random.randn(5, 1, 4, 4)\n",
    "b = np.random.randn(5,)\n",
    "conv_param = {'stride': 2, 'pad': 1}\n",
    "\n",
    "out, cache = conv_forward_naive(x,w,b,conv_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 5, 64, 64)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Naive backward pass\n",
    "# Dimension checked\n",
    "# JODO: check correcteness\n",
    "\n",
    "#from sequential_model.fast_layers import *\n",
    "#from sequential_model.gradient_check import eval_numerical_gradient_array, eval_numerical_gradient\n",
    "\n",
    "x = np.random.randn(10, 1, 128, 128)\n",
    "w = np.random.randn(5,1, 4,4)\n",
    "b = np.random.randn(5,)\n",
    "dout = np.random.randn(10, 5, 64, 64)\n",
    "conv_param = {'stride': 2, 'pad': 1}\n",
    "\n",
    "# dx_num = eval_numerical_gradient_array(lambda x: conv_forward_naive(x, w, b, conv_param)[0], x, dout)\n",
    "# dw_num = eval_numerical_gradient_array(lambda w: conv_forward_naive(x, w, b, conv_param)[0], w, dout)\n",
    "# db_num = eval_numerical_gradient_array(lambda b: conv_forward_naive(x, w, b, conv_param)[0], b, dout)\n",
    "\n",
    "out, cache = conv_forward_naive(x, w, b, conv_param)\n",
    "dx, dw, db = conv_backward_naive(dout, cache)\n",
    "\n",
    "# Your errors should be around 1e-9'\n",
    "# print('Testing conv_backward_naive function')\n",
    "# print('dx error: ', rel_error(dx, dx_num))\n",
    "# print('dw error: ', rel_error(dw, dw_num))\n",
    "# print('db error: ', rel_error(db, db_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 1, 128, 128)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Sandwich layer forward + backward\n",
    "# Dimension checked\n",
    "from sequential_model.layers import *\n",
    "from sequential_model.layer_utils import *\n",
    "import numpy as np\n",
    "## Conv_relu_forward/backward\n",
    "from sequential_model.layer_utils import conv_relu_pool_forward, conv_relu_pool_backward\n",
    "\n",
    "x = np.random.randn(10, 1, 128, 128)\n",
    "w = np.random.randn(5,1,4,4)\n",
    "b = np.random.randn(5,)\n",
    "dout = np.random.randn(10, 5, 64, 64)\n",
    "conv_param = {'stride': 2, 'pad': 1}\n",
    "pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2}\n",
    "\n",
    "out, cache = conv_relu_pool_forward(x, w, b, conv_param, pool_param)\n",
    "dx, dw, db = conv_relu_pool_backward(dout, cache)\n",
    "\n",
    "out, cache = conv_relu_forward(x, w, b, conv_param)\n",
    "dx, dw, db = conv_relu_backward(dout, cache)\n",
    "\n",
    "# dx_num = eval_numerical_gradient_array(lambda x: conv_relu_pool_forward(x, w, b, conv_param, pool_param)[0], x, dout)\n",
    "# dw_num = eval_numerical_gradient_array(lambda w: conv_relu_pool_forward(x, w, b, conv_param, pool_param)[0], w, dout)\n",
    "# db_num = eval_numerical_gradient_array(lambda b: conv_relu_pool_forward(x, w, b, conv_param, pool_param)[0], b, dout)\n",
    "\n",
    "# print 'Testing conv_relu_pool'\n",
    "# print 'dx error: ', rel_error(dx_num, dx)\n",
    "# print 'dw error: ', rel_error(dw_num, dw)\n",
    "# print 'db error: ', rel_error(db_num, db)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decisions for frame duration "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spec_per_sec= spectrogram.shape[1]/librosa.get_duration(y=y,sr=sr)\n",
    "print('Number of Spectrogram columns per sec:', spec_per_sec)\n",
    "duration_per_specrogram_dim = len(y)/(spectrogram.shape[1]*sr) ## inverse of the above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame_dim = 128 #fixed\n",
    "print(\"Note: using %d spectrogram dim for each input, which is %.2f seconds per input\" % (frame_dim, duration_per_specrogram_dim*frame_dim)) \n",
    "seq_len = 7 #fixed\n",
    "print(\"Note: using %d inputs for each sequence, that is %.2f seconds per sequence\" % (seq_len, duration_per_specrogram_dim*frame_dim*seq_len)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting into audio data with right dimension \n",
    "formatting audio file step by step to:\n",
    "-> spectrogram of dim (128, _) \n",
    "-> around 20 sec sequences of dim (7, 128, 128)\n",
    "-> padding of each song of irregular length to (k, 7, 128, 128) where k depends on song duration\n",
    "-> reformatting for cnn (N, 1, 128, 128) where N = 7*k, and 1 is the channel for raw cnn input\n",
    "TODO Note: N should first be merged to other songs and then build batch from there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def song_to_input_matrix(song_filename, frame_dim = 128, seq_len = 7):\n",
    "    y, sr = librosa.load(filename)\n",
    "    spectrogram = librosa.feature.melspectrogram(y=y, sr=sr)\n",
    "    song_db = librosa.power_to_db(spectrogram, ref=np.max) ## now shape is (128, _)\n",
    "    max_seq_dim = frame_dim * seq_len\n",
    "    num_seq = int(song_db.shape[1]/max_seq_dim) + 1\n",
    "    input_matrix = np.zeros((num_seq, seq_len,128, frame_dim)) #128 is frequency dim which is also fixed\n",
    "    left_over_frame_dim = song_db.shape[1]\n",
    "    for i in range(num_seq*seq_len):\n",
    "        seq_index = int(i / 7)\n",
    "        seq_pos = i % 7\n",
    "        if left_over_frame_dim < frame_dim:\n",
    "            left_over_frame = song_db[:, i*frame_dim:]\n",
    "            padding = np.zeros((128,frame_dim - left_over_frame_dim))\n",
    "            last_piece = np.concatenate((left_over_frame,padding), axis=1)\n",
    "            input_matrix[seq_index,seq_pos,:,:] = last_piece\n",
    "            left_over_frame_dim = max(0, left_over_frame_dim - frame_dim) ## Note: continue to pad till the end of the sequence\n",
    "        else:\n",
    "            input_matrix[seq_index,seq_pos,:,:] = song_db[:,i*frame_dim:(i+1)*frame_dim]\n",
    "            left_over_frame_dim -= frame_dim\n",
    "    \n",
    "    cnn_input_matrix = np.expand_dims(input_matrix, axis=2).reshape((-1,1,128,128))\n",
    "    return cnn_input_matrix\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70, 1, 128, 128)\n"
     ]
    }
   ],
   "source": [
    "filename = 'test.mp3'\n",
    "check_input = song_to_input_matrix(filename)\n",
    "print(check_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Formatting one song into a set of training data\n",
    "sample_data = check_input\n",
    "sample_label = np.random.randint(2, size=sample_data.shape[0])\n",
    "train_ratio = 0.7\n",
    "train_id = np.random.choice(sample_data.shape[0], int(sample_data.shape[0]*train_ratio))\n",
    "\n",
    "data = {\n",
    "    'X_train': sample_data[train_id], # training data\n",
    "    'y_train': sample_label[train_id], # training labels\n",
    "    'X_val': sample_data[-train_id],# validation data\n",
    "    'y_val': sample_label[-train_id]# validation labels\n",
    "  }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial loss (no regularization):  0.6931471804405496\n",
      "Initial loss (with regularization):  0.6931667512749604\n"
     ]
    }
   ],
   "source": [
    "## checking forward pass\n",
    "model = ConvNet(num_filters = [3,4], input_dim = (1,128,128),filter_sizes = [4,4], hidden_dims = [40], \n",
    "                conv_param = {'stride': 2, 'pad': 1}, pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2})\n",
    "\n",
    "X = check_input #np.random.randn(N, 1, 128, 128)\n",
    "N = X.shape[0]\n",
    "y = np.random.randint(2, size=N)\n",
    "\n",
    "loss, grads = model.loss(X, y)\n",
    "print('Initial loss (no regularization): ', loss)\n",
    "\n",
    "model.reg = 0.5\n",
    "loss, grads = model.loss(X, y)\n",
    "print('Initial loss (with regularization): ', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Iteration 1 / 4) loss: 0.693147\n",
      "(Epoch 1 / 4) train acc: 0.530612; val_acc: 0.530612\n",
      "(Epoch 2 / 4) train acc: 0.530612; val_acc: 0.530612\n",
      "(Epoch 3 / 4) train acc: 0.469388; val_acc: 0.469388\n",
      "(Epoch 4 / 4) train acc: 0.469388; val_acc: 0.469388\n"
     ]
    }
   ],
   "source": [
    "## checking for training\n",
    "model = ConvNet(num_filters = [3,4], input_dim = (1,128,128),filter_sizes = [4,4], hidden_dims = [40], \n",
    "                conv_param = {'stride': 2, 'pad': 1}, pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2},\n",
    "                weight_scale=0.001, reg=0.001)\n",
    "\n",
    "solver = Solver(model, data,\n",
    "                num_epochs=4, batch_size=50,\n",
    "                update_rule='adam',\n",
    "                optim_config={\n",
    "                  'learning_rate': 1e-3,\n",
    "                },\n",
    "                verbose=True, print_every=20)\n",
    "solver.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}