6-augmented-pre-processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Classifying audio data with convolutional neural networks\n",
    "\n",
    "<br/>\n",
    "by Eduardo Garcia Rajo<br/>\n",
    "<br/>\n",
    "\n",
    "This notebook is part of the project \"Urban sounds classification with Covnolutional Neural Networks\" on [my Github](https://github.com/GorillaBus/urban-audio-classifier).<br/>\n",
    "<br/>\n",
    "Licensed under the GNU LESSER GENERAL PUBLIC LICENSE Version 3, 29 June 2007<br/>\n",
    "<br/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Augmented data pre-processing\n",
    "\n",
    "MFCC and Log-Mel Spectrogram Coefficients extraction from augmented dataset.<br/>\n",
    "<br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Required libraries\n",
    "import sys\n",
    "import os\n",
    "import IPython as IP\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import librosa\n",
    "import librosa.display\n",
    "import pickle\n",
    "from include import helpers\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils import to_categorical\n",
    "from IPython.display import clear_output, display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### * Setup\n",
    "Pay attention to the very simple path variables configured in this section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set your path to the dataset\n",
    "us8k_path = os.path.abspath('./UrbanSound8K')\n",
    "audio_path = os.path.join(us8k_path, 'audio')\n",
    "augmented_path = os.path.join(audio_path, 'augmented')\n",
    "\n",
    "# Metadata\n",
    "metadata_augmented_path = os.path.abspath('data/augmented-data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### * Load metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metadata length: 69856\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>augment</th>\n",
       "      <th>class</th>\n",
       "      <th>class_id</th>\n",
       "      <th>file</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>69851</td>\n",
       "      <td>pitch_2</td>\n",
       "      <td>car_horn</td>\n",
       "      <td>1</td>\n",
       "      <td>199769-1-0-6.wav</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69852</td>\n",
       "      <td>pitch_2</td>\n",
       "      <td>car_horn</td>\n",
       "      <td>1</td>\n",
       "      <td>18594-1-1-0.wav</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69853</td>\n",
       "      <td>pitch_2</td>\n",
       "      <td>car_horn</td>\n",
       "      <td>1</td>\n",
       "      <td>151359-1-0-0.wav</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69854</td>\n",
       "      <td>pitch_2</td>\n",
       "      <td>car_horn</td>\n",
       "      <td>1</td>\n",
       "      <td>18594-1-0-0.wav</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69855</td>\n",
       "      <td>pitch_2</td>\n",
       "      <td>car_horn</td>\n",
       "      <td>1</td>\n",
       "      <td>199769-1-0-12.wav</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       augment     class  class_id               file  fold\n",
       "69851  pitch_2  car_horn         1   199769-1-0-6.wav     3\n",
       "69852  pitch_2  car_horn         1    18594-1-1-0.wav     3\n",
       "69853  pitch_2  car_horn         1   151359-1-0-0.wav     3\n",
       "69854  pitch_2  car_horn         1    18594-1-0-0.wav     3\n",
       "69855  pitch_2  car_horn         1  199769-1-0-12.wav     3"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the metadata from the generated CSV\n",
    "metadata = pd.read_csv(metadata_augmented_path)\n",
    "\n",
    "# Examine dataframe\n",
    "print(\"Metadata length:\", len(metadata))\n",
    "metadata.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. MFCC extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 69856/69856\n",
      "Last file:  /home/edu/Projects/urban-audio-classifier/UrbanSound8K/audio/fold3/199769-1-0-12.wav\n",
      "Finished: 69855/69856\n"
     ]
    }
   ],
   "source": [
    "# Iterate through all audio files and extract MFCC\n",
    "features = []\n",
    "labels = []\n",
    "frames_max = 0\n",
    "counter = 0\n",
    "total_samples = len(metadata)\n",
    "n_mfcc = 40\n",
    "\n",
    "for index, row in metadata.iterrows():\n",
    "    file_path = os.path.join(os.path.abspath(audio_path), 'fold' + str(row[\"fold\"]), str(row[\"file\"]))\n",
    "    class_label = row[\"class\"]\n",
    "\n",
    "    # Extract MFCCs (do not add padding)\n",
    "    mfccs = helpers.get_mfcc(file_path, 0, n_mfcc)\n",
    "    \n",
    "    # Save current frame count\n",
    "    num_frames = mfccs.shape[1]\n",
    "    \n",
    "    # Add row (feature / label)\n",
    "    features.append(mfccs)\n",
    "    labels.append(class_label)\n",
    "\n",
    "    # Update frames maximum\n",
    "    if (num_frames > frames_max):\n",
    "        frames_max = num_frames\n",
    "        \n",
    "    clear_output(wait=True)\n",
    "    print(\"Progress: {}/{}\".format(index+1, total_samples))\n",
    "    print(\"Last file: \", file_path)\n",
    "\n",
    "    counter += 1\n",
    "    \n",
    "print(\"Finished: {}/{}\".format(index, total_samples))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Add padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "padded = []\n",
    "\n",
    "# Add padding\n",
    "mfcc_max_padding = frames_max\n",
    "for i in range(len(features)):\n",
    "    size = len(features[i][0])\n",
    "    if (size < mfcc_max_padding):\n",
    "        pad_width = mfcc_max_padding - size\n",
    "        px = np.pad(features[i], \n",
    "                    pad_width=((0, 0), (0, pad_width)), \n",
    "                    mode='constant', \n",
    "                    constant_values=(0,))\n",
    "    \n",
    "    padded.append(px)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Save X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert features (X) and labels (y) to Numpy arrays\n",
    "\n",
    "X = np.array(padded)\n",
    "y = np.array(labels)\n",
    "\n",
    "# Optionally save the features to disk\n",
    "np.save(\"data/X-mfcc-augmented\", X)\n",
    "np.save(\"data/y-mfcc-augmented\", y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw features length: 69856\n",
      "Padded features length: 69856\n",
      "Feature labels length: 69856\n",
      "X: (69856, 40, 174), y: (69856,)\n"
     ]
    }
   ],
   "source": [
    "# Verify shapes\n",
    "print(\"Raw features length: {}\".format(len(features)))\n",
    "print(\"Padded features length: {}\".format(len(padded)))\n",
    "print(\"Feature labels length: {}\".format(len(features)))\n",
    "print(\"X: {}, y: {}\".format(X.shape, y.shape))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. Log-Mel Spectrogram extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 69856/69856\n",
      "Last file:  /home/edu/Projects/urban-audio-classifier/UrbanSound8K/audio/fold3/199769-1-0-12.wav\n",
      "Finished: 69855/69856\n"
     ]
    }
   ],
   "source": [
    "# Iterate through all audio files and extract Log-Mel Spectrograms\n",
    "features = []\n",
    "labels = []\n",
    "frames_max = 0\n",
    "counter = 0\n",
    "total_samples = len(metadata)\n",
    "n_mels = 40\n",
    "\n",
    "for index, row in metadata.iterrows():\n",
    "    file_path = os.path.join(os.path.abspath(audio_path), 'fold' + str(row[\"fold\"]), str(row[\"file\"]))\n",
    "    class_label = row[\"class\"]\n",
    "\n",
    "    # Extract Log-Mel Spectrograms (do not add padding)\n",
    "    mels = helpers.get_mel_spectrogram(file_path, 0, n_mels=n_mels)\n",
    "    \n",
    "    # Save current frame count\n",
    "    num_frames = mels.shape[1]\n",
    "    \n",
    "    # Add row (feature / label)\n",
    "    features.append(mels)\n",
    "    labels.append(class_label)\n",
    "\n",
    "    # Update frames maximum\n",
    "    if (num_frames > frames_max):\n",
    "        frames_max = num_frames\n",
    "        \n",
    "    clear_output(wait=True)\n",
    "    print(\"Progress: {}/{}\".format(index+1, total_samples))\n",
    "    print(\"Last file: \", file_path)\n",
    "\n",
    "    counter += 1\n",
    "    \n",
    "print(\"Finished: {}/{}\".format(index, total_samples))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Add padding for a consistent shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "padded = []\n",
    "\n",
    "# Add padding\n",
    "mels_max_padding = frames_max\n",
    "for i in range(len(features)):\n",
    "    size = len(features[i][0])\n",
    "    if (size < mels_max_padding):\n",
    "        pad_width = mels_max_padding - size\n",
    "        px = np.pad(features[i], \n",
    "                    pad_width=((0, 0), (0, pad_width)), \n",
    "                    mode='constant', \n",
    "                    constant_values=(0,))\n",
    "    \n",
    "    padded.append(px)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Save X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert features (X) and labels (y) to Numpy arrays\n",
    "\n",
    "X = np.array(padded)\n",
    "y = np.array(labels)\n",
    "\n",
    "# Optionally save the features to disk\n",
    "np.save(\"data/X-mel_spec-augmented\", X)\n",
    "np.save(\"data/y-mel_spec-augmented\", y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw features length: 69856\n",
      "Padded features length: 69856\n",
      "Feature labels length: 69856\n",
      "X: (69856, 40, 174), y: (69856,)\n"
     ]
    }
   ],
   "source": [
    "# Verify shapes\n",
    "print(\"Raw features length: {}\".format(len(features)))\n",
    "print(\"Padded features length: {}\".format(len(padded)))\n",
    "print(\"Feature labels length: {}\".format(len(features)))\n",
    "print(\"X: {}, y: {}\".format(X.shape, y.shape))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}