3-5-10-SpectrogramsMFCCs.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "3-5-10-SpectogramsMFCCs.ipynb",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tinyMLx/colabs/blob/master/3-5-10-SpectrogramsMFCCs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z9GCyPWNuOm7"
      },
      "source": [
        "# First load in the function to record your own audio!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upgCc3gXybsA"
      },
      "source": [
        "### Install required Packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uUDYyMZRfkX4"
      },
      "source": [
        "!pip install ffmpeg-python &> 0\n",
        "!pip install tensorflow-io &> 0\n",
        "!pip install python_speech_features &> 0\n",
        "print(\"Packages Installed\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bGzzi-siATEZ"
      },
      "source": [
        "### Import everything we will need"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WV8869MMAZKa"
      },
      "source": [
        "from IPython.display import HTML, Audio\n",
        "from google.colab.output import eval_js\n",
        "from base64 import b64decode\n",
        "import numpy as np\n",
        "from scipy.io.wavfile import read as wav_read\n",
        "import io\n",
        "import ffmpeg\n",
        "import tensorflow as tf\n",
        "import tensorflow_io as tfio\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from python_speech_features import mfcc\n",
        "from matplotlib import cm\n",
        "import pickle\n",
        "import librosa\n",
        "print(\"Packages Imported\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MJPb0BZuAcit"
      },
      "source": [
        "### Define the audio importing function.\n",
        "Adapted from: https://ricardodeazambuja.com/deep_learning/2019/03/09/audio_and_video_google_colab/ and https://colab.research.google.com/drive/1Z6VIRZ_sX314hyev3Gm5gBqvm1wQVo-a#scrollTo=RtMcXr3o6gxN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xamv_S585KgI"
      },
      "source": [
        "AUDIO_HTML = \"\"\"\n",
        "<script>\n",
        "var my_div = document.createElement(\"DIV\");\n",
        "var my_p = document.createElement(\"P\");\n",
        "var my_btn = document.createElement(\"BUTTON\");\n",
        "var t = document.createTextNode(\"Press to start recording\");\n",
        "\n",
        "my_btn.appendChild(t);\n",
        "//my_p.appendChild(my_btn);\n",
        "my_div.appendChild(my_btn);\n",
        "document.body.appendChild(my_div);\n",
        "\n",
        "var base64data = 0;\n",
        "var reader;\n",
        "var recorder, gumStream;\n",
        "var recordButton = my_btn;\n",
        "\n",
        "var handleSuccess = function(stream) {\n",
        "  gumStream = stream;\n",
        "  var options = {\n",
        "    //bitsPerSecond: 8000, //chrome seems to ignore, always 48k\n",
        "    mimeType : 'audio/webm;codecs=opus'\n",
        "    //mimeType : 'audio/webm;codecs=pcm'\n",
        "  };            \n",
        "  //recorder = new MediaRecorder(stream, options);\n",
        "  recorder = new MediaRecorder(stream);\n",
        "  recorder.ondataavailable = function(e) {            \n",
        "    var url = URL.createObjectURL(e.data);\n",
        "    var preview = document.createElement('audio');\n",
        "    preview.controls = true;\n",
        "    preview.src = url;\n",
        "    document.body.appendChild(preview);\n",
        "\n",
        "    reader = new FileReader();\n",
        "    reader.readAsDataURL(e.data); \n",
        "    reader.onloadend = function() {\n",
        "      base64data = reader.result;\n",
        "      //console.log(\"Inside FileReader:\" + base64data);\n",
        "    }\n",
        "  };\n",
        "  recorder.start();\n",
        "  };\n",
        "\n",
        "recordButton.innerText = \"Recording... press to stop\";\n",
        "\n",
        "navigator.mediaDevices.getUserMedia({audio: true}).then(handleSuccess);\n",
        "\n",
        "\n",
        "function toggleRecording() {\n",
        "  if (recorder && recorder.state == \"recording\") {\n",
        "      recorder.stop();\n",
        "      gumStream.getAudioTracks()[0].stop();\n",
        "      recordButton.innerText = \"Saving the recording... pls wait!\"\n",
        "  }\n",
        "}\n",
        "\n",
        "// https://stackoverflow.com/a/951057\n",
        "function sleep(ms) {\n",
        "  return new Promise(resolve => setTimeout(resolve, ms));\n",
        "}\n",
        "\n",
        "var data = new Promise(resolve=>{\n",
        "//recordButton.addEventListener(\"click\", toggleRecording);\n",
        "recordButton.onclick = ()=>{\n",
        "toggleRecording()\n",
        "\n",
        "sleep(2000).then(() => {\n",
        "  // wait 2000ms for the data to be available...\n",
        "  // ideally this should use something like await...\n",
        "  //console.log(\"Inside data:\" + base64data)\n",
        "  resolve(base64data.toString())\n",
        "\n",
        "});\n",
        "\n",
        "}\n",
        "});\n",
        "      \n",
        "</script>\n",
        "\"\"\"\n",
        "\n",
        "def get_audio():\n",
        "  display(HTML(AUDIO_HTML))\n",
        "  data = eval_js(\"data\")\n",
        "  binary = b64decode(data.split(',')[1])\n",
        "  \n",
        "  process = (ffmpeg\n",
        "    .input('pipe:0')\n",
        "    .output('pipe:1', format='wav')\n",
        "    .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True, quiet=True, overwrite_output=True)\n",
        "  )\n",
        "  output, err = process.communicate(input=binary)\n",
        "  \n",
        "  riff_chunk_size = len(output) - 8\n",
        "  # Break up the chunk size into four bytes, held in b.\n",
        "  q = riff_chunk_size\n",
        "  b = []\n",
        "  for i in range(4):\n",
        "      q, r = divmod(q, 256)\n",
        "      b.append(r)\n",
        "\n",
        "  # Replace bytes 4:8 in proc.stdout with the actual size of the RIFF chunk.\n",
        "  riff = output[:4] + bytes(b) + output[8:]\n",
        "\n",
        "  sr, audio = wav_read(io.BytesIO(riff))\n",
        "\n",
        "  return audio, sr\n",
        "\n",
        "print(\"Chrome Audio Recorder Defined\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AY-xYPiLAj5j"
      },
      "source": [
        "# Load in the Audio Samples\n",
        "### Record your own audio samples!\n",
        "After you run each cell wait for the stop button to appear then start recording and then press the button to stop the recording once you have said the word! **If you do not want to record audio then see below for a way to load in the default audio used in the lecture slides.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-SSG2lv5Mg9"
      },
      "source": [
        "audio_yes_loud, sr_yes_loud = get_audio()\n",
        "print(\"DONE\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GyB7qaDM6iQj"
      },
      "source": [
        "audio_yes_quiet, sr_yes_quiet = get_audio()\n",
        "print(\"DONE\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EhtrUYWT6ti6"
      },
      "source": [
        "audio_no_loud, sr_no_loud = get_audio()\n",
        "print(\"DONE\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wO4wQp9D98Ds"
      },
      "source": [
        "audio_no_quiet, sr_no_quiet = get_audio()\n",
        "print(\"DONE\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bQiPGQWubCjp"
      },
      "source": [
        "# if you would like to save your files for later, uncomment and run the following\n",
        "# then you can find your files in the folder icon to the left\n",
        "\n",
        "# audio_files = {\n",
        "#   'audio_yes_loud': audio_yes_loud, 'sr_yes_loud': sr_yes_loud,\n",
        "#   'audio_yes_quiet': audio_yes_quiet, 'sr_yes_quiet': sr_yes_quiet,\n",
        "#   'audio_no_loud': audio_no_loud, 'sr_no_loud': sr_no_loud,\n",
        "#   'audio_no_quiet': audio_no_quiet, 'sr_no_quiet': sr_no_quiet,\n",
        "# }\n",
        "# with open('audio_files.pkl', 'wb') as fid:\n",
        "#   pickle.dump(audio_files,fid)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m1mc3l3bZbJv"
      },
      "source": [
        "### Or load in the default audio samples\n",
        "By uncommenting and running the below"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2vIjpKveaSzm"
      },
      "source": [
        "# !wget --no-check-certificate --content-disposition https://github.com/tinyMLx/colabs/blob/master/audio_files.pkl?raw=true\n",
        "# print(\"Wait a minute for the file to sync in the Colab and then run the next cell!\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gb3WbdPPdCmu"
      },
      "source": [
        "# fid = open('audio_files.pkl', 'rb')\n",
        "# audio_files = pickle.load(fid)\n",
        "# audio_yes_loud = audio_files['audio_yes_loud']\n",
        "# sr_yes_loud = audio_files['sr_yes_loud']\n",
        "# audio_yes_quiet = audio_files['audio_yes_quiet']\n",
        "# sr_yes_quiet = audio_files['sr_yes_quiet']\n",
        "# audio_no_loud = audio_files['audio_no_loud']\n",
        "# sr_no_loud = audio_files['sr_no_loud']\n",
        "# audio_no_quiet = audio_files['audio_no_quiet']\n",
        "# sr_no_quiet = audio_files['sr_no_quiet']"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5FKjT0VHZLvD"
      },
      "source": [
        "You can hear the audio files you loaded by uncommenting and running the below"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toHWCy1JZgzH"
      },
      "source": [
        "# Audio(audio_yes_loud, rate=sr_yes_loud)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YUPW18JEZr33"
      },
      "source": [
        "# Audio(audio_yes_quiet, rate=sr_yes_quiet)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BhsfxQk5ZsB_"
      },
      "source": [
        "# Audio(audio_no_loud, rate=sr_no_loud)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NPcNYzcaZsJy"
      },
      "source": [
        "# Audio(audio_no_quiet, rate=sr_no_quiet)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VBE2WWudAo4D"
      },
      "source": [
        "# Visualize the samples\n",
        "### First as signals"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "il58WctF5fPV"
      },
      "source": [
        "# Plot the figure\n",
        "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)\n",
        "max_val = max(np.append(np.append(np.append(audio_yes_loud,audio_yes_quiet),audio_no_loud),audio_no_quiet))\n",
        "ax1.plot(audio_yes_loud)\n",
        "ax1.set_title(\"Yes Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax1.set_ylim(-max_val, max_val)\n",
        "ax2.plot(audio_yes_quiet)\n",
        "ax2.set_title(\"Yes Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax2.set_ylim(-max_val, max_val)\n",
        "ax3.plot(audio_no_loud)\n",
        "ax3.set_title(\"No Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax3.set_ylim(-max_val, max_val)\n",
        "ax4.plot(audio_no_quiet)\n",
        "ax4.set_title(\"No Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax4.set_ylim(-max_val, max_val)\n",
        "fig.set_size_inches(18,12)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61oIZujzLU4d"
      },
      "source": [
        "### Then view the Fourier Transform of the Signal\n",
        "Viewing the signal in the frequency domain."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gf18UcyhM571"
      },
      "source": [
        "#Adapted from https://makersportal.com/blog/2018/9/13/audio-processing-in-python-part-i-sampling-and-the-fast-fourier-transform\n",
        "# compute the FFT and take the single-sided spectrum only and remove imaginary part\n",
        "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)\n",
        "ft_audio_yes_loud = np.abs(2*np.fft.fft(audio_yes_loud))\n",
        "ft_audio_yes_quiet = np.abs(2*np.fft.fft(audio_yes_quiet))\n",
        "ft_audio_no_loud = np.abs(2*np.fft.fft(audio_no_loud))\n",
        "ft_audio_no_quiet = np.abs(2*np.fft.fft(audio_no_quiet))\n",
        "\n",
        "# Plot the figure\n",
        "ax1.plot(ft_audio_yes_loud)\n",
        "ax1.set_xscale('log')\n",
        "ax1.set_yscale('log')\n",
        "ax1.set_title(\"Yes Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax2.plot(ft_audio_yes_quiet)\n",
        "ax2.set_xscale('log')\n",
        "ax2.set_yscale('log')\n",
        "ax2.set_title(\"Yes Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax3.plot(ft_audio_no_loud)\n",
        "ax3.set_xscale('log')\n",
        "ax3.set_yscale('log')\n",
        "ax3.set_title(\"No Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax4.plot(ft_audio_no_quiet)\n",
        "ax4.set_xscale('log')\n",
        "ax4.set_yscale('log')\n",
        "ax4.set_title(\"No Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "fig.set_size_inches(18,12)\n",
        "fig.text(0.5, 0.06, 'Frequency [Hz]', {'fontsize':20, 'fontweight':'bold'}, ha='center');\n",
        "fig.text(0.08, 0.5, 'Amplitude', {'fontsize':20, 'fontweight':'bold'}, va='center', rotation='vertical');"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ou4Jf5U1Avuo"
      },
      "source": [
        "### Then as spectrograms\n",
        "Can you see how spectrograms can help machine learning models better differentiate between audio samples?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIOmPv975gYW"
      },
      "source": [
        "# Convert to spectrogram and display\n",
        "# adapted from https://aruno14.medium.com/comparaison-of-audio-representation-in-tensorflow-b6c33a83d77f\n",
        "spectrogram_yes_loud = tfio.audio.spectrogram(audio_yes_loud/1.0, nfft=2048, window=len(audio_yes_loud), stride=int(sr_yes_loud * 0.008))\n",
        "spectrogram_yes_quiet = tfio.audio.spectrogram(audio_yes_quiet/1.0, nfft=2048, window=len(audio_yes_quiet), stride=int(sr_yes_quiet * 0.008))\n",
        "spectrogram_no_loud = tfio.audio.spectrogram(audio_no_loud/1.0, nfft=2048, window=len(audio_no_loud), stride=int(sr_no_loud * 0.008))\n",
        "spectrogram_no_quiet = tfio.audio.spectrogram(audio_no_quiet/1.0, nfft=2048, window=len(audio_no_quiet), stride=int(sr_no_quiet * 0.008))\n",
        "\n",
        "# Plot the figure\n",
        "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)\n",
        "ax1.imshow(tf.math.log(spectrogram_yes_loud).numpy(), aspect='auto')\n",
        "ax1.set_title(\"Yes Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax2.imshow(tf.math.log(spectrogram_yes_quiet).numpy(), aspect='auto')\n",
        "ax2.set_title(\"Yes Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax3.imshow(tf.math.log(spectrogram_no_loud).numpy(), aspect='auto')\n",
        "ax3.set_title(\"No Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax4.imshow(tf.math.log(spectrogram_no_quiet).numpy(), aspect='auto')\n",
        "ax4.set_title(\"No Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "fig.set_size_inches(18,12)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "thaEF4eV0Wol"
      },
      "source": [
        "### Then as MFCCs\n",
        "Using the Mel Scale to better associate the features to human hearing!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qh-D5tV20VwG"
      },
      "source": [
        "# Convert to MFCC using the Mel Scale\n",
        "# adapted from: https://towardsdatascience.com/getting-to-know-the-mel-spectrogram-31bca3e2d9d0\n",
        "mfcc_yes_loud = librosa.power_to_db(librosa.feature.melspectrogram(\n",
        "    y=np.float32(audio_yes_loud), sr=sr_yes_loud, n_fft=2048, hop_length=512, n_mels=128), ref=np.max)\n",
        "mfcc_yes_quiet = librosa.power_to_db(librosa.feature.melspectrogram(\n",
        "    y=np.float32(audio_yes_quiet), sr=sr_yes_quiet, n_fft=2048, hop_length=512, n_mels=128), ref=np.max)\n",
        "mfcc_no_loud = librosa.power_to_db(librosa.feature.melspectrogram(\n",
        "    y=np.float32(audio_no_loud), sr=sr_no_loud, n_fft=2048, hop_length=512, n_mels=128), ref=np.max)\n",
        "mfcc_no_quiet = librosa.power_to_db(librosa.feature.melspectrogram(\n",
        "    y=np.float32(audio_no_quiet), sr=sr_no_quiet, n_fft=2048, hop_length=512, n_mels=128), ref=np.max)\n",
        "\n",
        "# Plot the figure\n",
        "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)\n",
        "ax1.imshow(np.swapaxes(mfcc_yes_loud, 0 ,1), interpolation='nearest', cmap=cm.viridis, origin='lower', aspect='auto')\n",
        "ax1.set_title(\"Yes Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax1.set_ylim(ax1.get_ylim()[::-1])\n",
        "ax2.imshow(np.swapaxes(mfcc_yes_quiet, 0 ,1), interpolation='nearest', cmap=cm.viridis, origin='lower', aspect='auto')\n",
        "ax2.set_title(\"Yes Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax2.set_ylim(ax2.get_ylim()[::-1])\n",
        "ax3.imshow(np.swapaxes(mfcc_no_loud, 0 ,1), interpolation='nearest', cmap=cm.viridis, origin='lower', aspect='auto')\n",
        "ax3.set_title(\"No Loud\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax3.set_ylim(ax3.get_ylim()[::-1])\n",
        "ax4.imshow(np.swapaxes(mfcc_no_quiet, 0 ,1), interpolation='nearest', cmap=cm.viridis, origin='lower', aspect='auto')\n",
        "ax4.set_title(\"No Quiet\", {'fontsize':20, 'fontweight':'bold'})\n",
        "ax4.set_ylim(ax4.get_ylim()[::-1])\n",
        "fig.set_size_inches(18,12)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}