SpeechRecognition_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Datasets from audio files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.misc  # for image resizing\n",
    "\n",
    "#import scipy.io.wavfile\n",
    "\n",
    "# pip install soundfile\n",
    "import soundfile\n",
    "\n",
    "from IPython.display import Audio as audio_playback_widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = './data/raw-from-phone.wav'\n",
    "#f = './data/num_phone_en-UK_m_Martin15.wav'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normally an audio file needs clipping\n",
    "\n",
    "The following let's us examine the audio input, and choose the region of interest (in seconds from the start of the input audio file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in the original file\n",
    "samples, sample_rate = soundfile.read(f)\n",
    "\n",
    "def show_waveform(sound):\n",
    "    n_samples = sound.shape[0]\n",
    "\n",
    "    plt.figure(figsize=(12,2))\n",
    "    plt.plot(np.arange(0.0, n_samples)/sample_rate, sound)\n",
    "    plt.xticks( np.arange(0.0, n_samples/sample_rate, 0.5), rotation=90 )\n",
    "\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "show_waveform(samples)\n",
    "audio_playback_widget(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's select the region of interest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crop = (3.25, 16.25)  # in seconds (from waveform graph above)\n",
    "\n",
    "cropped = samples[ int(crop[0]*sample_rate):int(crop[1]*sample_rate) ]\n",
    "\n",
    "show_waveform(cropped)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When satisfied, write the file to disk - and update the name as appropriate (it's also possible to over-write the existing file).\n",
    "\n",
    "Be **careful** with this step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Only do this (set it to 1) if you want to replace the file with the cropped version...\n",
    "if 1:\n",
    "    f = './data/cropped-raw-from-phone.wav'  \n",
    "    soundfile.write(f, cropped, samplerate=sample_rate)\n",
    "    print(\"Wrote '%s'\" % (f,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now look at the audio spectrograms\n",
    "\n",
    "First, we'll do this 'by hand', so that the code is laid out clearly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = './data/num_phone_en-UK_m_Martin00.wav'\n",
    "#f = './data/num_Bing_en-UK_f_Susan.wav'\n",
    "\n",
    "#f = './data/animals_phone_en-UK_m_Martin02.wav'\n",
    "\n",
    "#f = './data/num_phone_en-UK_m_Martin00.ogg'\n",
    "#f = './data/num_Bing_en-UK_f_Susan.ogg'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following defines a function that does the spectrogram (FFT, etc), and then we define a smoothing function that will help us segment the audio into words later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spectrogram(wav_filepath):\n",
    "    samples, sample_rate = soundfile.read(wav_filepath)\n",
    "\n",
    "    # Original code from :\n",
    "    # https://mail.python.org/pipermail/chicago/2010-December/007314.html\n",
    "\n",
    "    # Rescale so that max/min are ~ +/- 1 around 0\n",
    "    data_av = np.mean(samples)\n",
    "    data_max = np.max(np.absolute(samples-data_av))\n",
    "    sound_data = (samples - data_av)/data_max\n",
    "    \n",
    "    ## Parameters: 10ms step, 30ms window\n",
    "    nstep = int(sample_rate * 0.01)\n",
    "    nwin  = int(sample_rate * 0.03)\n",
    "    nfft = 2*int(nwin/2)\n",
    "\n",
    "    window = np.hamming(nwin)\n",
    "\n",
    "    # will take windows x[n1:n2].  generate and loop over \n",
    "    # n2 such that all frames fit within the waveform\n",
    "    nn = range(nwin, len(sound_data), nstep)\n",
    "\n",
    "    X = np.zeros( (len(nn), nfft//2) )\n",
    "\n",
    "    for i,n in enumerate(nn):\n",
    "        segment = sound_data[ n-nwin:n ]\n",
    "        z = np.fft.fft(window * segment, nfft)\n",
    "        X[i,:] = np.log(np.absolute(z[:nfft//2]))\n",
    "        \n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is a function that smooths a time-series\n",
    "#   which enables us to segment the input into words by looking at the 'energy' profile\n",
    "def smooth(x, window_len=31):  # , window='hanning'\n",
    "    # http://scipy-cookbook.readthedocs.io/items/SignalSmooth.html\n",
    "    #s = np.r_[ x[window_len-1:0:-1], x, x[-1:-window_len:-1]]\n",
    "    s = np.r_[ np.zeros( ((window_len-1)//2,) ), x, np.zeros( ((window_len-1)//2,) ) ]\n",
    "    w=np.hamming(window_len)\n",
    "    return np.convolve(w/w.sum(), s, mode='valid') #[window_len-1 : -(window_len-1) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = spectrogram(f)\n",
    "print(\"X.shape=\", X.shape)\n",
    "\n",
    "#Y = np.std(X, axis=1)\n",
    "Y = np.max(X, axis=1)\n",
    "Y_min = np.min(Y)\n",
    "Y_range = Y.max()-Y_min\n",
    "Y = (Y - Y_min)/Y_range\n",
    "\n",
    "print(\"Y.shape=\", Y.shape)\n",
    "\n",
    "Y_crop = np.where(Y>0.25, 1.0, 0.0)\n",
    "\n",
    "# Apply some smoothing\n",
    "Y_crop = smooth(Y_crop)\n",
    "\n",
    "Y_crop = np.where(Y_crop>0.01, 1.0, 0.0)\n",
    "print(\"Y_crop.shape=\", Y_crop.shape)\n",
    "\n",
    "plt.figure(figsize=(12,3))\n",
    "plt.imshow(X.T, interpolation='nearest', origin='lower', aspect='auto')\n",
    "plt.xlim(xmin=0)\n",
    "plt.ylim(ymin=0)\n",
    "\n",
    "plt.plot(Y * X.shape[1])\n",
    "\n",
    "plt.plot(Y_crop * X.shape[1])\n",
    "\n",
    "plt.show()\n",
    "#Y.min(), Y.max()\n",
    "#X[100,:]\n",
    "print( np.argmin(X)/248, np.argmax(X)/248 )\n",
    "\n",
    "audio_playback_widget(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Work out the contiguous region of high enery (== sound) so that we can split the file into voiced segments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#http://stackoverflow.com/questions/4494404/find-large-number-of-consecutive-values-fulfilling-condition-in-a-numpy-array\n",
    "def contiguous_regions(condition):\n",
    "    idx = []\n",
    "    i = 0\n",
    "    while i < len(condition):\n",
    "        x1 = i + condition[i:].argmax()\n",
    "        try:\n",
    "            x2 = x1 + condition[x1:].argmin()\n",
    "        except:\n",
    "            x2 = x1 + 1\n",
    "        if x1 == x2:\n",
    "            if condition[x1] == True:\n",
    "                x2 = len(condition)\n",
    "            else:\n",
    "                break\n",
    "        idx.append( [x1,x2] )\n",
    "        i = x2\n",
    "    return idx\n",
    "\n",
    "contiguous_regions(Y_crop>0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Next : Think about lists of words for the datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "remove_punc = re.compile('[\\,\\.\\?\\!]')\n",
    "squash_spaces = re.compile('\\s+')\n",
    "def words(s):\n",
    "    s = remove_punc.sub(' ', s)\n",
    "    s = squash_spaces.sub(' ', s)\n",
    "    return s.strip().lower()\n",
    "\n",
    "sentences=dict(\n",
    "    num=words(\"zero one two three four five six seven eight nine.\"),\n",
    "    \n",
    "    animals=words(\"cat dog fox bird.\"),\n",
    "    \n",
    "    # https://www.quora.com/Is-there-a-text-that-covers-the-entire-English-phonetic-range/\n",
    "    qbf=words(\"That quick beige fox jumped in the air over each thin dog.  \"+\n",
    "              \"Look out, I shout, for he's foiled you again, creating chaos.\"),\n",
    "    shy=words(\"Are those shy Eurasian footwear, cowboy chaps, \"+\n",
    "              \"or jolly earthmoving headgear?\"),\n",
    "    ate=words(\"The hungry purple dinosaur ate the kind, zingy fox, the jabbering crab, \"+\n",
    "              \"and the mad whale and started vending and quacking.\"),\n",
    "    suz=words(\"With tenure, Suzie'd have all the more leisure for yachting, \"+\n",
    "              \"but her publications are no good.\"),\n",
    "    tbh=words(\"Shaw, those twelve beige hooks are joined if I patch a young, gooey mouth.\"),\n",
    "    \n",
    "    #  https://en.wikipedia.org/wiki/The_North_Wind_and_the_Sun          #594\n",
    "    #  http://videoweb.nie.edu.sg/phonetic/courses/aae103-web/wolf.html  #1111\n",
    ")\n",
    "sentences['num']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also generate voices synthetically - and Bing has a nice interface for that at https://www.microsoft.com/cognitive-services/en-us/speech-api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def for_msft(prefixes):  # comma separated\n",
    "    return ' '.join([sentences[a] for a in prefixes.split(',')]).replace(' ', '\\n') \n",
    "\"\"\"\n",
    "This is the SSML that will be sent to the service:\n",
    "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" \n",
    "      xmlns:mstts=\"http://www.w3.org/2001/mstts\" xml:lang=\"en-GB\">\n",
    "  <voice xml:lang=\"en-GB\" name=\"Microsoft Server Speech Text to Speech Voice (en-GB, Susan, Apollo)\">\n",
    "zero\n",
    "one\n",
    "two\n",
    "three\n",
    "four\n",
    "five\n",
    "six\n",
    "seven\n",
    "eight\n",
    "nine\n",
    "  </voice>\n",
    "</speak>\n",
    "\"\"\"\n",
    "\n",
    "# https://www.microsoft.com/cognitive-services/en-us/Speech-api/documentation/API-Reference-REST/BingVoiceOutput\n",
    "a=for_msft('num')  # 49 long...\n",
    "#a=for_msft('qbf,shy,ate,suz,tbh')  # 474 long...\n",
    "\n",
    "print(\"length_in_chars=%d\\n%s\" % (len(a),a,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you want to do some manipulations on raw audio in Linux, ```sox``` is the perfect tool."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sox_ogg_param='--rate 16000 --channels 1'\n",
    "# sox_wav_param=\"${sox_ogg_param} --encoding signed-integer\"\n",
    "\n",
    "# sox english.au ${sox_wav_param} english.wav norm -3\n",
    "# sox english.au ${sox_ogg_param} english.ogg norm -3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now use 'proper' audio tools for segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install python_speech_features\n",
    "import python_speech_features\n",
    "\n",
    "sample_window_step = 0.01 # in seconds (10ms)\n",
    "\n",
    "def get_sample_features(samples, sample_rate):\n",
    "    #sample_feat = python_speech_features.mfcc(samples, sample_rate, numcep=13, nfilt=26, appendEnergy=True)\n",
    "    #sample_feat = python_speech_features.mfcc(samples, sample_rate, numcep=28, nfilt=56, appendEnergy=True)\n",
    "\n",
    "    #sample_feat, e = python_speech_features.fbank(samples,samplerate=sample_rate,\n",
    "    #      winlen=0.025,winstep=0.01,nfilt=26,nfft=512,\n",
    "    #      lowfreq=0,highfreq=None,preemph=0.97, winfunc=lambda x:np.ones((x,)))\n",
    "\n",
    "    features, energy = python_speech_features.fbank(samples, samplerate=sample_rate, \n",
    "                            winlen=0.025, winstep=sample_window_step, \n",
    "                            nfilt=32,nfft=512,\n",
    "                            lowfreq=0,highfreq=None,preemph=0.25,\n",
    "                            winfunc=lambda x:np.hamming( x ))\n",
    "    return features, energy\n",
    "    \n",
    "def get_sample_isolated_words(energy, plot=False):\n",
    "    log_e = np.log(energy)\n",
    "    if plot: plt.plot(log_e-5)\n",
    "\n",
    "    #log_e = smooth(log_e)\n",
    "    #if plot: plt.plot(log_e)\n",
    "    \n",
    "    log_e_hurdle = (log_e.max() - log_e.min())*0.25 + log_e.min()\n",
    "\n",
    "    log_e_crop = np.where(log_e>log_e_hurdle, 1.0, 0.0)\n",
    "    if plot: plt.plot(log_e_crop * 25 - 2.5)\n",
    "\n",
    "    # By smoothing, and applying a very low hurdle, we expand the crop area safely\n",
    "    log_e_crop_expanded = np.where( smooth(log_e_crop, )>0.01, 1.0, 0.0)\n",
    "    if plot: plt.plot(log_e_crop_expanded * 30 -5)\n",
    "    \n",
    "    return contiguous_regions(log_e_crop_expanded>0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo the calculation above, but using the 'proper' tools.  Notice how the scaling, contrast, etc, are better 'looking'.  \n",
    "\n",
    "Actually, the 'look' is something that we actually care about here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples, sample_rate = soundfile.read(f)\n",
    "\n",
    "sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "\n",
    "plt.figure(figsize=(12,3))\n",
    "plt.imshow(np.log(sample_feat.T), interpolation='nearest', origin='lower', aspect='auto')\n",
    "plt.xlim(xmin=0)\n",
    "\n",
    "word_ranges = get_sample_isolated_words(energy, plot=True)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(sample_feat.shape, energy.shape, energy[10])\n",
    "audio_playback_widget(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the dataset : in steps\n",
    "\n",
    "Break sound into separate WAVs in word-based directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_combined_file_into_wavs(f, prefix='num'):\n",
    "    # f ~ './data/num_Bing_en-UK_f_Susan.wav'\n",
    "    f_base_orig = os.path.basename( f )\n",
    "    if not f_base_orig.startswith(prefix+\"_\"): \n",
    "        print(\"Wrong prefix for '%s'\" % (f_base_orig,))\n",
    "        return\n",
    "    \n",
    "    # Here's the new filename (directory to be calculated per-word)\n",
    "    f_base = os.path.splitext(f_base_orig)[0][len(prefix)+1:] + '.wav'\n",
    "    \n",
    "    samples, sample_rate = soundfile.read(f)\n",
    "    sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "    word_ranges = get_sample_isolated_words(energy, plot=False)\n",
    "    #print(word_ranges)\n",
    "    \n",
    "    words = sentences[prefix].split(' ')\n",
    "    if len(word_ranges) != len(words):\n",
    "        print(\"Found %d segments, rather than %d, in '%s'\" % (len(word_ranges), len(words), f,))\n",
    "        return\n",
    "    \n",
    "    for i, word in enumerate(words):\n",
    "        word_path = os.path.join('data', prefix, word)\n",
    "        os.makedirs(word_path, exist_ok=True)\n",
    "        \n",
    "        wr = word_ranges[i]\n",
    "        fac = int(sample_window_step*sample_rate)\n",
    "        soundfile.write(os.path.join(word_path, f_base), samples[ wr[0]*fac:wr[1]*fac ], samplerate=sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_combined_file_into_wavs('./data/num_Bing_en-UK_f_Susan.wav')\n",
    "#split_combined_file_into_wavs('./data/num_phone_en-UK_m_Martin00.wav')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Iterate through all the audio files with a given prefix, and unfold them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_all_combined_files_into_wavs(prefix='num'):\n",
    "    for audio_file in sorted(os.listdir( './data' )):\n",
    "        filename_stub, ext = os.path.splitext(audio_file)\n",
    "        if not (ext=='.wav' or ext=='.ogg'): continue\n",
    "        if not filename_stub.startswith( prefix+'_'): continue\n",
    "    \n",
    "        print(\"Splitting %s\" % (audio_file,))\n",
    "        split_combined_file_into_wavs( './data/'+audio_file, prefix=prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_all_combined_files_into_wavs(prefix='num')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert WAVs to 'stamps'\n",
    "\n",
    "Now we have some nice WAV files placed into folders which are named according to the word inside, let's create a function that preprocesses the audio clips into 'stamp' files that are essentially spectrograms with a fixed size (and ```uint8``` data type - which makes the ```numpy``` array small)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert a given (isolated word) WAV into a 'stamp' - using a helper function\n",
    "\n",
    "def samples_to_stamp(samples, sample_rate):\n",
    "    sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "    \n",
    "    data = np.log(sample_feat)\n",
    "    \n",
    "    # Now normalize each vertical slice so that the minimum energy is ==0\n",
    "    data_mins = np.min(data, axis=1)\n",
    "    data_min0 = data - data_mins[:, np.newaxis]\n",
    "    \n",
    "    # Force the data into the 'stamp size' as an image (implicit range normalization occurs)\n",
    "    stamp = scipy.misc.imresize(data_min0, (64, 32), 'bilinear')\n",
    "    \n",
    "    # https://github.com/scipy/scipy/issues/4458 :: The stamps are stored as uint8...\n",
    "    return stamp\n",
    "\n",
    "def wav_to_stamp(prefix, word, wav):\n",
    "    samples, sample_rate = soundfile.read( os.path.join('data', prefix, word, wav) )\n",
    "    return samples_to_stamp(samples, sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show what the 'visual stamp' for a given word looks like\n",
    "stamp = wav_to_stamp('num', 'six', 'phone_en-UK_m_Martin00.wav')\n",
    "\n",
    "plt.imshow(stamp.T, interpolation='nearest', origin='lower', aspect='auto')\n",
    "plt.show()\n",
    "\n",
    "print( np.min(stamp), np.max(stamp) )\n",
    "audio_playback_widget( os.path.join('data', 'num', 'six', 'phone_en-UK_m_Martin00.wav') )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collect the WAVs into a 'stamp' dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine all words from a given prefix into a dataset of 'stamps'\n",
    "import pickle\n",
    "\n",
    "def create_dataset_from_folders(prefix, save_as='.pkl', seed=13):\n",
    "    words = sentences[prefix].split(' ')\n",
    "    stamps, labels = [], []\n",
    "    \n",
    "    for label_i, word in enumerate( words ):\n",
    "        # Find all the files for this word\n",
    "        for stamp_file in os.listdir( os.path.join('data', prefix, word )):\n",
    "            if not f.endswith('.wav'): continue\n",
    "            #print(stamp_file)\n",
    "            stamp = wav_to_stamp(prefix, word, stamp_file)\n",
    "            \n",
    "            stamps.append(stamp)\n",
    "            labels.append(label_i)\n",
    "\n",
    "    if save_as is None: # Return the data directly\n",
    "        return stamps, labels, words\n",
    "    \n",
    "    np.random.seed(seed)\n",
    "    data_dictionary = dict(\n",
    "        stamp=stamps, label=labels, \n",
    "        rand=np.random.rand( len(labels) ), # This is to enable us to sample the data (based on hurdles)\n",
    "        words=words, \n",
    "    )\n",
    "    ds_file = os.path.join('data', prefix+save_as)\n",
    "    pickle.dump(data_dictionary, open(ds_file, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    print(\"Created dataset : %s\" % (ds_file, ))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#if not os.path.exists('data/num.pkl'):\n",
    "if True:\n",
    "    create_dataset_from_folders('num')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test that the dataset can be read back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in the dataset\n",
    "dataset = pickle.load(open(os.path.join('data', 'num.pkl'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot all of a given 'word'\n",
    "indices = [ i for i,label in enumerate(dataset['label']) if dataset['words'][label]=='six']\n",
    "\n",
    "plt.figure(figsize=(12, 2))\n",
    "for pos, i in enumerate(indices[0:16]):  # at most 16\n",
    "    plt.subplot(2, 8, pos+1)  # nrows, ncols, subplot#\n",
    "    plt.imshow(dataset['stamp'][i].T, cmap='gray', origin='lower', interpolation='nearest')\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Enable 'ad-hoc' look-see testing\n",
    "\n",
    "This allows us to just pop single words, etc, into a folder, and have them labelled with the filename only (no actual label).  This is really just for show-and-tell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now do something similar for 'test files', create a dataset for all the audio files in the given folder\n",
    "\n",
    "def create_dataset_from_adhoc_wavs(prefix, save_as='.pkl', seed=13):\n",
    "    stamps, labels, words = [], [], []\n",
    "    \n",
    "    for audio_file in sorted(os.listdir( os.path.join('data', prefix) )):\n",
    "        filename_stub, ext = os.path.splitext(audio_file)\n",
    "        if not (ext=='.wav' or ext=='.ogg'): continue\n",
    "            \n",
    "        samples, sample_rate = soundfile.read( os.path.join('data', prefix, audio_file) )\n",
    "        sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "        word_ranges = get_sample_isolated_words(energy, plot=False)\n",
    "    \n",
    "        for i, wr in enumerate(word_ranges):\n",
    "            wr = word_ranges[i]\n",
    "            fac = int(sample_window_step*sample_rate)\n",
    "            segment = samples[ wr[0]*fac:wr[1]*fac ]\n",
    "\n",
    "            stamp = samples_to_stamp(segment, sample_rate)\n",
    "            \n",
    "            print(\"Adding : %s #%2d : (%d,%d)\" % (filename_stub, i, wr[0], wr[1],))\n",
    "            stamps.append(stamp)\n",
    "            labels.append(-1)\n",
    "            words.append(\"%s_%d\" % (filename_stub, i))\n",
    "    \n",
    "    np.random.seed(seed)\n",
    "    data_dictionary = dict(\n",
    "        stamp=stamps, label=labels, \n",
    "        rand=np.random.rand( len(labels) ),\n",
    "        words=words, \n",
    "    )\n",
    "    ds_file = os.path.join('data', prefix+save_as)\n",
    "    pickle.dump(data_dictionary, open(ds_file, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    print(\"Created dataset : %s\" % (ds_file, ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_prefix = 'num' +'-test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_dataset_from_adhoc_wavs(test_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in the ad-hoc test dataset\n",
    "dataset = pickle.load(open(os.path.join('data', 'num-test.pkl'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,2))\n",
    "for pos in range(len(dataset['stamp'][0:16])):  # at most 16\n",
    "    plt.subplot(2, 8, pos+1)  # nrows, ncols, subplot#\n",
    "    plt.imshow(dataset['stamp'][pos].T, cmap='gray', origin='lower', interpolation='nearest')\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All done : Let's Train a Neural Network!\n",
    "\n",
    "(please go to the '_Learn' Speech Recognition notebook : The dataset preparation for the numbers is done..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------\n",
    "\n",
    "\n",
    "### Extra! : Create 'animals' datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First a training set\n",
    "split_all_combined_files_into_wavs(prefix='animals')\n",
    "create_dataset_from_folders('animals')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# And then some ad-hoc test cases\n",
    "test_prefix = 'animals' +'-test'\n",
    "create_dataset_from_adhoc_wavs(test_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_playback_widget( os.path.join('data', test_prefix, 'cat_dog_fox_bird.wav') )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now see whether we can learn the 'animal' words using the 'numbers' network..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}