BasicModelAudio_part2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "BasicModelAudio-part2.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "4jdaTqPlNcz8",
        "colab_type": "code",
        "colab": {},
        "outputId": "62b76e1a-9970-4c35-9293-8ba2ac266a58"
      },
      "source": [
        "import os\n",
        "import numpy as np\n",
        "from scipy.fftpack import fft\n",
        "from scipy.io import wavfile\n",
        "from scipy import signal\n",
        "from glob import glob\n",
        "import re\n",
        "import pandas as pd\n",
        "import gc\n",
        "from scipy.io import wavfile\n",
        "\n",
        "from keras import optimizers, losses, activations, models\n",
        "from keras.layers import Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization\n",
        "from sklearn.model_selection import train_test_split\n",
        "import keras\n",
        "\n",
        "import torch\n",
        "import torchaudio"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cI8TshmKNc0G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "L = 16000\n",
        "legal_labels = 'yes no up down left right on off stop go silence unknown'.split()\n",
        "\n",
        "#src folders\n",
        "root_path = r'..'\n",
        "out_path = r'.'\n",
        "model_path = r'.'\n",
        "train_data_path = \"C:\\\\Users\\\\prisarkar\\\\Desktop\\\\LanguageEngine\\\\train\\\\audio\"\n",
        "output_file=\"C:\\\\Users\\\\prisarkar\\\\Desktop\\\\LanguageEngine\\\\output.wav\"\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5wR4Fp-zNc0M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def log_specgram(audio, sample_rate, window_size=20,\n",
        "                 step_size=10, eps=1e-10):\n",
        "    nperseg = int(round(window_size * sample_rate / 1e3))\n",
        "    noverlap = int(round(step_size * sample_rate / 1e3))\n",
        "    freqs, times, spec = signal.spectrogram(audio,\n",
        "                                    fs=sample_rate,\n",
        "                                    window='hann',\n",
        "                                    nperseg=nperseg,\n",
        "                                    noverlap=noverlap,\n",
        "                                    detrend=False)\n",
        "    return freqs, times, np.log(spec.T.astype(np.float32) + eps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qTzz7uOCNc0U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######### This function reads within sub folders and create the variabke set\n",
        "def list_wavs_fname(dirpath, ext='wav'):\n",
        "    fpaths = glob(os.path.join(dirpath, r'*/*' + ext))\n",
        "    pat = r'.+/(\\w+)/\\w+\\.' + ext + '$'\n",
        "    labels = []\n",
        "    for fpath in fpaths:\n",
        "        name=fpath.split(\"\\\\\")[-2]\n",
        "        labels.append(name)\n",
        "                         \n",
        "    fnames = []\n",
        "    for fpath in fpaths:\n",
        "        name=fpath.split(\"\\\\\")[-1]\n",
        "        fnames.append(name)\n",
        "        \n",
        "    return labels, fnames\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Op6renOcNc0a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def pad_audio(samples):\n",
        "    if len(samples) >= L: return samples\n",
        "    else: return np.pad(samples, pad_width=(L - len(samples), 0), mode='constant', constant_values=(0, 0))\n",
        "\n",
        "def chop_audio(samples, L=16000, num=20):\n",
        "    for i in range(num):\n",
        "        beg = np.random.randint(0, len(samples) - L)\n",
        "        yield samples[beg: beg + L]\n",
        "\n",
        "def label_transform(labels):\n",
        "    nlabels = []\n",
        "    for label in labels:\n",
        "        if label == '_background_noise_':\n",
        "            nlabels.append('silence')\n",
        "        elif label not in legal_labels:\n",
        "            nlabels.append('unknown')\n",
        "        else:\n",
        "            nlabels.append(label)\n",
        "    return pd.get_dummies(pd.Series(nlabels))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4-mY9LyNc0g",
        "colab_type": "code",
        "colab": {},
        "outputId": "0556fcef-d208-4293-8ca1-eb97c36cd17d"
      },
      "source": [
        "labels,fnames = list_wavs_fname(train_data_path)\n",
        "\n",
        "new_sample_rate = 16000\n",
        "y_train = []\n",
        "x_train = []\n",
        "\n",
        "for label, fname in zip(labels, fnames):\n",
        "    samples, sample_rate = torchaudio.load(os.path.join(train_data_path, label, fname))\n",
        "    samples = pad_audio(samples)\n",
        "    if len(samples) > 16000:\n",
        "        n_samples = chop_audio(samples)\n",
        "    else: n_samples = [samples]\n",
        "    for samples in n_samples:\n",
        "        resampled=torchaudio.transforms.Resample(sample_rate, new_sample_rate)(samples)\n",
        "        mfcc=torchaudio.transforms.MFCC()(resampled))\n",
        "        y_train.append(label)\n",
        "        x_train.append(mfcc)\n",
        "\n",
        "x_train = np.array(x_train)\n",
        "y_train=np.array(y_train)\n",
        "\n",
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "y_train=le.fit_transform(y_train)\n",
        "label_index = le.classes_\n",
        "gc.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\prisarkar\\Anaconda3\\lib\\site-packages\\scipy\\io\\wavfile.py:273: WavFileWarning: Chunk (non-data) not understood, skipping it.\n",
            "  WavFileWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oqJO70mPNc0l",
        "colab_type": "code",
        "colab": {},
        "outputId": "f5e01831-41be-441d-9473-2af8fa4d6f72"
      },
      "source": [
        "x_train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(24222, 99, 161)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rPBWcLLdNc0u",
        "colab_type": "code",
        "colab": {},
        "outputId": "f02ce71d-60fc-48c1-e8f1-a32932e7bd85"
      },
      "source": [
        "label_index"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['_background_noise_', 'down', 'go', 'left', 'no', 'off', 'on',\n",
              "       'right', 'stop', 'up', 'yes', 'स्वागत'], dtype='<U18')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WjF5XADhNc03",
        "colab_type": "code",
        "colab": {},
        "outputId": "85738a48-a2f3-427c-db09-6308680b4ff5"
      },
      "source": [
        "from tensorflow.keras.layers import Dense, SimpleRNN, LSTM, Bidirectional, TimeDistributed, Conv1D, ZeroPadding1D, GRU\n",
        "from tensorflow.keras.layers import Lambda, Input, Dropout, Masking, BatchNormalization, Activation\n",
        "from tensorflow.keras.models import Model\n",
        "import tensorflow as tf\n",
        "\n",
        "\n",
        "def cnn_lstm(input_dim, output_dim, dropout=0.2, n_layers=1):\n",
        "\n",
        "#     # Input data type\n",
        "    dtype = 'float32'\n",
        "\n",
        "    # ---- Network model ----\n",
        "    input_data = Input(name='the_input', shape=input_dim, dtype=dtype)\n",
        "\n",
        "    # 1 x 1D convolutional layers with strides 4\n",
        "    x = Conv1D(filters=256, kernel_size=10, strides=4, name='conv_1')(input_data)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = Activation('relu')(x)\n",
        "    x = Dropout(dropout, name='dropout_1')(x)\n",
        "        \n",
        "    x = LSTM(128, activation='relu', return_sequences=True,\n",
        "             dropout=dropout, name='lstm_1')(x)\n",
        "    x = LSTM(128, activation='relu', return_sequences=False,\n",
        "             dropout=dropout, name='lstm_2')(x)\n",
        "\n",
        "#     # 1 fully connected layer DNN ReLu with default 20% dropout\n",
        "    x = Dense(units=64, activation='relu', name='fc')(x)\n",
        "    x = Dropout(dropout, name='dropout_2')(x)\n",
        "\n",
        "    # Output layer with softmax\n",
        "    y_pred = Dense(units=output_dim, activation='softmax', name='softmax')(x)\n",
        "\n",
        "\n",
        "    network_model = Model(inputs=input_data, outputs=y_pred)\n",
        "    \n",
        "    return network_model\n",
        "\n",
        "input_dim = (99, 161)\n",
        "classes = len(legal_labels)\n",
        "\n",
        "model = cnn_lstm(input_dim, classes)\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From C:\\Users\\prisarkar\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "the_input (InputLayer)       [(None, 99, 161)]         0         \n",
            "_________________________________________________________________\n",
            "conv_1 (Conv1D)              (None, 23, 256)           412416    \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 23, 256)           1024      \n",
            "_________________________________________________________________\n",
            "activation (Activation)      (None, 23, 256)           0         \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 23, 256)           0         \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 23, 128)           197120    \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 128)               131584    \n",
            "_________________________________________________________________\n",
            "fc (Dense)                   (None, 64)                8256      \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "softmax (Dense)              (None, 12)                780       \n",
            "=================================================================\n",
            "Total params: 751,180\n",
            "Trainable params: 750,668\n",
            "Non-trainable params: 512\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8kMKXhkvNc08",
        "colab_type": "code",
        "colab": {},
        "outputId": "72a643ee-b369-45df-ae78-43518313e28f"
      },
      "source": [
        "from tensorflow.keras.optimizers import Adam, SGD\n",
        "from tensorflow.keras.callbacks import TensorBoard\n",
        "\n",
        "sgd = SGD(lr=0.00001, clipnorm=1.0)\n",
        "adam = Adam(lr=1e-4, clipnorm=1.0)\n",
        "\n",
        "model.compile(loss='sparse_categorical_crossentropy',\n",
        "              optimizer=adam,\n",
        "              metrics=['accuracy'])\n",
        "history = model.fit(x_train, y_train,\n",
        "                    batch_size=128, epochs=10,verbose=1)\n",
        "#                     validation_data=(X_val, Y_val),\n",
        "#                    callbacks=[TensorBoard(log_dir='logs',\n",
        "#                                           histogram_freq=1,\n",
        "#                                           update_freq='epoch')])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From C:\\Users\\prisarkar\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "Epoch 1/10\n",
            "24222/24222 [==============================] - 64s 3ms/sample - loss: 2.2375 - acc: 0.2056\n",
            "Epoch 2/10\n",
            "24222/24222 [==============================] - 63s 3ms/sample - loss: 1.7788 - acc: 0.3768\n",
            "Epoch 3/10\n",
            "24222/24222 [==============================] - 63s 3ms/sample - loss: 1.3502 - acc: 0.5399\n",
            "Epoch 4/10\n",
            "24222/24222 [==============================] - 63s 3ms/sample - loss: 1.0942 - acc: 0.6257\n",
            "Epoch 5/10\n",
            "24222/24222 [==============================] - 63s 3ms/sample - loss: 0.9249 - acc: 0.6828\n",
            "Epoch 6/10\n",
            "24222/24222 [==============================] - 62s 3ms/sample - loss: 0.8114 - acc: 0.7250\n",
            "Epoch 7/10\n",
            "24222/24222 [==============================] - 62s 3ms/sample - loss: 0.7243 - acc: 0.7653\n",
            "Epoch 8/10\n",
            "24222/24222 [==============================] - 62s 3ms/sample - loss: 0.6210 - acc: 0.7977\n",
            "Epoch 9/10\n",
            "24222/24222 [==============================] - 62s 3ms/sample - loss: 0.5540 - acc: 0.8229\n",
            "Epoch 10/10\n",
            "24222/24222 [==============================] - 63s 3ms/sample - loss: 0.4986 - acc: 0.8450\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vtQZPofpNc1C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#model.save(os.path.join(model_path, 'rnn.model'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "geBfOnSrNc1I",
        "colab_type": "code",
        "colab": {},
        "outputId": "b6e7c0ee-4908-4756-b4b6-3d0d4eb8e825"
      },
      "source": [
        "import pyaudio\n",
        "import wave\n",
        "\n",
        "CHUNK = 1024\n",
        "FORMAT = pyaudio.paInt16\n",
        "CHANNELS = 1\n",
        "RATE = 16000\n",
        "RECORD_SECONDS = 2\n",
        "WAVE_OUTPUT_FILENAME = output_file\n",
        "\n",
        "p = pyaudio.PyAudio()\n",
        "\n",
        "stream = p.open(format=FORMAT,\n",
        "                channels=CHANNELS,\n",
        "                rate=RATE,\n",
        "                input=True,\n",
        "                frames_per_buffer=CHUNK)\n",
        "\n",
        "print(\"* recording\")\n",
        "\n",
        "frames = []\n",
        "\n",
        "for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):\n",
        "    data = stream.read(CHUNK)\n",
        "    frames.append(data)\n",
        "\n",
        "print(\"* done recording\")\n",
        "\n",
        "stream.stop_stream()\n",
        "stream.close()\n",
        "p.terminate()\n",
        "\n",
        "wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')\n",
        "wf.setnchannels(CHANNELS)\n",
        "wf.setsampwidth(p.get_sample_size(FORMAT))\n",
        "wf.setframerate(RATE)\n",
        "wf.writeframes(b''.join(frames))\n",
        "wf.close()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "* recording\n",
            "* done recording\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X3Oa7b5nNc1N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "samples,sample_rate  = torchaudio.load(output_file)\n",
        "samples = pad_audio(samples)\n",
        "if len(samples) > 16000:\n",
        "    n_samples = chop_audio(samples)\n",
        "else: n_samples = [samples]\n",
        "for samples in n_samples:\n",
        "    resampled=torchaudio.transforms.Resample(sample_rate, new_sample_rate)(samples)\n",
        "    mfcc=torchaudio.transforms.MFCC()(resampled))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GimEJePNc1T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "imgs = np.array(mfcc)\n",
        "imgs = imgs.reshape(tuple([1]+list(imgs.shape)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IArrSO5ENc1Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predicts = model.predict(imgs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bgUzl7q7Nc1f",
        "colab_type": "code",
        "colab": {},
        "outputId": "040d428f-6617-43f4-82f5-7fbe2dc5558c"
      },
      "source": [
        "label_index[np.argmax(predicts)]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'up'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J_jeLKF2Nc1k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}