audio_classification_transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "audio_classification_transformer.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/facundodeza/transfomer-audio-classification/blob/master/audio_classification_transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQlNIs0pi-hX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# Code from the next sources were used\n",
        "#[4] A transformer chatbot tutorial with Tensorflow 2.0. Retrieved from https://medium.com/tensorflow/a-transformer-chatbot-tutorial-with-tensorflow-2-0-88bf59e66fe2\n",
        "#[6] https://www.kaggle.com/maxwell110/beginner-s-guide-to-audio-data-2\n",
        "#[7] https://www.kaggle.com/daisukelab/cnn-2d-basic-solution-powered-by-fast-ai\n",
        "\n",
        "# We load the requiere libraries\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "import os\n",
        "import git\n",
        "import librosa\n",
        "import matplotlib.pyplot as plt\n",
        "import gc\n",
        "\n",
        "from tqdm import tqdm, tqdm_notebook\n",
        "from sklearn.metrics import label_ranking_average_precision_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import zipfile\n",
        "\n",
        "tqdm.pandas()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xUqQSW-NGIXF",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ebyq6pJaFXyT",
        "outputId": "470a1a84-98fe-4f4f-b58d-58b4336ec7c8",
        "colab": {}
      },
      "source": [
        "#We load csv with files name. We will use for reading the data\n",
        "df = pd.read_csv('/home/facudeza/train_curated.csv')\n",
        "#df_noisy = pd.read_csv('../input/freesound-audio-tagging-2019/train_noisy.csv')\n",
        "\n",
        "#train=pd.concat([df, df_noisy], axis=0)\n",
        "\n",
        "test_df = pd.read_csv('/home/facudeza//sample_submission.csv')\n",
        "\n",
        "len(test_df)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3361"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fu1s4aUVFXyX",
        "colab": {}
      },
      "source": [
        "# We get the labels from the test data, we start from 1 because 0 is the file name\n",
        "# Also, test set only has 80 classes and want to predict according to that.\n",
        "# Of course, you could create \n",
        "# From here we will create Y target\n",
        "\n",
        "label_columns = test_df.columns[1:]\n",
        "label_mapping = dict((label, index) for index, label in enumerate(label_columns))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uTmsBU7CKu7p",
        "outputId": "911c334c-9fe4-4a3d-9962-a78de1de940f",
        "colab": {}
      },
      "source": [
        "## We can see that several targets are in same space. We need to reshape it into a hot encode format\n",
        "print(df['labels'][:10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0                     Bark\n",
            "1                 Raindrop\n",
            "2          Finger_snapping\n",
            "3                      Run\n",
            "4          Finger_snapping\n",
            "5               Whispering\n",
            "6    Acoustic_guitar,Strum\n",
            "7                   Hi-hat\n",
            "8                Bass_drum\n",
            "9           Crowd,Cheering\n",
            "Name: labels, dtype: object\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "H5K-aYlWFXyr",
        "colab": {}
      },
      "source": [
        "def split_and_label(rows_labels):\n",
        "    \n",
        "    row_labels_list = []\n",
        "    for row in rows_labels:\n",
        "        row_labels = row.split(',')\n",
        "        labels_array = np.zeros((80))\n",
        "        \n",
        "        for label in row_labels:\n",
        "            index = label_mapping[label]\n",
        "            labels_array[index] = 1\n",
        "        \n",
        "        row_labels_list.append(labels_array)\n",
        "    \n",
        "    return row_labels_list\n",
        "\n",
        "\n",
        "train_curated_labels = split_and_label(df['labels'])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_HMtSma0i-h1",
        "colab_type": "code",
        "outputId": "cdcf1f8d-cf12-42ab-abe3-f804bd08252b",
        "colab": {}
      },
      "source": [
        "for f in label_columns:\n",
        "    df[f] = 0.0\n",
        "    #train_noisy[f] = 0.0\n",
        "\n",
        "df[label_columns] = train_curated_labels\n",
        "#train_noisy[label_columns]   = train_noisy_labels\n",
        "\n",
        "df['num_labels'] = df[label_columns].sum(axis=1)\n",
        "#train_noisy['num_labels']   = train_noisy[label_columns].sum(axis=1)\n",
        "\n",
        "#train_curated['path'] = '../input/train_curated/'+train_curated['fname']\n",
        "#train_noisy  ['path'] = '../input/train_noisy/'+train_noisy['fname']\n",
        "\n",
        "df.head()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fname</th>\n",
              "      <th>labels</th>\n",
              "      <th>Accelerating_and_revving_and_vroom</th>\n",
              "      <th>Accordion</th>\n",
              "      <th>Acoustic_guitar</th>\n",
              "      <th>Applause</th>\n",
              "      <th>Bark</th>\n",
              "      <th>Bass_drum</th>\n",
              "      <th>Bass_guitar</th>\n",
              "      <th>Bathtub_(filling_or_washing)</th>\n",
              "      <th>...</th>\n",
              "      <th>Traffic_noise_and_roadway_noise</th>\n",
              "      <th>Trickle_and_dribble</th>\n",
              "      <th>Walk_and_footsteps</th>\n",
              "      <th>Water_tap_and_faucet</th>\n",
              "      <th>Waves_and_surf</th>\n",
              "      <th>Whispering</th>\n",
              "      <th>Writing</th>\n",
              "      <th>Yell</th>\n",
              "      <th>Zipper_(clothing)</th>\n",
              "      <th>num_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0006ae4e.wav</td>\n",
              "      <td>Bark</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0019ef41.wav</td>\n",
              "      <td>Raindrop</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>001ec0ad.wav</td>\n",
              "      <td>Finger_snapping</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0026c7cb.wav</td>\n",
              "      <td>Run</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0026f116.wav</td>\n",
              "      <td>Finger_snapping</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 83 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          fname           labels  Accelerating_and_revving_and_vroom  \\\n",
              "0  0006ae4e.wav             Bark                                 0.0   \n",
              "1  0019ef41.wav         Raindrop                                 0.0   \n",
              "2  001ec0ad.wav  Finger_snapping                                 0.0   \n",
              "3  0026c7cb.wav              Run                                 0.0   \n",
              "4  0026f116.wav  Finger_snapping                                 0.0   \n",
              "\n",
              "   Accordion  Acoustic_guitar  Applause  Bark  Bass_drum  Bass_guitar  \\\n",
              "0        0.0              0.0       0.0   1.0        0.0          0.0   \n",
              "1        0.0              0.0       0.0   0.0        0.0          0.0   \n",
              "2        0.0              0.0       0.0   0.0        0.0          0.0   \n",
              "3        0.0              0.0       0.0   0.0        0.0          0.0   \n",
              "4        0.0              0.0       0.0   0.0        0.0          0.0   \n",
              "\n",
              "   Bathtub_(filling_or_washing)  ...  Traffic_noise_and_roadway_noise  \\\n",
              "0                           0.0  ...                              0.0   \n",
              "1                           0.0  ...                              0.0   \n",
              "2                           0.0  ...                              0.0   \n",
              "3                           0.0  ...                              0.0   \n",
              "4                           0.0  ...                              0.0   \n",
              "\n",
              "   Trickle_and_dribble  Walk_and_footsteps  Water_tap_and_faucet  \\\n",
              "0                  0.0                 0.0                   0.0   \n",
              "1                  0.0                 0.0                   0.0   \n",
              "2                  0.0                 0.0                   0.0   \n",
              "3                  0.0                 0.0                   0.0   \n",
              "4                  0.0                 0.0                   0.0   \n",
              "\n",
              "   Waves_and_surf  Whispering  Writing  Yell  Zipper_(clothing)  num_labels  \n",
              "0             0.0         0.0      0.0   0.0                0.0         1.0  \n",
              "1             0.0         0.0      0.0   0.0                0.0         1.0  \n",
              "2             0.0         0.0      0.0   0.0                0.0         1.0  \n",
              "3             0.0         0.0      0.0   0.0                0.0         1.0  \n",
              "4             0.0         0.0      0.0   0.0                0.0         1.0  \n",
              "\n",
              "[5 rows x 83 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wigpoc-oFXyf",
        "colab": {}
      },
      "source": [
        "# unzipping our files\n",
        "\n",
        "\n",
        "with zipfile.ZipFile('/home/facudeza/train_curated.zip', 'r') as zip_ref:\n",
        "    zip_ref.extractall('/home/jupyter/Untitled Folder')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vJ0DebAQFXyo",
        "colab": {}
      },
      "source": [
        "#import zipfile\n",
        "#with zipfile.ZipFile('../input/freesound-audio-tagging-2019/train_noisy.zip', 'r') as zip_ref:\n",
        " #   zip_ref.extractall('../output/kaggle/working/train')\n",
        "\n",
        "    #\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_mJMOENuFXyk",
        "colab": {}
      },
      "source": [
        "## We define this parameters for our audio processing\n",
        "\n",
        "class conf:\n",
        "    # Preprocessing settings\n",
        "    sampling_rate = 44100\n",
        "    duration = 4\n",
        "    hop_length = 300\n",
        "    fmin = 20\n",
        "    fmax = sampling_rate // 2\n",
        "    n_mels = 128\n",
        "    n_fft = n_mels * 20\n",
        "    samples = sampling_rate * duration\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eneZDekCNtNF",
        "colab": {}
      },
      "source": [
        "train_path = '/home/jupyter/Untitled Folder'\n",
        "#train_noisy_path = '../input/train_noisy/'\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "m_6O_hIOFXzX",
        "colab": {}
      },
      "source": [
        "## We read the audio data, and trim or padded, if it is needed\n",
        "## This code was borrowed from https://www.kaggle.com/christofhenkel/keras-baseline-lstm-attention-5-fold\n",
        "\n",
        "def read_audio(conf, pathname, trim_long_data):\n",
        "    y, sr = librosa.load(pathname, sr=conf.sampling_rate)\n",
        "    # trim silence\n",
        "    if 0 < len(y): # workaround: 0 length causes error\n",
        "        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)\n",
        "    # make it unified length to conf.samples\n",
        "    if len(y) > conf.samples: # long enough\n",
        "        if trim_long_data:\n",
        "            y = y[0:0+conf.samples]\n",
        "    else: # pad blank\n",
        "        padding = conf.samples - len(y)    # add padding at both ends\n",
        "        offset = padding // 2\n",
        "        y = np.pad(y, (offset, conf.samples - len(y) - offset), 'constant')\n",
        "    return y\n",
        "\n",
        "\n",
        "## We convert our raw audio into a mel spectogram\n",
        "def audio_to_melspectrogram(conf, audio):\n",
        "    spectrogram = librosa.feature.melspectrogram(audio, \n",
        "                                                 sr=conf.sampling_rate,\n",
        "                                                 n_mels=conf.n_mels,\n",
        "                                                 hop_length=conf.hop_length,\n",
        "                                                 n_fft=conf.n_fft,\n",
        "                                                 fmin=conf.fmin,\n",
        "                                                 fmax=conf.fmax)\n",
        "    spectrogram = librosa.power_to_db(spectrogram)\n",
        "    spectrogram = spectrogram.astype(np.float32)\n",
        "    return spectrogram\n",
        "\n",
        "#This implement the previous two function for one file\n",
        "def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):\n",
        "    x = read_audio(conf, pathname, trim_long_data)\n",
        "    mels = audio_to_melspectrogram(conf, x)\n",
        "    if debug_display:\n",
        "        IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))\n",
        "        show_melspectrogram(conf, mels)\n",
        "    return mels\n",
        "\n",
        "#This generalizes for all the audio files you have\n",
        "def convert_wav_to_image(df, source):\n",
        "    X = []\n",
        "    for i, row in tqdm_notebook(df.iterrows()):\n",
        "        try:\n",
        "            x = read_as_melspectrogram(conf, source[0]+'/'+str(row.fname), trim_long_data=True)\n",
        "        except:\n",
        "            x = read_as_melspectrogram(conf, source[1]+'/'+str(row.fname), trim_long_data=True)\n",
        "\n",
        "        X.append(x.transpose())\n",
        "    return X"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4vAdpu4BFXza",
        "outputId": "7da8f902-8335-4a9e-a9b2-339d4b91363d",
        "colab": {
          "referenced_widgets": [
            "b8f8bc2431744d809fed238c4f8ef542",
            "991302236e84415ba1b4d067ae49c70b"
          ]
        }
      },
      "source": [
        "#We get all our spectrograms\n",
        "X = np.array(convert_wav_to_image(df, source=[train_path]))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:45: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "991302236e84415ba1b4d067ae49c70b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qX7R-Rtgi-iR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#np.savez('/home/jupyter/curated_subset.npz',X=X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HXdnwT_Ei-iU",
        "colab_type": "code",
        "outputId": "edc69954-46e4-43d2-8d6e-f934e21a9086",
        "colab": {}
      },
      "source": [
        "X=np.array(np.load('/home/jupyter/curated_subset.npz')['X'])\n",
        "X.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4970, 589, 128)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EIv5qx9Li-iY",
        "colab_type": "code",
        "outputId": "a841cecf-5db2-45a6-e6a9-881b5a3043f1",
        "colab": {}
      },
      "source": [
        "df.columns"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['fname', 'labels', 'Accelerating_and_revving_and_vroom', 'Accordion',\n",
              "       'Acoustic_guitar', 'Applause', 'Bark', 'Bass_drum', 'Bass_guitar',\n",
              "       'Bathtub_(filling_or_washing)', 'Bicycle_bell',\n",
              "       'Burping_and_eructation', 'Bus', 'Buzz', 'Car_passing_by', 'Cheering',\n",
              "       'Chewing_and_mastication', 'Child_speech_and_kid_speaking',\n",
              "       'Chink_and_clink', 'Chirp_and_tweet', 'Church_bell', 'Clapping',\n",
              "       'Computer_keyboard', 'Crackle', 'Cricket', 'Crowd',\n",
              "       'Cupboard_open_or_close', 'Cutlery_and_silverware',\n",
              "       'Dishes_and_pots_and_pans', 'Drawer_open_or_close', 'Drip',\n",
              "       'Electric_guitar', 'Fart', 'Female_singing',\n",
              "       'Female_speech_and_woman_speaking', 'Fill_(with_liquid)',\n",
              "       'Finger_snapping', 'Frying_(food)', 'Gasp', 'Glockenspiel', 'Gong',\n",
              "       'Gurgling', 'Harmonica', 'Hi-hat', 'Hiss', 'Keys_jangling', 'Knock',\n",
              "       'Male_singing', 'Male_speech_and_man_speaking', 'Marimba_and_xylophone',\n",
              "       'Mechanical_fan', 'Meow', 'Microwave_oven', 'Motorcycle', 'Printer',\n",
              "       'Purr', 'Race_car_and_auto_racing', 'Raindrop', 'Run', 'Scissors',\n",
              "       'Screaming', 'Shatter', 'Sigh', 'Sink_(filling_or_washing)',\n",
              "       'Skateboard', 'Slam', 'Sneeze', 'Squeak', 'Stream', 'Strum', 'Tap',\n",
              "       'Tick-tock', 'Toilet_flush', 'Traffic_noise_and_roadway_noise',\n",
              "       'Trickle_and_dribble', 'Walk_and_footsteps', 'Water_tap_and_faucet',\n",
              "       'Waves_and_surf', 'Whispering', 'Writing', 'Yell', 'Zipper_(clothing)',\n",
              "       'num_labels'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdDN4kIUi-ic",
        "colab_type": "code",
        "outputId": "dfd4e3c6-b2cc-44f8-ac3a-8df2e6a51df6",
        "colab": {}
      },
      "source": [
        "#df['number_labels'].mean()\n",
        "Y=np.array(df.drop(['fname', 'labels','num_labels'], axis=1))\n",
        "Y.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4970, 80)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KMPHUERZFXzg",
        "colab": {}
      },
      "source": [
        "def scaled_dot_product_attention(query, key, value, mask):\n",
        "  matmul_qk = tf.matmul(query, key, transpose_b=True)\n",
        "\n",
        "  depth = tf.cast(tf.shape(key)[-1], tf.float32)\n",
        "  logits = matmul_qk / tf.math.sqrt(depth)\n",
        "\n",
        "  # add the mask zero out padding tokens.\n",
        "  if mask is not None:\n",
        "    logits += (mask * -1e9)\n",
        "\n",
        "  attention_weights = tf.nn.softmax(logits, axis=-1)\n",
        "\n",
        "  return tf.matmul(attention_weights, value)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-F4_iB_KFXzj",
        "colab": {}
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "\n",
        "  def __init__(self, d_model, num_heads, name=\"multi_head_attention\"):\n",
        "    super(MultiHeadAttention, self).__init__(name=name)\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "\n",
        "    assert d_model % self.num_heads == 0\n",
        "\n",
        "    self.depth = d_model // self.num_heads\n",
        "\n",
        "    self.query_dense = tf.keras.layers.Dense(units=d_model)\n",
        "    self.key_dense = tf.keras.layers.Dense(units=d_model)\n",
        "    self.value_dense = tf.keras.layers.Dense(units=d_model)\n",
        "\n",
        "    self.dense = tf.keras.layers.Dense(units=d_model)\n",
        "\n",
        "  def split_heads(self, inputs, batch_size):\n",
        "    inputs = tf.reshape(\n",
        "        inputs, shape=(batch_size, -1, self.num_heads, self.depth))\n",
        "    return tf.transpose(inputs, perm=[0, 2, 1, 3])\n",
        "\n",
        "  def call(self, inputs):\n",
        "    query, key, value, mask = inputs['query'], inputs['key'], inputs[\n",
        "        'value'], inputs['mask']\n",
        "    batch_size = tf.shape(query)[0]\n",
        "\n",
        "    # linear layers\n",
        "    query = self.query_dense(query)\n",
        "    key = self.key_dense(key)\n",
        "    value = self.value_dense(value)\n",
        "\n",
        "    # split heads\n",
        "    query = self.split_heads(query, batch_size)\n",
        "    key = self.split_heads(key, batch_size)\n",
        "    value = self.split_heads(value, batch_size)\n",
        "\n",
        "    scaled_attention = scaled_dot_product_attention(query, key, value, mask)\n",
        "\n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
        "\n",
        "    concat_attention = tf.reshape(scaled_attention,\n",
        "                                  (batch_size, -1, self.d_model))\n",
        "\n",
        "    outputs = self.dense(concat_attention)\n",
        "\n",
        "    return outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xsfTXzebFXzn",
        "colab": {}
      },
      "source": [
        "class PositionalEncoding(tf.keras.layers.Layer):\n",
        "\n",
        "  def __init__(self, position, d_model):\n",
        "    super(PositionalEncoding, self).__init__()\n",
        "    self.pos_encoding = self.positional_encoding(position, d_model)\n",
        "\n",
        "  def get_angles(self, position, i, d_model):\n",
        "    angles = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))\n",
        "    return position * angles\n",
        "\n",
        "  def positional_encoding(self, position, d_model):\n",
        "    angle_rads = self.get_angles(\n",
        "        position=tf.range(position, dtype=tf.float32)[:, tf.newaxis],\n",
        "        i=tf.range(d_model, dtype=tf.float32)[tf.newaxis, :],\n",
        "        d_model=d_model)\n",
        "    # apply sin to even index in the array\n",
        "    sines = tf.math.sin(angle_rads[:, 0::2])\n",
        "    # apply cos to odd index in the array\n",
        "    cosines = tf.math.cos(angle_rads[:, 1::2])\n",
        "\n",
        "    pos_encoding = tf.concat([sines, cosines], axis=-1)\n",
        "    pos_encoding = pos_encoding[tf.newaxis, ...]\n",
        "    return tf.cast(pos_encoding, tf.float32)\n",
        "\n",
        "  def call(self, inputs):\n",
        "    return inputs + self.pos_encoding[:, :tf.shape(inputs)[1], :]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4pq5OKITFXzz",
        "colab": {}
      },
      "source": [
        "# This allows to the transformer to know where there is real data and where it is padded\n",
        "def create_padding_mask(seq):\n",
        "  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  \n",
        "  # add extra dimensions to add the padding\n",
        "  # to the attention logits.\n",
        "  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "U389WZgtFXzq",
        "colab": {}
      },
      "source": [
        "def encoder_layer(units, d_model, num_heads, dropout,name=\"encoder_layer\"):\n",
        "  inputs = tf.keras.Input(shape=(None,d_model ), name=\"inputs\")\n",
        "  padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
        "\n",
        "  attention = MultiHeadAttention(\n",
        "      d_model, num_heads, name=\"attention\")({\n",
        "          'query': inputs,\n",
        "          'key': inputs,\n",
        "          'value': inputs,\n",
        "          'mask': padding_mask\n",
        "      })\n",
        "  attention = tf.keras.layers.Dropout(rate=dropout)(attention)\n",
        "  attention = tf.keras.layers.LayerNormalization(\n",
        "      epsilon=1e-6)(inputs + attention)\n",
        "\n",
        "  outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention)\n",
        "  outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
        "  outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
        "  outputs = tf.keras.layers.LayerNormalization(\n",
        "      epsilon=1e-6)(attention + outputs)\n",
        "\n",
        "  return tf.keras.Model(\n",
        "      inputs=[inputs, padding_mask], outputs=outputs, name=name)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QPEbBpxqFXzt",
        "colab": {}
      },
      "source": [
        "def encoder(time_steps,\n",
        "            num_layers,\n",
        "            units,\n",
        "            d_model,\n",
        "            num_heads,\n",
        "            dropout,\n",
        "            projection,\n",
        "            name=\"encoder\"):\n",
        "  inputs = tf.keras.Input(shape=(None,d_model), name=\"inputs\")\n",
        "  padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
        "  \n",
        "  if projection=='linear':\n",
        "    ## We implement a linear projection based on Very Deep Self-Attention Networks for End-to-End Speech Recognition. Retrieved from https://arxiv.org/abs/1904.13377\n",
        "    projection=tf.keras.layers.Dense( d_model,use_bias=True, activation='linear')(inputs)\n",
        "    print('linear')\n",
        "  \n",
        "  else:\n",
        "    projection=tf.identity(inputs)\n",
        "    print('none')\n",
        "   \n",
        "  projection *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
        "  projection = PositionalEncoding(time_steps, d_model)(projection)\n",
        "\n",
        "  outputs = tf.keras.layers.Dropout(rate=dropout)(projection)\n",
        "\n",
        "  for i in range(num_layers):\n",
        "    outputs = encoder_layer(\n",
        "        units=units,\n",
        "        d_model=d_model,\n",
        "        num_heads=num_heads,\n",
        "        dropout=dropout,\n",
        "        name=\"encoder_layer_{}\".format(i),\n",
        "    )([outputs, padding_mask])\n",
        " \n",
        " \n",
        "  \n",
        "\n",
        " \n",
        "  return tf.keras.Model(\n",
        "      inputs=[inputs, padding_mask], outputs=outputs, name=name)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "n9uLLTcPFXzw",
        "colab": {}
      },
      "source": [
        "def transformer(time_steps,\n",
        "                num_layers,\n",
        "                units,\n",
        "                d_model,\n",
        "                num_heads,\n",
        "                dropout,\n",
        "                output_size,\n",
        "                projection,\n",
        "                name=\"transformer\"):\n",
        "  inputs = tf.keras.Input(shape=(None,d_model), name=\"inputs\")\n",
        "  \n",
        "  \n",
        "  enc_padding_mask = tf.keras.layers.Lambda(\n",
        "      create_padding_mask, output_shape=(1, 1, None),\n",
        "      name='enc_padding_mask')(tf.dtypes.cast(\n",
        "          \n",
        "    #Like our input has a dimension of length X d_model but the masking is applied to a vector\n",
        "    # We get the sum for each row and result is a vector. So, if result is 0 it is because in that position was masked      \n",
        "    tf.math.reduce_sum(\n",
        "    inputs,\n",
        "    axis=2,\n",
        "    keepdims=False,\n",
        "    name=None\n",
        "), tf.int32))\n",
        "  \n",
        "\n",
        "  enc_outputs = encoder(\n",
        "      time_steps=time_steps,\n",
        "      num_layers=num_layers,\n",
        "      units=units,\n",
        "      d_model=d_model,\n",
        "      num_heads=num_heads,\n",
        "      dropout=dropout,\n",
        "      projection=projection,\n",
        "      name='encoder'\n",
        "  )(inputs=[inputs, enc_padding_mask])\n",
        "\n",
        "  #We reshape for feeding our FC in the next step\n",
        "  outputs=tf.reshape(enc_outputs,(-1,time_steps*d_model))\n",
        "  \n",
        "  #We predict our class\n",
        "  outputs = tf.keras.layers.Dense(units=output_size,use_bias=True,activation='softmax', name=\"outputs\")(outputs)\n",
        "\n",
        "  return tf.keras.Model(inputs=[inputs], outputs=outputs, name='audio_class')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OU4hWuxYFX0C",
        "colab": {}
      },
      "source": [
        "#We get our train and test set\n",
        "X_train,X_test, Y_train, Y_test =train_test_split(X,Y, test_size=0.2, random_state=27)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "oT948RgHgKMJ",
        "colab": {}
      },
      "source": [
        "projection=['linear','none']\n",
        "accuracy=[]\n",
        "proj_implemented=[]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "y2ifMZKEFX0J",
        "outputId": "6b38d307-bc5c-49ed-ad77-8df70d887c9b",
        "colab": {}
      },
      "source": [
        "for i in projection:\n",
        "  NUM_LAYERS = 2\n",
        "  D_MODEL = X.shape[2]\n",
        "  NUM_HEADS = 4\n",
        "  UNITS = 1024\n",
        "  DROPOUT = 0.1\n",
        "  TIME_STEPS= X.shape[1]\n",
        "  OUTPUT_SIZE=80\n",
        "  EPOCHS = 10\n",
        "  EXPERIMENTS=10\n",
        "\n",
        "  \n",
        "  for j in range(EXPERIMENTS):\n",
        "    \n",
        "    \n",
        "    model = transformer(time_steps=TIME_STEPS,\n",
        "      num_layers=NUM_LAYERS,\n",
        "      units=UNITS,\n",
        "      d_model=D_MODEL,\n",
        "      num_heads=NUM_HEADS,\n",
        "      dropout=DROPOUT,\n",
        "      output_size=OUTPUT_SIZE,  \n",
        "      projection=i  )\n",
        "    \n",
        "    model.compile(optimizer=tf.keras.optimizers.Adam(0.000001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "\n",
        "    history=model.fit(X_train,Y_train, epochs=EPOCHS, validation_data=(X_test, Y_test))\n",
        "\n",
        "    accuracy.append(max(history.history['val_accuracy']))\n",
        "      \n",
        "    proj_implemented.append(i)\n",
        "    \n",
        "    del model\n",
        "    \n",
        "    del history\n",
        "      \n",
        "   \n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 302s 76ms/sample - loss: 5.6303 - accuracy: 0.0128 - val_loss: 5.0906 - val_accuracy: 0.0191\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.1811 - accuracy: 0.0216 - val_loss: 4.9470 - val_accuracy: 0.0282\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.0594 - accuracy: 0.0324 - val_loss: 4.8501 - val_accuracy: 0.0292\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.9784 - accuracy: 0.0375 - val_loss: 4.7577 - val_accuracy: 0.0463\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.8880 - accuracy: 0.0440 - val_loss: 4.7025 - val_accuracy: 0.0553\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.8408 - accuracy: 0.0475 - val_loss: 4.6361 - val_accuracy: 0.0573\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.7758 - accuracy: 0.0508 - val_loss: 4.6141 - val_accuracy: 0.0614\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.7233 - accuracy: 0.0563 - val_loss: 4.5840 - val_accuracy: 0.0644\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.6834 - accuracy: 0.0606 - val_loss: 4.5367 - val_accuracy: 0.0694\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.6441 - accuracy: 0.0629 - val_loss: 4.5201 - val_accuracy: 0.0694\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 297s 75ms/sample - loss: 5.5263 - accuracy: 0.0194 - val_loss: 5.0452 - val_accuracy: 0.0211\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 5.1330 - accuracy: 0.0272 - val_loss: 4.8611 - val_accuracy: 0.0382\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.9739 - accuracy: 0.0327 - val_loss: 4.7324 - val_accuracy: 0.0453\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.8429 - accuracy: 0.0468 - val_loss: 4.6194 - val_accuracy: 0.0664\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.7419 - accuracy: 0.0558 - val_loss: 4.5508 - val_accuracy: 0.0755\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.6429 - accuracy: 0.0629 - val_loss: 4.4748 - val_accuracy: 0.0875\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.5857 - accuracy: 0.0752 - val_loss: 4.4244 - val_accuracy: 0.0926\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.5077 - accuracy: 0.0827 - val_loss: 4.3580 - val_accuracy: 0.1066\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.4449 - accuracy: 0.0880 - val_loss: 4.3383 - val_accuracy: 0.1137\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.3929 - accuracy: 0.0898 - val_loss: 4.2891 - val_accuracy: 0.1147\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 296s 74ms/sample - loss: 5.4429 - accuracy: 0.0169 - val_loss: 4.9802 - val_accuracy: 0.0241\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 5.1282 - accuracy: 0.0224 - val_loss: 4.8578 - val_accuracy: 0.0302\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 292s 74ms/sample - loss: 4.9912 - accuracy: 0.0289 - val_loss: 4.7547 - val_accuracy: 0.0302\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.9053 - accuracy: 0.0385 - val_loss: 4.6684 - val_accuracy: 0.0553\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 292s 74ms/sample - loss: 4.8133 - accuracy: 0.0485 - val_loss: 4.6032 - val_accuracy: 0.0654\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.7281 - accuracy: 0.0536 - val_loss: 4.5368 - val_accuracy: 0.0765\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 74ms/sample - loss: 4.6740 - accuracy: 0.0599 - val_loss: 4.4966 - val_accuracy: 0.0815\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.6088 - accuracy: 0.0616 - val_loss: 4.4538 - val_accuracy: 0.0855\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.5700 - accuracy: 0.0704 - val_loss: 4.4100 - val_accuracy: 0.0956\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 4.5034 - accuracy: 0.0800 - val_loss: 4.3725 - val_accuracy: 0.1016\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 298s 75ms/sample - loss: 5.4924 - accuracy: 0.0161 - val_loss: 5.0152 - val_accuracy: 0.0181\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 296s 75ms/sample - loss: 5.1087 - accuracy: 0.0194 - val_loss: 4.8535 - val_accuracy: 0.0312\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.9771 - accuracy: 0.0302 - val_loss: 4.7579 - val_accuracy: 0.0382\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 296s 75ms/sample - loss: 4.8719 - accuracy: 0.0385 - val_loss: 4.6770 - val_accuracy: 0.0443\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.8129 - accuracy: 0.0412 - val_loss: 4.6049 - val_accuracy: 0.0533\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.7232 - accuracy: 0.0490 - val_loss: 4.5778 - val_accuracy: 0.0614\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 300s 75ms/sample - loss: 4.6792 - accuracy: 0.0591 - val_loss: 4.5370 - val_accuracy: 0.0604\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 298s 75ms/sample - loss: 4.6227 - accuracy: 0.0584 - val_loss: 4.4902 - val_accuracy: 0.0714\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 300s 76ms/sample - loss: 4.5824 - accuracy: 0.0684 - val_loss: 4.4606 - val_accuracy: 0.0785\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 299s 75ms/sample - loss: 4.5399 - accuracy: 0.0777 - val_loss: 4.4242 - val_accuracy: 0.0724\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 301s 76ms/sample - loss: 5.4812 - accuracy: 0.0176 - val_loss: 4.9388 - val_accuracy: 0.0372\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 5.0533 - accuracy: 0.0297 - val_loss: 4.7304 - val_accuracy: 0.0483\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.8682 - accuracy: 0.0418 - val_loss: 4.5975 - val_accuracy: 0.0755\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.7253 - accuracy: 0.0531 - val_loss: 4.4825 - val_accuracy: 0.0936\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.6520 - accuracy: 0.0573 - val_loss: 4.3969 - val_accuracy: 0.1117\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.5491 - accuracy: 0.0744 - val_loss: 4.3312 - val_accuracy: 0.1117\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.4785 - accuracy: 0.0830 - val_loss: 4.2631 - val_accuracy: 0.1046\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.4003 - accuracy: 0.0943 - val_loss: 4.2189 - val_accuracy: 0.1197\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.3383 - accuracy: 0.0968 - val_loss: 4.1661 - val_accuracy: 0.1197\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.2884 - accuracy: 0.1051 - val_loss: 4.1431 - val_accuracy: 0.1268\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 297s 75ms/sample - loss: 5.4491 - accuracy: 0.0143 - val_loss: 5.0256 - val_accuracy: 0.0292\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 5.1424 - accuracy: 0.0181 - val_loss: 4.8690 - val_accuracy: 0.0392\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 5.0093 - accuracy: 0.0314 - val_loss: 4.7240 - val_accuracy: 0.0563\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.8984 - accuracy: 0.0385 - val_loss: 4.6303 - val_accuracy: 0.0704\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.8092 - accuracy: 0.0405 - val_loss: 4.5587 - val_accuracy: 0.0926\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.7093 - accuracy: 0.0551 - val_loss: 4.4954 - val_accuracy: 0.0905\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.6306 - accuracy: 0.0626 - val_loss: 4.4335 - val_accuracy: 0.1056\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.6047 - accuracy: 0.0709 - val_loss: 4.3596 - val_accuracy: 0.1147\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.5424 - accuracy: 0.0805 - val_loss: 4.3345 - val_accuracy: 0.1147\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.4796 - accuracy: 0.0785 - val_loss: 4.2957 - val_accuracy: 0.1147\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.4934 - accuracy: 0.0143 - val_loss: 4.9456 - val_accuracy: 0.0392\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 5.1174 - accuracy: 0.0287 - val_loss: 4.7585 - val_accuracy: 0.0473\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.9700 - accuracy: 0.0387 - val_loss: 4.6481 - val_accuracy: 0.0614\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.8317 - accuracy: 0.0473 - val_loss: 4.5772 - val_accuracy: 0.0815\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.7564 - accuracy: 0.0508 - val_loss: 4.5184 - val_accuracy: 0.0865\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.6837 - accuracy: 0.0644 - val_loss: 4.4781 - val_accuracy: 0.0915\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.6298 - accuracy: 0.0674 - val_loss: 4.4247 - val_accuracy: 0.0885\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.5670 - accuracy: 0.0772 - val_loss: 4.3966 - val_accuracy: 0.1026\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5163 - accuracy: 0.0762 - val_loss: 4.3724 - val_accuracy: 0.1137\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.4751 - accuracy: 0.0893 - val_loss: 4.3431 - val_accuracy: 0.1087\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.3396 - accuracy: 0.0189 - val_loss: 4.9212 - val_accuracy: 0.0282\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 292s 74ms/sample - loss: 5.0319 - accuracy: 0.0340 - val_loss: 4.7809 - val_accuracy: 0.0543\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.9038 - accuracy: 0.0392 - val_loss: 4.6995 - val_accuracy: 0.0604\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.8203 - accuracy: 0.0453 - val_loss: 4.6227 - val_accuracy: 0.0755\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.7643 - accuracy: 0.0541 - val_loss: 4.5747 - val_accuracy: 0.0755\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.6714 - accuracy: 0.0589 - val_loss: 4.5286 - val_accuracy: 0.0956\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 74ms/sample - loss: 4.6176 - accuracy: 0.0704 - val_loss: 4.4915 - val_accuracy: 0.0875\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5936 - accuracy: 0.0760 - val_loss: 4.4695 - val_accuracy: 0.1076\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5472 - accuracy: 0.0707 - val_loss: 4.4304 - val_accuracy: 0.1066\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.5030 - accuracy: 0.0762 - val_loss: 4.3948 - val_accuracy: 0.1117\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.4952 - accuracy: 0.0156 - val_loss: 5.0539 - val_accuracy: 0.0101\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 5.1140 - accuracy: 0.0216 - val_loss: 4.8734 - val_accuracy: 0.0201\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.9506 - accuracy: 0.0367 - val_loss: 4.7534 - val_accuracy: 0.0292\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.8376 - accuracy: 0.0415 - val_loss: 4.6688 - val_accuracy: 0.0402\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.7496 - accuracy: 0.0541 - val_loss: 4.5777 - val_accuracy: 0.0563\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.6835 - accuracy: 0.0596 - val_loss: 4.5254 - val_accuracy: 0.0584\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.6246 - accuracy: 0.0646 - val_loss: 4.4634 - val_accuracy: 0.0684\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5617 - accuracy: 0.0734 - val_loss: 4.4252 - val_accuracy: 0.0795\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5041 - accuracy: 0.0883 - val_loss: 4.3874 - val_accuracy: 0.0875\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.4604 - accuracy: 0.0822 - val_loss: 4.3480 - val_accuracy: 0.0956\n",
            "linear\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 5.3616 - accuracy: 0.0181 - val_loss: 4.9210 - val_accuracy: 0.0282\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 5.0339 - accuracy: 0.0282 - val_loss: 4.7894 - val_accuracy: 0.0563\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.9325 - accuracy: 0.0365 - val_loss: 4.7025 - val_accuracy: 0.0634\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.8225 - accuracy: 0.0435 - val_loss: 4.6314 - val_accuracy: 0.0734\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.7488 - accuracy: 0.0578 - val_loss: 4.5711 - val_accuracy: 0.0744\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.6875 - accuracy: 0.0576 - val_loss: 4.5350 - val_accuracy: 0.0885\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.6275 - accuracy: 0.0634 - val_loss: 4.5104 - val_accuracy: 0.0855\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5992 - accuracy: 0.0656 - val_loss: 4.4686 - val_accuracy: 0.0865\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5482 - accuracy: 0.0669 - val_loss: 4.4692 - val_accuracy: 0.0956\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.5097 - accuracy: 0.0797 - val_loss: 4.4443 - val_accuracy: 0.0885\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 5.5624 - accuracy: 0.0148 - val_loss: 5.0425 - val_accuracy: 0.0282\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 5.1575 - accuracy: 0.0287 - val_loss: 4.8649 - val_accuracy: 0.0372\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 286s 72ms/sample - loss: 5.0140 - accuracy: 0.0319 - val_loss: 4.7515 - val_accuracy: 0.0453\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 286s 72ms/sample - loss: 4.9145 - accuracy: 0.0455 - val_loss: 4.6798 - val_accuracy: 0.0553\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 286s 72ms/sample - loss: 4.8003 - accuracy: 0.0483 - val_loss: 4.6332 - val_accuracy: 0.0654\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.7224 - accuracy: 0.0599 - val_loss: 4.6031 - val_accuracy: 0.0734\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.6688 - accuracy: 0.0644 - val_loss: 4.5949 - val_accuracy: 0.0694\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 288s 73ms/sample - loss: 4.6204 - accuracy: 0.0619 - val_loss: 4.5522 - val_accuracy: 0.0785\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.5786 - accuracy: 0.0727 - val_loss: 4.5490 - val_accuracy: 0.0825\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.5002 - accuracy: 0.0807 - val_loss: 4.5452 - val_accuracy: 0.0815\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.4717 - accuracy: 0.0131 - val_loss: 5.1196 - val_accuracy: 0.0272\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 5.1340 - accuracy: 0.0317 - val_loss: 4.9346 - val_accuracy: 0.0322\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 5.0232 - accuracy: 0.0350 - val_loss: 4.8195 - val_accuracy: 0.0352\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 286s 72ms/sample - loss: 4.9106 - accuracy: 0.0402 - val_loss: 4.7336 - val_accuracy: 0.0493\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.8398 - accuracy: 0.0553 - val_loss: 4.6799 - val_accuracy: 0.0543\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 299s 75ms/sample - loss: 4.7719 - accuracy: 0.0589 - val_loss: 4.6402 - val_accuracy: 0.0594\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.7000 - accuracy: 0.0629 - val_loss: 4.6010 - val_accuracy: 0.0594\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.6475 - accuracy: 0.0659 - val_loss: 4.5680 - val_accuracy: 0.0684\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.6181 - accuracy: 0.0717 - val_loss: 4.5377 - val_accuracy: 0.0714\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.5618 - accuracy: 0.0845 - val_loss: 4.5376 - val_accuracy: 0.0724\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.5058 - accuracy: 0.0179 - val_loss: 4.9730 - val_accuracy: 0.0392\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 286s 72ms/sample - loss: 5.1405 - accuracy: 0.0241 - val_loss: 4.8195 - val_accuracy: 0.0463\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 5.0328 - accuracy: 0.0299 - val_loss: 4.7335 - val_accuracy: 0.0573\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.9313 - accuracy: 0.0410 - val_loss: 4.6870 - val_accuracy: 0.0624\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.8629 - accuracy: 0.0428 - val_loss: 4.6546 - val_accuracy: 0.0684\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.8133 - accuracy: 0.0448 - val_loss: 4.6005 - val_accuracy: 0.0664\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.7660 - accuracy: 0.0513 - val_loss: 4.5843 - val_accuracy: 0.0734\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.7077 - accuracy: 0.0596 - val_loss: 4.5660 - val_accuracy: 0.0865\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.6837 - accuracy: 0.0596 - val_loss: 4.5651 - val_accuracy: 0.0885\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 288s 73ms/sample - loss: 4.6445 - accuracy: 0.0609 - val_loss: 4.5496 - val_accuracy: 0.0875\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.4888 - accuracy: 0.0169 - val_loss: 5.0446 - val_accuracy: 0.0211\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.2358 - accuracy: 0.0194 - val_loss: 4.9270 - val_accuracy: 0.0302\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 5.1374 - accuracy: 0.0246 - val_loss: 4.8329 - val_accuracy: 0.0463\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.0494 - accuracy: 0.0294 - val_loss: 4.7605 - val_accuracy: 0.0433\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.9800 - accuracy: 0.0332 - val_loss: 4.7019 - val_accuracy: 0.0553\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.9218 - accuracy: 0.0372 - val_loss: 4.6646 - val_accuracy: 0.0553\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.8740 - accuracy: 0.0438 - val_loss: 4.6360 - val_accuracy: 0.0644\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.8259 - accuracy: 0.0463 - val_loss: 4.6116 - val_accuracy: 0.0624\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.7852 - accuracy: 0.0490 - val_loss: 4.5756 - val_accuracy: 0.0795\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.7535 - accuracy: 0.0548 - val_loss: 4.5654 - val_accuracy: 0.0755\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 5.5271 - accuracy: 0.0153 - val_loss: 5.0705 - val_accuracy: 0.0282\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 288s 73ms/sample - loss: 5.1386 - accuracy: 0.0277 - val_loss: 4.8508 - val_accuracy: 0.0443\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.9558 - accuracy: 0.0352 - val_loss: 4.7491 - val_accuracy: 0.0463\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.8160 - accuracy: 0.0538 - val_loss: 4.6407 - val_accuracy: 0.0553\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.7247 - accuracy: 0.0511 - val_loss: 4.5599 - val_accuracy: 0.0714\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.6195 - accuracy: 0.0646 - val_loss: 4.5240 - val_accuracy: 0.0734\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.5739 - accuracy: 0.0744 - val_loss: 4.4882 - val_accuracy: 0.0825\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.5235 - accuracy: 0.0868 - val_loss: 4.4483 - val_accuracy: 0.1006\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.4347 - accuracy: 0.0853 - val_loss: 4.4359 - val_accuracy: 0.1117\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.4064 - accuracy: 0.0903 - val_loss: 4.4109 - val_accuracy: 0.1147\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.6165 - accuracy: 0.0143 - val_loss: 5.1479 - val_accuracy: 0.0292\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 5.2479 - accuracy: 0.0249 - val_loss: 4.9810 - val_accuracy: 0.0312\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 5.0966 - accuracy: 0.0292 - val_loss: 4.8550 - val_accuracy: 0.0372\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.9666 - accuracy: 0.0390 - val_loss: 4.8077 - val_accuracy: 0.0463\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.8851 - accuracy: 0.0430 - val_loss: 4.7281 - val_accuracy: 0.0523\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.7893 - accuracy: 0.0480 - val_loss: 4.6833 - val_accuracy: 0.0594\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.7277 - accuracy: 0.0604 - val_loss: 4.6524 - val_accuracy: 0.0694\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.6906 - accuracy: 0.0599 - val_loss: 4.6238 - val_accuracy: 0.0684\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.6334 - accuracy: 0.0624 - val_loss: 4.6310 - val_accuracy: 0.0765\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.5989 - accuracy: 0.0762 - val_loss: 4.6100 - val_accuracy: 0.0845\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 299s 75ms/sample - loss: 5.5222 - accuracy: 0.0161 - val_loss: 5.0469 - val_accuracy: 0.0211\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.1618 - accuracy: 0.0216 - val_loss: 4.9170 - val_accuracy: 0.0392\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.0677 - accuracy: 0.0307 - val_loss: 4.8348 - val_accuracy: 0.0553\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.9706 - accuracy: 0.0410 - val_loss: 4.7686 - val_accuracy: 0.0513\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.8961 - accuracy: 0.0448 - val_loss: 4.7311 - val_accuracy: 0.0674\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.8683 - accuracy: 0.0453 - val_loss: 4.6990 - val_accuracy: 0.0614\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 74ms/sample - loss: 4.7815 - accuracy: 0.0604 - val_loss: 4.6549 - val_accuracy: 0.0765\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.7685 - accuracy: 0.0629 - val_loss: 4.6707 - val_accuracy: 0.0714\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.7088 - accuracy: 0.0644 - val_loss: 4.6684 - val_accuracy: 0.0744\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.6874 - accuracy: 0.0621 - val_loss: 4.6353 - val_accuracy: 0.0815\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 5.5137 - accuracy: 0.0176 - val_loss: 4.9698 - val_accuracy: 0.0372\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 5.1449 - accuracy: 0.0269 - val_loss: 4.7963 - val_accuracy: 0.0402\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 5.0345 - accuracy: 0.0357 - val_loss: 4.6660 - val_accuracy: 0.0563\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.9196 - accuracy: 0.0410 - val_loss: 4.5829 - val_accuracy: 0.0664\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.8321 - accuracy: 0.0531 - val_loss: 4.5116 - val_accuracy: 0.0654\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.7735 - accuracy: 0.0543 - val_loss: 4.4590 - val_accuracy: 0.0765\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.6955 - accuracy: 0.0636 - val_loss: 4.4135 - val_accuracy: 0.0885\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 4.6135 - accuracy: 0.0719 - val_loss: 4.3764 - val_accuracy: 0.1006\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.6257 - accuracy: 0.0727 - val_loss: 4.3671 - val_accuracy: 0.0905\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 287s 72ms/sample - loss: 4.5705 - accuracy: 0.0772 - val_loss: 4.3261 - val_accuracy: 0.0926\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.4401 - accuracy: 0.0179 - val_loss: 4.9788 - val_accuracy: 0.0171\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 288s 72ms/sample - loss: 5.1410 - accuracy: 0.0264 - val_loss: 4.8214 - val_accuracy: 0.0493\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 5.0008 - accuracy: 0.0327 - val_loss: 4.7167 - val_accuracy: 0.0563\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.8878 - accuracy: 0.0448 - val_loss: 4.6204 - val_accuracy: 0.0644\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.7931 - accuracy: 0.0523 - val_loss: 4.5435 - val_accuracy: 0.0724\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.7050 - accuracy: 0.0581 - val_loss: 4.4858 - val_accuracy: 0.0734\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.6301 - accuracy: 0.0734 - val_loss: 4.4381 - val_accuracy: 0.0845\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5730 - accuracy: 0.0702 - val_loss: 4.3913 - val_accuracy: 0.0956\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.5120 - accuracy: 0.0835 - val_loss: 4.3689 - val_accuracy: 0.0946\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.4743 - accuracy: 0.0850 - val_loss: 4.3243 - val_accuracy: 0.1117\n",
            "none\n",
            "Train on 3976 samples, validate on 994 samples\n",
            "Epoch 1/10\n",
            "3976/3976 [==============================] - 295s 74ms/sample - loss: 5.4950 - accuracy: 0.0138 - val_loss: 5.0329 - val_accuracy: 0.0161\n",
            "Epoch 2/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 5.1755 - accuracy: 0.0206 - val_loss: 4.9031 - val_accuracy: 0.0211\n",
            "Epoch 3/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 5.0622 - accuracy: 0.0262 - val_loss: 4.8342 - val_accuracy: 0.0362\n",
            "Epoch 4/10\n",
            "3976/3976 [==============================] - 291s 73ms/sample - loss: 4.9766 - accuracy: 0.0352 - val_loss: 4.7734 - val_accuracy: 0.0443\n",
            "Epoch 5/10\n",
            "3976/3976 [==============================] - 294s 74ms/sample - loss: 4.9167 - accuracy: 0.0377 - val_loss: 4.7395 - val_accuracy: 0.0513\n",
            "Epoch 6/10\n",
            "3976/3976 [==============================] - 293s 74ms/sample - loss: 4.8620 - accuracy: 0.0423 - val_loss: 4.7049 - val_accuracy: 0.0473\n",
            "Epoch 7/10\n",
            "3976/3976 [==============================] - 292s 73ms/sample - loss: 4.8099 - accuracy: 0.0423 - val_loss: 4.7016 - val_accuracy: 0.0523\n",
            "Epoch 8/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.7677 - accuracy: 0.0483 - val_loss: 4.6765 - val_accuracy: 0.0423\n",
            "Epoch 9/10\n",
            "3976/3976 [==============================] - 290s 73ms/sample - loss: 4.7491 - accuracy: 0.0521 - val_loss: 4.6953 - val_accuracy: 0.0453\n",
            "Epoch 10/10\n",
            "3976/3976 [==============================] - 289s 73ms/sample - loss: 4.7333 - accuracy: 0.0485 - val_loss: 4.7062 - val_accuracy: 0.0483\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "chxvhHsAFX02",
        "colab": {}
      },
      "source": [
        "accuracy=pd.DataFrame(accuracy, columns=['accuracy'])\n",
        "proj_implemented=pd.DataFrame(proj_implemented, columns=['projection'])\n",
        "results=pd.concat([accuracy,proj_implemented],axis=1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4LJeC1iwccFA",
        "outputId": "021aea92-09b4-43c5-b9b0-6494948a8026",
        "colab": {}
      },
      "source": [
        "results.groupby('projection').mean()\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>projection</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>linear</th>\n",
              "      <td>0.102213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>none</th>\n",
              "      <td>0.086821</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            accuracy\n",
              "projection          \n",
              "linear      0.102213\n",
              "none        0.086821"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bA0Y-5O0dUtz",
        "outputId": "9ffd8d73-97aa-4eb1-e885-9067a6046086",
        "colab": {}
      },
      "source": [
        "results.groupby('projection').std()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>projection</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>linear</th>\n",
              "      <td>0.017835</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>none</th>\n",
              "      <td>0.018532</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            accuracy\n",
              "projection          \n",
              "linear      0.017835\n",
              "none        0.018532"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H1cYVHkci-jJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1BXYtzMni-jM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}