BERT_detecting_depression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT detecting depression.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sm6eu60IvMaL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "a80e7bd1-2b0f-48c7-b58f-02e9337711e9"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UzskX6uFv5SF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        },
        "outputId": "21df2bef-3d50-4f35-f5c2-cd61127a1753"
      },
      "source": [
        "!pip install chart_studio"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting chart_studio\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ca/ce/330794a6b6ca4b9182c38fc69dd2a9cbff60fd49421cb8648ee5fee352dc/chart_studio-1.1.0-py3-none-any.whl (64kB)\n",
            "\r\u001b[K     |█████                           | 10kB 21.5MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 20kB 3.2MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 30kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 40kB 4.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 51kB 3.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 61kB 4.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 71kB 3.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from chart_studio) (1.3.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from chart_studio) (2.23.0)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from chart_studio) (4.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from chart_studio) (1.12.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (2.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->chart_studio) (3.0.4)\n",
            "Installing collected packages: chart-studio\n",
            "Successfully installed chart-studio-1.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N0qRYYsfv8KP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0bcfe8f5-3859-426a-83f8-ac1e90def90a"
      },
      "source": [
        "#importing the libraries\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "import os\n",
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import chart_studio.plotly as py\n",
        "import plotly.graph_objs as go\n",
        "import itertools\n",
        "from scipy import stats\n",
        "from ast import literal_eval\n",
        "import spacy\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rjo584Duv5Qo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim\n",
        "import json\n",
        "import pickle\n",
        "\n",
        "import nltk\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import PorterStemmer\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.utils.vis_utils import plot_model\n",
        "import keras.utils\n",
        "from keras import utils as np_utils\n",
        "\n",
        "#Keras Tokenizer just replaces certain punctuation characters and splits on the remaining space character.\n",
        "#NLTK Tokenizer uses the Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ANvcgXiwL1i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#WINDOWS_SIZE = 10\n",
        "labels=['none','mild','moderate','moderately severe', 'severe']\n",
        "num_classes = len(labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SGGYZWytwTUE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#converting the transcript files into a dataframe\n",
        "def transcripts_to_dataframe(directory):\n",
        "    rows_list = []\n",
        "        \n",
        "    filenames = os.listdir(directory)\n",
        "    \n",
        "    if \".DS_Store\" in filenames:\n",
        "        filenames.remove(\".DS_Store\")\n",
        "        \n",
        "    for filename in filenames:\n",
        "        transcript_path = os.path.join(directory, filename)\n",
        "        transcript = pd.read_csv(transcript_path, sep='\\t')\n",
        "        m = re.search(\"(\\d{3})_TRANSCRIPT.csv\", filename)\n",
        "        if m:\n",
        "            person_id = m.group(1)\n",
        "            p = {}\n",
        "            question = \"\"\n",
        "            answer = \"\"\n",
        "            lines = len(transcript)\n",
        "            for i in range(0, lines):\n",
        "                row = transcript.iloc[i]\n",
        "                if (row[\"speaker\"] == \"Ellie\") or (i == lines - 1):\n",
        "                    p[\"personId\"] = person_id\n",
        "                    if \"(\" in str(question):\n",
        "                        question = question[question.index(\"(\") + 1:question.index(\")\")]\n",
        "                    p[\"question\"] = question\n",
        "                    p[\"answer\"] = answer\n",
        "                    if question != \"\":\n",
        "                        rows_list.append(p)\n",
        "                    p = {}\n",
        "                    answer = \"\"\n",
        "                    question = row[\"value\"]\n",
        "                else:\n",
        "                    answer = str(answer) + \" \" + str(row[\"value\"])\n",
        "\n",
        "    all_participants = pd.DataFrame(rows_list, columns=['personId', 'question', 'answer'])\n",
        "    all_participants.to_csv(directory + 'all.csv', sep=',')\n",
        "    print(\"File was created\")\n",
        "    return all_participants"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-r4MwZNxKDWk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0633221d-285f-4c4e-b84e-e73df7bb00f6"
      },
      "source": [
        "#loading the data\n",
        "data_path = \"/content/drive/My Drive/transcripts/\"\n",
        "all_participants = transcripts_to_dataframe(data_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "File was created\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-VxmgvfOzG5v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 788
        },
        "outputId": "930b7de9-7c09-4b53-e102-45b33c3abf45"
      },
      "source": [
        "all_participants.head(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>301</td>\n",
              "      <td>hi i'm ellie thanks for coming in today i was ...</td>\n",
              "      <td>thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>301</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>mmm k</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>301</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>301</td>\n",
              "      <td>how are you doing today</td>\n",
              "      <td>i'm doing good thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>301</td>\n",
              "      <td>that's good</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>301</td>\n",
              "      <td>where are you from originally</td>\n",
              "      <td>i'm from los angeles</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>301</td>\n",
              "      <td>really me too</td>\n",
              "      <td>oh great</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>301</td>\n",
              "      <td>where do you live</td>\n",
              "      <td>i live in west los angeles the west side</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>301</td>\n",
              "      <td>nice</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>301</td>\n",
              "      <td>how do you like your living situation</td>\n",
              "      <td>it's alright i xxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>301</td>\n",
              "      <td>do you have roommates</td>\n",
              "      <td>no i live alone so</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>301</td>\n",
              "      <td>that's great</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>301</td>\n",
              "      <td>how do you like l_a</td>\n",
              "      <td>i love it i'm from here so i grew up here it'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>301</td>\n",
              "      <td>yeah</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>301</td>\n",
              "      <td>what are some things you really like about l_a</td>\n",
              "      <td>the weather um well the weather it's always g...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>301</td>\n",
              "      <td>&lt;laughter&gt;</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>301</td>\n",
              "      <td>mhm</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>301</td>\n",
              "      <td>what are some things you don't really like abo...</td>\n",
              "      <td>the traffic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>301</td>\n",
              "      <td>i see what you mean</td>\n",
              "      <td>the traffic is horrible well probably traffic...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>301</td>\n",
              "      <td>yeah</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   personId  ...                                             answer\n",
              "0       301  ...                                          thank you\n",
              "1       301  ...                                              mmm k\n",
              "2       301  ...                                                   \n",
              "3       301  ...                           i'm doing good thank you\n",
              "4       301  ...                                                   \n",
              "5       301  ...                               i'm from los angeles\n",
              "6       301  ...                                           oh great\n",
              "7       301  ...           i live in west los angeles the west side\n",
              "8       301  ...                                                   \n",
              "9       301  ...                                 it's alright i xxx\n",
              "10      301  ...                                 no i live alone so\n",
              "11      301  ...                                                   \n",
              "12      301  ...   i love it i'm from here so i grew up here it'...\n",
              "13      301  ...                                                   \n",
              "14      301  ...   the weather um well the weather it's always g...\n",
              "15      301  ...                                                   \n",
              "16      301  ...                                                   \n",
              "17      301  ...                                        the traffic\n",
              "18      301  ...   the traffic is horrible well probably traffic...\n",
              "19      301  ...                                                   \n",
              "\n",
              "[20 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ts4c4BeG0SoW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 745
        },
        "outputId": "8436e078-47e4-485b-b8b3-5227a1aa1135"
      },
      "source": [
        "#Selecting the data from the data from based on the questions asked  ['where', 'when', 'how', 'why', 'are', 'what', 'do', 'have', 'can', 'did', 'is', 'could', 'so', 'tell', 'who', 'has']\n",
        "interrogative = [\"where\", \"when\", \"how\",\"why\",\"are\",\"what\",\"do\",\"have\",\"can\",\"did\",\"is\", \"could\", \"so\", \"tell\", \"who\", \"has\"]\n",
        "rslt_df = all_participants[all_participants.question.str.contains('|'.join(interrogative),na=False)]\n",
        "rslt_df['answer'].replace('', np.nan, inplace=True)\n",
        "rslt_df.dropna(subset = [\"answer\"], inplace=True)\n",
        "rslt_df.reset_index(drop=True, inplace=True)\n",
        "rslt_df\n",
        "#rslt_df = rslt_df.to_csv(\"/content/drive/My Drive/rslt_df.csv\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py:6746: SettingWithCopyWarning:\n",
            "\n",
            "\n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning:\n",
            "\n",
            "\n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>301</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>mmm k</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>301</td>\n",
              "      <td>how are you doing today</td>\n",
              "      <td>i'm doing good thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>301</td>\n",
              "      <td>where are you from originally</td>\n",
              "      <td>i'm from los angeles</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>301</td>\n",
              "      <td>where do you live</td>\n",
              "      <td>i live in west los angeles the west side</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>301</td>\n",
              "      <td>how do you like your living situation</td>\n",
              "      <td>it's alright i xxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>452</th>\n",
              "      <td>307</td>\n",
              "      <td>what's one of your most memorable experiences</td>\n",
              "      <td>[laughter] that's a tough question um probabl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>453</th>\n",
              "      <td>307</td>\n",
              "      <td>awesome</td>\n",
              "      <td>yeah</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>454</th>\n",
              "      <td>307</td>\n",
              "      <td>how would your best friend describe you</td>\n",
              "      <td>bubbly  fun  maybe moody you never know what ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>455</th>\n",
              "      <td>307</td>\n",
              "      <td>when was the last time you felt really happy</td>\n",
              "      <td>huh  really happy oh when i was in mexico ove...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>456</th>\n",
              "      <td>307</td>\n",
              "      <td>that sounds like a great situation</td>\n",
              "      <td>yeah it was cool</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>457 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    personId  ...                                             answer\n",
              "0        301  ...                                              mmm k\n",
              "1        301  ...                           i'm doing good thank you\n",
              "2        301  ...                               i'm from los angeles\n",
              "3        301  ...           i live in west los angeles the west side\n",
              "4        301  ...                                 it's alright i xxx\n",
              "..       ...  ...                                                ...\n",
              "452      307  ...   [laughter] that's a tough question um probabl...\n",
              "453      307  ...                                               yeah\n",
              "454      307  ...   bubbly  fun  maybe moody you never know what ...\n",
              "455      307  ...   huh  really happy oh when i was in mexico ove...\n",
              "456      307  ...                                   yeah it was cool\n",
              "\n",
              "[457 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bv2EyOPL0cah",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "outputId": "5d210322-de17-4fbc-96cd-5de76caba4c7"
      },
      "source": [
        "rslt_df.sample(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>300</td>\n",
              "      <td>how do you like l_a</td>\n",
              "      <td>i love it</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>335</th>\n",
              "      <td>309</td>\n",
              "      <td>what do you do now</td>\n",
              "      <td>um currently &lt;curr&gt; uh currently a student</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>229</th>\n",
              "      <td>304</td>\n",
              "      <td>how do you know them</td>\n",
              "      <td>um he's my boyfriend i've been knowing for wh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>222</th>\n",
              "      <td>304</td>\n",
              "      <td>how do you know them</td>\n",
              "      <td>um through high school</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223</th>\n",
              "      <td>304</td>\n",
              "      <td>how close are you to them</td>\n",
              "      <td>we're pretty close we try to talk as often as...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>434</th>\n",
              "      <td>307</td>\n",
              "      <td>what's your dream job</td>\n",
              "      <td>dang it that is a tough one how 'bout just ge...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>395</th>\n",
              "      <td>308</td>\n",
              "      <td>do you feel that way often</td>\n",
              "      <td>yes it eh yes that happens often</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>300</td>\n",
              "      <td>is there anything you regret</td>\n",
              "      <td>i would say going to college right after high...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>302</td>\n",
              "      <td>what do you enjoy about traveling</td>\n",
              "      <td>meeting new people different locations scener...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>225</th>\n",
              "      <td>304</td>\n",
              "      <td>when was the last time that happened</td>\n",
              "      <td>when what happened i was just</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    personId  ...                                             answer\n",
              "85       300  ...                                          i love it\n",
              "335      309  ...         um currently <curr> uh currently a student\n",
              "229      304  ...   um he's my boyfriend i've been knowing for wh...\n",
              "222      304  ...                            um through high school \n",
              "223      304  ...   we're pretty close we try to talk as often as...\n",
              "434      307  ...   dang it that is a tough one how 'bout just ge...\n",
              "395      308  ...                   yes it eh yes that happens often\n",
              "108      300  ...   i would say going to college right after high...\n",
              "72       302  ...   meeting new people different locations scener...\n",
              "225      304  ...                      when what happened i was just\n",
              "\n",
              "[10 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6wytW6DL4V2f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "bf2d1e6e-f5ce-4f23-ab73-d02ea9754359"
      },
      "source": [
        "rslt_df['question']= rslt_df.apply(lambda row: \"{0}?\".format(row.question), axis=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:\n",
            "\n",
            "\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GY0qQ2S4bzc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "outputId": "f77f9b86-bfbc-48e0-82c5-3a43a3b1e53e"
      },
      "source": [
        "rslt_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>301</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>mmm k</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>301</td>\n",
              "      <td>how are you doing today?</td>\n",
              "      <td>i'm doing good thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>301</td>\n",
              "      <td>where are you from originally?</td>\n",
              "      <td>i'm from los angeles</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>301</td>\n",
              "      <td>where do you live?</td>\n",
              "      <td>i live in west los angeles the west side</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>301</td>\n",
              "      <td>how do you like your living situation?</td>\n",
              "      <td>it's alright i xxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>452</th>\n",
              "      <td>307</td>\n",
              "      <td>what's one of your most memorable experiences ?</td>\n",
              "      <td>[laughter] that's a tough question um probabl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>453</th>\n",
              "      <td>307</td>\n",
              "      <td>awesome?</td>\n",
              "      <td>yeah</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>454</th>\n",
              "      <td>307</td>\n",
              "      <td>how would your best friend describe you?</td>\n",
              "      <td>bubbly  fun  maybe moody you never know what ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>455</th>\n",
              "      <td>307</td>\n",
              "      <td>when was the last time you felt really happy?</td>\n",
              "      <td>huh  really happy oh when i was in mexico ove...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>456</th>\n",
              "      <td>307</td>\n",
              "      <td>that sounds like a great situation?</td>\n",
              "      <td>yeah it was cool</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>457 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    personId  ...                                             answer\n",
              "0        301  ...                                              mmm k\n",
              "1        301  ...                           i'm doing good thank you\n",
              "2        301  ...                               i'm from los angeles\n",
              "3        301  ...           i live in west los angeles the west side\n",
              "4        301  ...                                 it's alright i xxx\n",
              "..       ...  ...                                                ...\n",
              "452      307  ...   [laughter] that's a tough question um probabl...\n",
              "453      307  ...                                               yeah\n",
              "454      307  ...   bubbly  fun  maybe moody you never know what ...\n",
              "455      307  ...   huh  really happy oh when i was in mexico ove...\n",
              "456      307  ...                                   yeah it was cool\n",
              "\n",
              "[457 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lSlreunn4b25",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 813
        },
        "outputId": "07bcf0f7-6de3-459d-d7c3-57441bb638f9"
      },
      "source": [
        "rslt_df['statements'] = rslt_df[['question', 'answer']].agg(':'.join, axis=1)\n",
        "rslt_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:\n",
            "\n",
            "\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>statements</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>301</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>mmm k</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>301</td>\n",
              "      <td>how are you doing today?</td>\n",
              "      <td>i'm doing good thank you</td>\n",
              "      <td>how are you doing today?: i'm doing good thank...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>301</td>\n",
              "      <td>where are you from originally?</td>\n",
              "      <td>i'm from los angeles</td>\n",
              "      <td>where are you from originally?: i'm from los a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>301</td>\n",
              "      <td>where do you live?</td>\n",
              "      <td>i live in west los angeles the west side</td>\n",
              "      <td>where do you live?: i live in west los angeles...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>301</td>\n",
              "      <td>how do you like your living situation?</td>\n",
              "      <td>it's alright i xxx</td>\n",
              "      <td>how do you like your living situation?: it's a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>452</th>\n",
              "      <td>307</td>\n",
              "      <td>what's one of your most memorable experiences ?</td>\n",
              "      <td>[laughter] that's a tough question um probabl...</td>\n",
              "      <td>what's one of your most memorable experiences ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>453</th>\n",
              "      <td>307</td>\n",
              "      <td>awesome?</td>\n",
              "      <td>yeah</td>\n",
              "      <td>awesome?: yeah</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>454</th>\n",
              "      <td>307</td>\n",
              "      <td>how would your best friend describe you?</td>\n",
              "      <td>bubbly  fun  maybe moody you never know what ...</td>\n",
              "      <td>how would your best friend describe you?: bubb...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>455</th>\n",
              "      <td>307</td>\n",
              "      <td>when was the last time you felt really happy?</td>\n",
              "      <td>huh  really happy oh when i was in mexico ove...</td>\n",
              "      <td>when was the last time you felt really happy?:...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>456</th>\n",
              "      <td>307</td>\n",
              "      <td>that sounds like a great situation?</td>\n",
              "      <td>yeah it was cool</td>\n",
              "      <td>that sounds like a great situation?: yeah it w...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>457 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    personId  ...                                         statements\n",
              "0        301  ...  think of me as a friend i don't judge i can't ...\n",
              "1        301  ...  how are you doing today?: i'm doing good thank...\n",
              "2        301  ...  where are you from originally?: i'm from los a...\n",
              "3        301  ...  where do you live?: i live in west los angeles...\n",
              "4        301  ...  how do you like your living situation?: it's a...\n",
              "..       ...  ...                                                ...\n",
              "452      307  ...  what's one of your most memorable experiences ...\n",
              "453      307  ...                                     awesome?: yeah\n",
              "454      307  ...  how would your best friend describe you?: bubb...\n",
              "455      307  ...  when was the last time you felt really happy?:...\n",
              "456      307  ...  that sounds like a great situation?: yeah it w...\n",
              "\n",
              "[457 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nSVK5Kl4b5y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_avec_dataset_file(path,score_column):\n",
        "    ds = pd.read_csv(path, sep=',')\n",
        "    ds['level'] = pd.cut(ds[score_column], bins=[-1,0,5,10,15,25], labels=[0,1,2,3,4])  #cut function used to segregate array into bins 5 levels - 'none','mild','moderate','moderately severe', 'severe'\n",
        "    ds['PHQ8_Score'] = ds[score_column]\n",
        "    ds['cat_level'] = keras.utils.to_categorical(ds['level'], num_classes).tolist() #categorical levels \n",
        "    ds = ds[['Participant_ID', 'level', 'cat_level', 'PHQ8_Score']]\n",
        "    ds = ds.astype({\"Participant_ID\": int, \"level\": int, 'PHQ8_Score': int})\n",
        "    return ds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1N92cmfC4nCk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "23ad7017-e080-411e-f896-cd0d30ffba92"
      },
      "source": [
        "train = load_avec_dataset_file('/content/drive/My Drive/Depression_detect/train_split_Depression_AVEC2017 (1).csv','PHQ8_Score')\n",
        "dev = load_avec_dataset_file('/content/drive/My Drive/Depression_detect/dev_split_Depression_AVEC2017.csv','PHQ8_Score')\n",
        "test = load_avec_dataset_file('/content/drive/My Drive/Depression_detect/full_test_split.csv','PHQ8_Score')\n",
        "print(\"Size: train= {}, dev= {}, test= {}\".format(len(train), len(dev), len(test)))\n",
        "train.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Size: train= 107, dev= 35, test= 47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>level</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>PHQ8_Score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>303</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>304</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>305</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>310</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>312</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Participant_ID  level                  cat_level  PHQ8_Score\n",
              "0             303      0  [1.0, 0.0, 0.0, 0.0, 0.0]           0\n",
              "1             304      2  [0.0, 0.0, 1.0, 0.0, 0.0]           6\n",
              "2             305      2  [0.0, 0.0, 1.0, 0.0, 0.0]           7\n",
              "3             310      1  [0.0, 1.0, 0.0, 0.0, 0.0]           4\n",
              "4             312      1  [0.0, 1.0, 0.0, 0.0, 0.0]           2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVlRmDwk4nFL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7359c16a-6bd1-43d9-d1dc-d53cd3b7c374"
      },
      "source": [
        "ds_total = pd.concat([train,dev,test],sort=True)\n",
        "total_phq8 = len(ds_total)\n",
        "print(\"Total size = {}\".format(total_phq8))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total size = 189\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_6uStIYg4zcD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "17968f02-5f44-43ce-ce5c-f9e99e007842"
      },
      "source": [
        "ds_total"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>PHQ8_Score</th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>level</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>303</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>6</td>\n",
              "      <td>304</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>7</td>\n",
              "      <td>305</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>310</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>312</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>0</td>\n",
              "      <td>467</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>3</td>\n",
              "      <td>469</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>3</td>\n",
              "      <td>470</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>1</td>\n",
              "      <td>480</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>7</td>\n",
              "      <td>481</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>189 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    PHQ8_Score  Participant_ID                  cat_level  level\n",
              "0            0             303  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "1            6             304  [0.0, 0.0, 1.0, 0.0, 0.0]      2\n",
              "2            7             305  [0.0, 0.0, 1.0, 0.0, 0.0]      2\n",
              "3            4             310  [0.0, 1.0, 0.0, 0.0, 0.0]      1\n",
              "4            2             312  [0.0, 1.0, 0.0, 0.0, 0.0]      1\n",
              "..         ...             ...                        ...    ...\n",
              "42           0             467  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "43           3             469  [0.0, 1.0, 0.0, 0.0, 0.0]      1\n",
              "44           3             470  [0.0, 1.0, 0.0, 0.0, 0.0]      1\n",
              "45           1             480  [0.0, 1.0, 0.0, 0.0, 0.0]      1\n",
              "46           7             481  [0.0, 0.0, 1.0, 0.0, 0.0]      2\n",
              "\n",
              "[189 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jl57Ly8V42es",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c88acee1-df09-4863-ae8e-2961427b7c27"
      },
      "source": [
        "len(ds_total)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "189"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fk2YWsMy45Vr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_by_phq_level(ds):\n",
        "    none_ds = ds[ds['level']==0]\n",
        "    mild_ds = ds[ds['level']==1]\n",
        "    moderate_ds = ds[ds['level']==2]\n",
        "    moderate_severe_ds = ds[ds['level']==3]\n",
        "    severe_ds = ds[ds['level']==4]\n",
        "    return (none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lv8yWV8i46nZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "d5c207f1-4104-4dc8-fa6b-f3f8b9e64306"
      },
      "source": [
        "none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds = split_by_phq_level(ds_total)\n",
        "print(\"Quantity per none_ds: {}, mild_ds: {}, moderate_ds {}, moderate_severe_ds: {}, severe_ds {}\".format(len(none_ds), len(mild_ds), len(moderate_ds), len(moderate_severe_ds), len(severe_ds)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Quantity per none_ds: 26, mild_ds: 70, moderate_ds 47, moderate_severe_ds: 24, severe_ds 22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oIWjg0D44825",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "749098b1-5d6f-46a1-b54b-989e08627f35"
      },
      "source": [
        "b_none_ds = ds_total[ds_total['level']==0]\n",
        "b_mild_ds = ds_total[ds_total['level']==1].sample(26)\n",
        "b_moderate_ds = ds_total[ds_total['level']==2].sample(26)\n",
        "b_moderate_severe_ds = ds_total[ds_total['level']==3]\n",
        "b_severe_ds = ds_total[ds_total['level']==4]\n",
        "\n",
        "ds_total_b = pd.concat([b_none_ds, b_mild_ds, b_moderate_ds, b_moderate_severe_ds, b_severe_ds])\n",
        "ds_total_b"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>PHQ8_Score</th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>level</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>303</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>0</td>\n",
              "      <td>363</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>0</td>\n",
              "      <td>364</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>0</td>\n",
              "      <td>366</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>0</td>\n",
              "      <td>369</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>18</td>\n",
              "      <td>332</td>\n",
              "      <td>[0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>18</td>\n",
              "      <td>354</td>\n",
              "      <td>[0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>17</td>\n",
              "      <td>405</td>\n",
              "      <td>[0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>17</td>\n",
              "      <td>453</td>\n",
              "      <td>[0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>17</td>\n",
              "      <td>461</td>\n",
              "      <td>[0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>124 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    PHQ8_Score  Participant_ID                  cat_level  level\n",
              "0            0             303  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "41           0             363  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "42           0             364  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "43           0             366  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "45           0             369  [1.0, 0.0, 0.0, 0.0, 0.0]      0\n",
              "..         ...             ...                        ...    ...\n",
              "9           18             332  [0.0, 0.0, 0.0, 0.0, 1.0]      4\n",
              "13          18             354  [0.0, 0.0, 0.0, 0.0, 1.0]      4\n",
              "23          17             405  [0.0, 0.0, 0.0, 0.0, 1.0]      4\n",
              "37          17             453  [0.0, 0.0, 0.0, 0.0, 1.0]      4\n",
              "38          17             461  [0.0, 0.0, 0.0, 0.0, 1.0]      4\n",
              "\n",
              "[124 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pAHO3uAJ4_c0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b8318657-b234-4153-a91d-e5f2d3fe36a7"
      },
      "source": [
        "len(ds_total_b)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "124"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LnqO-b9t5Fng",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "99c81b77-48ab-40f2-b66c-38f32ecee1f2"
      },
      "source": [
        "ds_lp = ds_total.merge(rslt_df,left_on=ds_total.Participant_ID.astype(int), right_on=rslt_df.personId.astype(int))\n",
        "ds_lp_b = ds_total_b.merge(rslt_df,left_on=ds_total_b.Participant_ID.astype(int).astype(int), right_on=rslt_df.personId.astype(int))\n",
        "len(ds_lp)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "457"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DkjLJj835Iv6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "64cf7079-46b1-4127-bbb5-201f039a620b"
      },
      "source": [
        "len(ds_lp)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "457"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJH0qGQc4nIU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def distribute_instances(ds, split_in = [70,14,16]):\n",
        "    ds_shuffled = ds.sample(frac=1)\n",
        "    none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds = split_by_phq_level(ds_shuffled)\n",
        "    eq_ds = dict()\n",
        "    prev_none = prev_mild = prev_moderate = prev_moderate_severe = prev_severe = 0\n",
        "    split = split_in\n",
        "    for p in split:\n",
        "        last_none = min(len(none_ds), prev_none + round(len(none_ds) * p/100))\n",
        "        last_mild = min(len(mild_ds), prev_mild + round(len(mild_ds) * p/100))\n",
        "        last_moderate = min(len(moderate_ds), prev_moderate + round(len(moderate_ds) * p/100))\n",
        "        last_moderate_severe = min(len(moderate_severe_ds), prev_moderate_severe + round(len(moderate_severe_ds) * p/100))\n",
        "        last_severe = min(len(severe_ds), prev_severe + round(len(severe_ds) * p/100))  \n",
        "        eq_ds['d'+str(p)] = pd.concat([none_ds[prev_none: last_none], mild_ds[prev_mild: last_mild], moderate_ds[prev_moderate: last_moderate], moderate_severe_ds[prev_moderate_severe: last_moderate_severe], severe_ds[prev_severe: last_severe]])\n",
        "        prev_none = last_none\n",
        "        prev_mild = last_mild\n",
        "        prev_moderate = last_moderate\n",
        "        prev_moderate_severe = last_moderate_severe\n",
        "        prev_severe = last_severe  \n",
        "    return (eq_ds['d70'], eq_ds['d14'], eq_ds['d16'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zBWAV_Dy5R_E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_lp, dev_lp, test_lp = distribute_instances(ds_lp)\n",
        "#train_lp_b, dev_lp_b, test_lp_b = distribute_instances(ds_lp_b)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FfI8D6625RCg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_lp = train_lp.sample(frac=1).reset_index(drop=True)\n",
        "test_lp=test_lp.sample(frac=1).reset_index(drop=True)\n",
        "dev_lp=dev_lp.sample(frac=1).reset_index(drop=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5xXmst5y5Vnw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "ed9d7de8-46de-4281-e9c8-1d72185c3bf2"
      },
      "source": [
        "train_lp.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>key_0</th>\n",
              "      <th>PHQ8_Score</th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>level</th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>statements</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>300</td>\n",
              "      <td>2</td>\n",
              "      <td>300</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "      <td>300</td>\n",
              "      <td>why?</td>\n",
              "      <td>um no specific reason i just don't travel a l...</td>\n",
              "      <td>why?: um no specific reason i just don't trave...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>301</td>\n",
              "      <td>3</td>\n",
              "      <td>301</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "      <td>301</td>\n",
              "      <td>when was the last time you felt really happy?</td>\n",
              "      <td>um when was the last time i really felt happy...</td>\n",
              "      <td>when was the last time you felt really happy?:...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>300</td>\n",
              "      <td>2</td>\n",
              "      <td>300</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "      <td>300</td>\n",
              "      <td>i'm sorry?</td>\n",
              "      <td>[laughter] it's alright</td>\n",
              "      <td>i'm sorry?: [laughter] it's alright</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>304</td>\n",
              "      <td>6</td>\n",
              "      <td>304</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "      <td>304</td>\n",
              "      <td>could you have done anything to avoid it ?</td>\n",
              "      <td>yes &lt;laughter&gt;</td>\n",
              "      <td>could you have done anything to avoid it ?: ye...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>304</td>\n",
              "      <td>6</td>\n",
              "      <td>304</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "      <td>304</td>\n",
              "      <td>tell me more about that?</td>\n",
              "      <td>i guess i've been hearing that lately how i b...</td>\n",
              "      <td>tell me more about that?: i guess i've been he...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>306</td>\n",
              "      <td>0</td>\n",
              "      <td>306</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>306</td>\n",
              "      <td>where are you from originally?</td>\n",
              "      <td>uh colorado</td>\n",
              "      <td>where are you from originally?: uh colorado</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>306</td>\n",
              "      <td>0</td>\n",
              "      <td>306</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>306</td>\n",
              "      <td>okay what's the best thing about being a parent?</td>\n",
              "      <td>um just seeing something that you're creating...</td>\n",
              "      <td>okay what's the best thing about being a paren...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>307</td>\n",
              "      <td>4</td>\n",
              "      <td>307</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "      <td>307</td>\n",
              "      <td>how do you know them?</td>\n",
              "      <td>one's from college and the other one we found...</td>\n",
              "      <td>how do you know them?: one's from college and ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>303</td>\n",
              "      <td>0</td>\n",
              "      <td>303</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "      <td>303</td>\n",
              "      <td>how are you at controlling your temper ?</td>\n",
              "      <td>i'm pretty much good because see by me being ...</td>\n",
              "      <td>how are you at controlling your temper ?: i'm ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>300</td>\n",
              "      <td>2</td>\n",
              "      <td>300</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>1</td>\n",
              "      <td>300</td>\n",
              "      <td>how did you feel in that moment?</td>\n",
              "      <td>uh frustrated</td>\n",
              "      <td>how did you feel in that moment?: uh frustrated</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   key_0  ...                                         statements\n",
              "0    300  ...  why?: um no specific reason i just don't trave...\n",
              "1    301  ...  when was the last time you felt really happy?:...\n",
              "2    300  ...                i'm sorry?: [laughter] it's alright\n",
              "3    304  ...  could you have done anything to avoid it ?: ye...\n",
              "4    304  ...  tell me more about that?: i guess i've been he...\n",
              "5    306  ...        where are you from originally?: uh colorado\n",
              "6    306  ...  okay what's the best thing about being a paren...\n",
              "7    307  ...  how do you know them?: one's from college and ...\n",
              "8    303  ...  how are you at controlling your temper ?: i'm ...\n",
              "9    300  ...    how did you feel in that moment?: uh frustrated\n",
              "\n",
              "[10 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1UJdfiV4MJ4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-2XGmb1g4nLU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import Dense, Input\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "import tensorflow_hub as hub\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZxKAFROsDmti",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vmugEZX-DreK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "f97a1599-1750-4b31-a6f3-a9d30e0ca85e"
      },
      "source": [
        "!pip install sentencepiece\n",
        "import tokenization"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/2c/8df20f3ac6c22ac224fff307ebc102818206c53fc454ecd37d8ac2060df5/sentencepiece-0.1.86-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\r\u001b[K     |▎                               | 10kB 22.4MB/s eta 0:00:01\r\u001b[K     |▋                               | 20kB 3.0MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 4.1MB/s eta 0:00:01\r\u001b[K     |█▎                              | 40kB 2.9MB/s eta 0:00:01\r\u001b[K     |█▋                              | 51kB 3.2MB/s eta 0:00:01\r\u001b[K     |██                              | 61kB 3.8MB/s eta 0:00:01\r\u001b[K     |██▏                             | 71kB 4.1MB/s eta 0:00:01\r\u001b[K     |██▌                             | 81kB 4.3MB/s eta 0:00:01\r\u001b[K     |██▉                             | 92kB 4.8MB/s eta 0:00:01\r\u001b[K     |███▏                            | 102kB 4.6MB/s eta 0:00:01\r\u001b[K     |███▌                            | 112kB 4.6MB/s eta 0:00:01\r\u001b[K     |███▉                            | 122kB 4.6MB/s eta 0:00:01\r\u001b[K     |████                            | 133kB 4.6MB/s eta 0:00:01\r\u001b[K     |████▍                           | 143kB 4.6MB/s eta 0:00:01\r\u001b[K     |████▊                           | 153kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████                           | 163kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 174kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 184kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████                          | 194kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 204kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 215kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████                         | 225kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 235kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 245kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 256kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 266kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 276kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 286kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 296kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 307kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 317kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████                      | 327kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 337kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 348kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████                     | 358kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 368kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 378kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████                    | 389kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 399kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 409kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 419kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 430kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 440kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 450kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 460kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 471kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 481kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 491kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 501kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 512kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████                | 522kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 532kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 542kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 552kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 563kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 573kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 583kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 593kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 604kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 614kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 624kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 634kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 645kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 655kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 665kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 675kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 686kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 696kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 706kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 716kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 727kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 737kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 747kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 757kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 768kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 778kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 788kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 798kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 808kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 819kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 829kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 839kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 849kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 860kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 870kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 880kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 890kB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 901kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 911kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 921kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 931kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 942kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 952kB 4.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 962kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 972kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 983kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 993kB 4.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.0MB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.0MB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.0MB 4.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.0MB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.0MB 4.6MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.86\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8sXnbNlKETSX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def bert_encode(texts, tokenizer, max_len=512):\n",
        "    all_tokens = []\n",
        "    all_masks = []\n",
        "    all_segments = []\n",
        "    \n",
        "    for text in texts:\n",
        "        text = tokenizer.tokenize(text)\n",
        "            \n",
        "        text = text[:max_len-2]\n",
        "        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n",
        "        pad_len = max_len - len(input_sequence)\n",
        "        \n",
        "        tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n",
        "        tokens += [0] * pad_len\n",
        "        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n",
        "        segment_ids = [0] * max_len\n",
        "        \n",
        "        all_tokens.append(tokens)\n",
        "        all_masks.append(pad_masks)\n",
        "        all_segments.append(segment_ids)\n",
        "    \n",
        "    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ae55YHX3Eau2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.regularizers import l2\n",
        "def build_model(bert_layer, max_len=512):\n",
        "    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n",
        "    input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n",
        "    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n",
        "\n",
        "    _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n",
        "    clf_output = sequence_output[:, 0, :]\n",
        "    hidden1=Dense(64, kernel_regularizer=l2(0.03), bias_regularizer=l2(0.03) ,activation='relu')(clf_output)\n",
        "    out = Dense(5 ,activation='softmax')(hidden1)\n",
        "    \n",
        "    model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n",
        "    model.compile(Adam(lr=2e-6), loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ye5pU4mkEeS9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "74848077-1723-455b-aa1b-26b02f8fc109"
      },
      "source": [
        "%%time\n",
        "module_url = \"https://tfhub.dev/tensorflow/bert_en_cased_L-24_H-1024_A-16/2\"\n",
        "bert_layer = hub.KerasLayer(module_url, trainable=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 18.6 s, sys: 3.92 s, total: 22.5 s\n",
            "Wall time: 33.2 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fYIAZB9xF6CL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
        "tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OjtN19BQF--a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_input = bert_encode(train_lp.answer.values, tokenizer, max_len=160)\n",
        "dev_input = bert_encode(dev_lp.answer.values, tokenizer, max_len=160)\n",
        "test_input = bert_encode(test_lp.answer.values, tokenizer, max_len=160)\n",
        "\n",
        "train_labels = train_lp.cat_level.values\n",
        "test_labels=test_lp.cat_level.values\n",
        "dev_labels=dev_lp.cat_level.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Er78v82Etazr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train_labels=np.array(train_labels)\n",
        "#dev_labels=np.array(dev_labels)\n",
        "#test_labels=np.array(test_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HhStUybz1FoY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "45a680a7-bda7-4d9b-dbbc-0970105f76e1"
      },
      "source": [
        "print(train_labels[0])\n",
        "print(type(train_input))\n",
        "print(type(train_input[0]))\n",
        "print(type(train_labels))\n",
        "print(type(train_labels[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0, 0.0, 0.0]\n",
            "<class 'tuple'>\n",
            "<class 'numpy.ndarray'>\n",
            "<class 'numpy.ndarray'>\n",
            "<class 'list'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ct810jtXv_eA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "train_labels1=[]\n",
        "for i in train_labels:\n",
        "  train_labels1.append( np.array(i))\n",
        "\n",
        "train_labels=np.array(train_labels1)\n",
        "\n",
        "#print(train_labels)\n",
        "\n",
        "\n",
        "\n",
        "dev_labels1=[]\n",
        "for i in dev_labels:\n",
        "  dev_labels1.append( np.array(i))\n",
        "\n",
        "dev_labels=np.array(dev_labels1)\n",
        "\n",
        "#print(dev_labels)\n",
        "\n",
        "\n",
        "\n",
        "test_labels1=[]\n",
        "for i in test_labels:\n",
        "  test_labels1.append( np.array(i))\n",
        "\n",
        "test_labels=np.array(test_labels1)\n",
        "\n",
        "#print(train_labels)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oQPswqF5LMpt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9a361aa2-775d-487e-ec2e-3f8b24670d59"
      },
      "source": [
        "dev_input[0].size"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10080"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "44Ga4UOhLRY4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d4e834dd-6b01-4051-f3ab-cbb009d23e67"
      },
      "source": [
        "test_input[0].size"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "11520"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "klKhCOvNGgZL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445
        },
        "outputId": "2d6be42d-0a94-49a8-daac-c33c79fc61d2"
      },
      "source": [
        "model = build_model(bert_layer, max_len=160)\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_word_ids (InputLayer)     [(None, 160)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_mask (InputLayer)         [(None, 160)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "segment_ids (InputLayer)        [(None, 160)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "keras_layer (KerasLayer)        [(None, 1024), (None 333579265   input_word_ids[0][0]             \n",
            "                                                                 input_mask[0][0]                 \n",
            "                                                                 segment_ids[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_strided_slice (Tens [(None, 1024)]       0           keras_layer[0][1]                \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 64)           65600       tf_op_layer_strided_slice[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 5)            325         dense[0][0]                      \n",
            "==================================================================================================\n",
            "Total params: 333,645,190\n",
            "Trainable params: 333,645,189\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9QsG5xcKR2ug",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4f712efe-8874-45e6-fe9b-a603bee66bb0"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "leRHuphtXWRD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filepath=\"weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5\"\n",
        "checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\n",
        "callbacks_list = [checkpoint]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kf8fpIh3rbe-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "b0c13e2d-41de-4d1a-fab8-62a058b33ac3"
      },
      "source": [
        "train_input"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[  101, 15276,  1185, ...,     0,     0,     0],\n",
              "        [  101, 15276,  1165, ...,     0,     0,     0],\n",
              "        [  101,   164,  7053, ...,     0,     0,     0],\n",
              "        ...,\n",
              "        [  101,  1185,   102, ...,     0,     0,     0],\n",
              "        [  101, 14863,  2608, ...,     0,     0,     0],\n",
              "        [  101,  8147,   102, ...,     0,     0,     0]]),\n",
              " array([[1, 1, 1, ..., 0, 0, 0],\n",
              "        [1, 1, 1, ..., 0, 0, 0],\n",
              "        [1, 1, 1, ..., 0, 0, 0],\n",
              "        ...,\n",
              "        [1, 1, 1, ..., 0, 0, 0],\n",
              "        [1, 1, 1, ..., 0, 0, 0],\n",
              "        [1, 1, 1, ..., 0, 0, 0]]),\n",
              " array([[0, 0, 0, ..., 0, 0, 0],\n",
              "        [0, 0, 0, ..., 0, 0, 0],\n",
              "        [0, 0, 0, ..., 0, 0, 0],\n",
              "        ...,\n",
              "        [0, 0, 0, ..., 0, 0, 0],\n",
              "        [0, 0, 0, ..., 0, 0, 0],\n",
              "        [0, 0, 0, ..., 0, 0, 0]]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3-YGISfJemhV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 717
        },
        "outputId": "56b95682-2e12-439a-9cb4-4154af153c4c"
      },
      "source": [
        "\n",
        "train_history = model.fit(\n",
        "    train_input, train_labels,\n",
        "    validation_data=(dev_input,dev_labels),\n",
        "    batch_size=16,\n",
        "    epochs=10,\n",
        "    callbacks=callbacks_list\n",
        "\n",
        ")\n",
        "\n",
        "model.save('model.h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 5.3370 - accuracy: 0.1944\n",
            "Epoch 00001: val_accuracy improved from -inf to 0.23810, saving model to weights-improvement-01-0.24.hdf5\n",
            "20/20 [==============================] - 95s 5s/step - loss: 5.3370 - accuracy: 0.1944 - val_loss: 5.2948 - val_accuracy: 0.2381\n",
            "Epoch 2/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 5.1793 - accuracy: 0.3041\n",
            "Epoch 00002: val_accuracy improved from 0.23810 to 0.30159, saving model to weights-improvement-02-0.30.hdf5\n",
            "20/20 [==============================] - 94s 5s/step - loss: 5.1793 - accuracy: 0.3041 - val_loss: 5.2167 - val_accuracy: 0.3016\n",
            "Epoch 3/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 5.0777 - accuracy: 0.3542\n",
            "Epoch 00003: val_accuracy improved from 0.30159 to 0.34921, saving model to weights-improvement-03-0.35.hdf5\n",
            "20/20 [==============================] - 103s 5s/step - loss: 5.0777 - accuracy: 0.3542 - val_loss: 5.1829 - val_accuracy: 0.3492\n",
            "Epoch 4/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 5.0156 - accuracy: 0.4232\n",
            "Epoch 00004: val_accuracy improved from 0.34921 to 0.38095, saving model to weights-improvement-04-0.38.hdf5\n",
            "20/20 [==============================] - 95s 5s/step - loss: 5.0156 - accuracy: 0.4232 - val_loss: 5.1576 - val_accuracy: 0.3810\n",
            "Epoch 5/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 4.9889 - accuracy: 0.4326\n",
            "Epoch 00005: val_accuracy improved from 0.38095 to 0.39683, saving model to weights-improvement-05-0.40.hdf5\n",
            "20/20 [==============================] - 102s 5s/step - loss: 4.9889 - accuracy: 0.4326 - val_loss: 5.1434 - val_accuracy: 0.3968\n",
            "Epoch 6/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 4.9458 - accuracy: 0.4859\n",
            "Epoch 00006: val_accuracy did not improve from 0.39683\n",
            "20/20 [==============================] - 19s 962ms/step - loss: 4.9458 - accuracy: 0.4859 - val_loss: 5.1399 - val_accuracy: 0.3810\n",
            "Epoch 7/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 4.9240 - accuracy: 0.4514\n",
            "Epoch 00007: val_accuracy did not improve from 0.39683\n",
            "20/20 [==============================] - 19s 961ms/step - loss: 4.9240 - accuracy: 0.4514 - val_loss: 5.1380 - val_accuracy: 0.3810\n",
            "Epoch 8/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 4.8786 - accuracy: 0.5141\n",
            "Epoch 00008: val_accuracy improved from 0.39683 to 0.41270, saving model to weights-improvement-08-0.41.hdf5\n",
            "20/20 [==============================] - 93s 5s/step - loss: 4.8786 - accuracy: 0.5141 - val_loss: 5.1322 - val_accuracy: 0.4127\n",
            "Epoch 9/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 4.8225 - accuracy: 0.5580\n",
            "Epoch 00009: val_accuracy did not improve from 0.41270\n",
            "20/20 [==============================] - 19s 962ms/step - loss: 4.8225 - accuracy: 0.5580 - val_loss: 5.1198 - val_accuracy: 0.3968\n",
            "Epoch 10/10\n",
            "20/20 [==============================] - ETA: 0s - loss: 4.7599 - accuracy: 0.5925\n",
            "Epoch 00010: val_accuracy did not improve from 0.41270\n",
            "20/20 [==============================] - 19s 961ms/step - loss: 4.7599 - accuracy: 0.5925 - val_loss: 5.1271 - val_accuracy: 0.4127\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hc1qn-vdJ1ff",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2nMAeygjjui",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_pred = model.predict(test_input)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_ER0AMdn0tM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c202bf04-0ec1-4d64-b5ce-c28803cb2548"
      },
      "source": [
        "print(test_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.1841854  0.5474751  0.13118438 0.0835271  0.05362801]\n",
            " [0.16828129 0.5131467  0.14177005 0.11743931 0.05936272]\n",
            " [0.11258365 0.63645566 0.07817651 0.11157141 0.06121273]\n",
            " [0.20757212 0.48984453 0.16286723 0.07168245 0.06803367]\n",
            " [0.1183167  0.46095082 0.11481845 0.04219804 0.26371595]\n",
            " [0.09319683 0.40376857 0.22475472 0.09629522 0.18198471]\n",
            " [0.1412621  0.36729705 0.2530121  0.09817425 0.14025444]\n",
            " [0.16109219 0.3921619  0.21465878 0.0919468  0.14014027]\n",
            " [0.21660182 0.26918942 0.31962177 0.14736907 0.04721792]\n",
            " [0.19412087 0.35874987 0.24527386 0.11725454 0.08460087]\n",
            " [0.2213484  0.4210125  0.2065774  0.07026854 0.0807932 ]\n",
            " [0.18351904 0.2760552  0.25654805 0.14080471 0.14307302]\n",
            " [0.23391707 0.4997168  0.06421568 0.10570125 0.09644914]\n",
            " [0.2515562  0.46157137 0.07560249 0.0448434  0.16642652]\n",
            " [0.06503819 0.3301228  0.42821944 0.13811873 0.03850085]\n",
            " [0.20623566 0.4072631  0.20051245 0.13033529 0.05565356]\n",
            " [0.10055017 0.17977901 0.63769025 0.06486443 0.0171161 ]\n",
            " [0.27384928 0.29607663 0.22380874 0.1263311  0.07993424]\n",
            " [0.24238442 0.4451285  0.11162636 0.08120175 0.11965889]\n",
            " [0.19170126 0.18779394 0.38893548 0.174084   0.0574853 ]\n",
            " [0.16828129 0.5131467  0.14177005 0.11743931 0.05936272]\n",
            " [0.08059068 0.12048402 0.66780776 0.08248086 0.04863664]\n",
            " [0.37661093 0.3060956  0.20971817 0.07255985 0.03501541]\n",
            " [0.3021688  0.4689268  0.05503315 0.01282739 0.16104388]\n",
            " [0.13775352 0.4815268  0.1797395  0.11304405 0.08793606]\n",
            " [0.11860669 0.45844653 0.28699803 0.07199731 0.06395151]\n",
            " [0.49928132 0.17113386 0.13176961 0.04478112 0.1530341 ]\n",
            " [0.07886649 0.27949792 0.49426103 0.1077211  0.0396534 ]\n",
            " [0.19457796 0.40804303 0.24612385 0.08360401 0.06765116]\n",
            " [0.16308093 0.5239874  0.12408213 0.10476533 0.08408421]\n",
            " [0.3636253  0.09007405 0.27149808 0.1187605  0.15604216]\n",
            " [0.1438317  0.19774829 0.40052256 0.21554671 0.04235078]\n",
            " [0.0730885  0.2851025  0.46574813 0.10144209 0.07461876]\n",
            " [0.14833112 0.6687603  0.0371456  0.05952897 0.08623405]\n",
            " [0.19472727 0.2724432  0.30459324 0.1587205  0.0695158 ]\n",
            " [0.17710452 0.37565273 0.1094706  0.05137622 0.28639588]\n",
            " [0.09691147 0.5841352  0.10031334 0.06994907 0.14869098]\n",
            " [0.12316438 0.5316222  0.14581023 0.11417421 0.08522901]\n",
            " [0.09368641 0.529736   0.111329   0.11004686 0.15520185]\n",
            " [0.15446454 0.460522   0.2255252  0.06304511 0.09644314]\n",
            " [0.16388941 0.50983644 0.16613477 0.02740902 0.1327303 ]\n",
            " [0.20471562 0.42094645 0.16763143 0.12319741 0.08350908]\n",
            " [0.13127394 0.50176686 0.23236716 0.06487488 0.06971712]\n",
            " [0.1410862  0.546383   0.20525073 0.06587327 0.04140677]\n",
            " [0.12768175 0.47143373 0.11168234 0.09086782 0.1983343 ]\n",
            " [0.12350278 0.6124711  0.10928189 0.07584002 0.07890428]\n",
            " [0.06365635 0.74789184 0.05179843 0.07730412 0.05934927]\n",
            " [0.266079   0.3639287  0.17368233 0.09257983 0.10373014]\n",
            " [0.04879788 0.16616824 0.59705746 0.16301383 0.02496258]\n",
            " [0.12770018 0.65033394 0.06369095 0.11429498 0.04397998]\n",
            " [0.20580868 0.26770964 0.37888274 0.10979755 0.03780136]\n",
            " [0.08090936 0.46957365 0.19179888 0.16670014 0.09101797]\n",
            " [0.11443099 0.5026185  0.24714768 0.09546401 0.04033883]\n",
            " [0.34123686 0.24114287 0.2556327  0.11758665 0.04440091]\n",
            " [0.273821   0.36566076 0.17009969 0.03607345 0.15434505]\n",
            " [0.35709146 0.39222118 0.07870105 0.04831546 0.12367089]\n",
            " [0.16871339 0.40812728 0.1765685  0.05689451 0.18969633]\n",
            " [0.26046908 0.30519396 0.2035273  0.14576204 0.08504763]\n",
            " [0.13510065 0.4683414  0.30487195 0.06005868 0.03162723]\n",
            " [0.15237682 0.20186965 0.3551386  0.19316943 0.09744541]\n",
            " [0.34123686 0.24114287 0.2556327  0.11758665 0.04440091]\n",
            " [0.1335779  0.45171818 0.28606147 0.07604396 0.05259854]\n",
            " [0.16828129 0.5131467  0.14177005 0.11743931 0.05936272]\n",
            " [0.29115602 0.30835932 0.17487325 0.1311826  0.09442879]\n",
            " [0.1682813  0.5131476  0.14176978 0.11743864 0.05936272]\n",
            " [0.21805885 0.528578   0.12955223 0.0818914  0.04191945]\n",
            " [0.16254048 0.55868447 0.08635405 0.12743537 0.06498561]\n",
            " [0.27053693 0.36648613 0.24091986 0.07899319 0.04306402]\n",
            " [0.12270106 0.4405447  0.13080381 0.10696132 0.19898905]\n",
            " [0.11365496 0.47309867 0.20447066 0.09942062 0.10935509]\n",
            " [0.06422735 0.3269355  0.33581626 0.20023458 0.07278629]\n",
            " [0.16749698 0.41355532 0.24035738 0.10936387 0.06922647]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fCGaCgRtOSnB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "c3fa840c-df3a-46fa-aab8-a1013298be52"
      },
      "source": [
        "score = model.evaluate(test_input, test_labels, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:',score[1] )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 5.093750953674316\n",
            "Test accuracy: 0.3611111044883728\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cCHCsFDZASJk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}