Lyrics Generation with Deep Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "isolated-container",
   "metadata": {},
   "source": [
    "# Lyrics Generation with Deep Learning\n",
    "###### Serkan UYSAL - [Data Source](https://www.kaggle.com/tgdivy/poetry-foundation-poems)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "international-transmission",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lined-questionnaire",
   "metadata": {},
   "source": [
    "## Loading and preprocessing aşamaları"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "talented-touch",
   "metadata": {},
   "outputs": [],
   "source": [
    "translator = str.maketrans('', '', string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "measured-intranet",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Poem</th>\n",
       "      <th>Poet</th>\n",
       "      <th>Tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\\r\\r\\n                    Objects Used to Prop...</td>\n",
       "      <td>\\r\\r\\nDog bone, stapler,\\r\\r\\ncribbage board, ...</td>\n",
       "      <td>Michelle Menting</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\r\\r\\n                    The New Church\\r\\r\\n...</td>\n",
       "      <td>\\r\\r\\nThe old cupola glinted above the clouds,...</td>\n",
       "      <td>Lucia Cherciu</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\\r\\r\\n                    Look for Me\\r\\r\\n   ...</td>\n",
       "      <td>\\r\\r\\nLook for me under the hood\\r\\r\\nof that ...</td>\n",
       "      <td>Ted Kooser</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\r\\r\\n                    Wild Life\\r\\r\\n     ...</td>\n",
       "      <td>\\r\\r\\nBehind the silo, the Mother Rabbit\\r\\r\\n...</td>\n",
       "      <td>Grace Cavalieri</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\r\\r\\n                    Umbrella\\r\\r\\n      ...</td>\n",
       "      <td>\\r\\r\\nWhen I push your button\\r\\r\\nyou fly off...</td>\n",
       "      <td>Connie Wanek</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  \\r\\r\\n                    Objects Used to Prop...   \n",
       "1  \\r\\r\\n                    The New Church\\r\\r\\n...   \n",
       "2  \\r\\r\\n                    Look for Me\\r\\r\\n   ...   \n",
       "3  \\r\\r\\n                    Wild Life\\r\\r\\n     ...   \n",
       "4  \\r\\r\\n                    Umbrella\\r\\r\\n      ...   \n",
       "\n",
       "                                                Poem              Poet Tags  \n",
       "0  \\r\\r\\nDog bone, stapler,\\r\\r\\ncribbage board, ...  Michelle Menting  NaN  \n",
       "1  \\r\\r\\nThe old cupola glinted above the clouds,...     Lucia Cherciu  NaN  \n",
       "2  \\r\\r\\nLook for me under the hood\\r\\r\\nof that ...        Ted Kooser  NaN  \n",
       "3  \\r\\r\\nBehind the silo, the Mother Rabbit\\r\\r\\n...   Grace Cavalieri  NaN  \n",
       "4  \\r\\r\\nWhen I push your button\\r\\r\\nyou fly off...      Connie Wanek  NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdf = pd.read_csv('PoetryFoundationData.csv', quotechar='\"', index_col=0)\n",
    "pdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "composed-portland",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Poem</th>\n",
       "      <th>Poet</th>\n",
       "      <th>Tags</th>\n",
       "      <th>single_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\\r\\r\\n                    Objects Used to Prop...</td>\n",
       "      <td>\\r\\r\\nDog bone, stapler,\\r\\r\\ncribbage board, ...</td>\n",
       "      <td>Michelle Menting</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dog bone stapler \\n cribbage board garlic pres...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\r\\r\\n                    The New Church\\r\\r\\n...</td>\n",
       "      <td>\\r\\r\\nThe old cupola glinted above the clouds,...</td>\n",
       "      <td>Lucia Cherciu</td>\n",
       "      <td>NaN</td>\n",
       "      <td>the old cupola glinted above the clouds shone ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\\r\\r\\n                    Look for Me\\r\\r\\n   ...</td>\n",
       "      <td>\\r\\r\\nLook for me under the hood\\r\\r\\nof that ...</td>\n",
       "      <td>Ted Kooser</td>\n",
       "      <td>NaN</td>\n",
       "      <td>look for me under the hood \\n of that old chev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\r\\r\\n                    Wild Life\\r\\r\\n     ...</td>\n",
       "      <td>\\r\\r\\nBehind the silo, the Mother Rabbit\\r\\r\\n...</td>\n",
       "      <td>Grace Cavalieri</td>\n",
       "      <td>NaN</td>\n",
       "      <td>behind the silo the mother rabbit \\n hunches l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\r\\r\\n                    Umbrella\\r\\r\\n      ...</td>\n",
       "      <td>\\r\\r\\nWhen I push your button\\r\\r\\nyou fly off...</td>\n",
       "      <td>Connie Wanek</td>\n",
       "      <td>NaN</td>\n",
       "      <td>when i push your button \\n you fly off the han...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  \\r\\r\\n                    Objects Used to Prop...   \n",
       "1  \\r\\r\\n                    The New Church\\r\\r\\n...   \n",
       "2  \\r\\r\\n                    Look for Me\\r\\r\\n   ...   \n",
       "3  \\r\\r\\n                    Wild Life\\r\\r\\n     ...   \n",
       "4  \\r\\r\\n                    Umbrella\\r\\r\\n      ...   \n",
       "\n",
       "                                                Poem              Poet Tags  \\\n",
       "0  \\r\\r\\nDog bone, stapler,\\r\\r\\ncribbage board, ...  Michelle Menting  NaN   \n",
       "1  \\r\\r\\nThe old cupola glinted above the clouds,...     Lucia Cherciu  NaN   \n",
       "2  \\r\\r\\nLook for me under the hood\\r\\r\\nof that ...        Ted Kooser  NaN   \n",
       "3  \\r\\r\\nBehind the silo, the Mother Rabbit\\r\\r\\n...   Grace Cavalieri  NaN   \n",
       "4  \\r\\r\\nWhen I push your button\\r\\r\\nyou fly off...      Connie Wanek  NaN   \n",
       "\n",
       "                                         single_text  \n",
       "0  dog bone stapler \\n cribbage board garlic pres...  \n",
       "1  the old cupola glinted above the clouds shone ...  \n",
       "2  look for me under the hood \\n of that old chev...  \n",
       "3  behind the silo the mother rabbit \\n hunches l...  \n",
       "4  when i push your button \\n you fly off the han...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdf['single_text'] = pdf['Poem'].apply(lambda x: ' \\n '.join([l.lower().strip().translate(translator) for l in x.splitlines() if len(l)>0]))\n",
    "pdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "latin-kitty",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(pdf['single_text'])\n",
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "lovely-tucson",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dog bone stapler \\n cribbage board garlic pres...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the old cupola glinted above the clouds shone ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>look for me under the hood \\n of that old chev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>behind the silo the mother rabbit \\n hunches l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>when i push your button \\n you fly off the han...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         single_text\n",
       "0  dog bone stapler \\n cribbage board garlic pres...\n",
       "1  the old cupola glinted above the clouds shone ...\n",
       "2  look for me under the hood \\n of that old chev...\n",
       "3  behind the silo the mother rabbit \\n hunches l...\n",
       "4  when i push your button \\n you fly off the han..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "productive-stake",
   "metadata": {},
   "source": [
    "## RNN yapısı oluşturma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "turned-think",
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_FREQ = 7\n",
    "MIN_SEQ = 5\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "text_as_list = []\n",
    "freq = {}\n",
    "uncommon_words = ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "classical-basin",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_text(text):\n",
    "    global text_as_list\n",
    "    text_as_list += [w for w in text.split(' ') if w.strip() != '' or w =='\\n']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "spare-lawyer",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['single_text'].apply(extract_text)\n",
    "\n",
    "for w in text_as_list:\n",
    "    freq[w] = freq.get(w, 0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "quarterly-plate",
   "metadata": {},
   "outputs": [],
   "source": [
    "uncommon_words = set([key for key in freq.keys() if freq[key] < MIN_FREQ])\n",
    "words = sorted(set([key for key in freq.keys() if freq[key] >= MIN_FREQ]))\n",
    "\n",
    "num_words = len(words)\n",
    "\n",
    "word_indices = dict((w, i) for i, w in enumerate(words))\n",
    "indices_word = dict((i, w) for i, w in enumerate(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "loving-situation",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_seq = []\n",
    "end_seq_words = []\n",
    "\n",
    "for i in range(len(text_as_list) - MIN_SEQ):\n",
    "    end_slice = i + MIN_SEQ + 1\n",
    "    if len(set(text_as_list[i:end_slice]).intersection(uncommon_words))==0:\n",
    "        valid_seq.append(text_as_list[i: i+ MIN_SEQ])\n",
    "        end_seq_words.append(text_as_list[i + MIN_SEQ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cathedral-married",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid sequences of size 5: 2774707\n"
     ]
    }
   ],
   "source": [
    "print('Valid sequences of size {}: {}'.format(MIN_SEQ, len(valid_seq)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fifth-jacob",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split data\n",
    "X_train, X_test, y_train, y_test = train_test_split(valid_seq, end_seq_words, test_size=0.02, random_state=25) # why we always use 42 for random_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "earned-moses",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['do', 'feign', '\\n', 'could', 'once'], ['it', 'among', 'engraved', 'furrows', '\\n'], ['songs', 'torn', 'from', 'the', 'air']]\n"
     ]
    }
   ],
   "source": [
    "print(X_train[2:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "uniform-dimension",
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataGenerator\n",
    "def generator(sentece_list, next_word_list, batch_size):\n",
    "    index = 0\n",
    "    while True:\n",
    "        x = np.zeros((batch_size, MIN_SEQ), dtype=np.int32)\n",
    "        y = np.zeros((batch_size), dtype=np.int32)\n",
    "        \n",
    "        for i in range(batch_size):\n",
    "            for t, w in enumerate(sentece_list[index % len(sentece_list)]):\n",
    "                x[i, t] = word_indices[w]\n",
    "            \n",
    "            y[i] = word_indices[next_word_list[index % len(sentece_list)]]\n",
    "            index += 1\n",
    "        yield x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "essential-merchant",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(preds, temperature=1.0):\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.log(preds) / temperature\n",
    "    \n",
    "    exp_preds = np.exp(preds)\n",
    "    probas = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    \n",
    "    return np.argmax(probas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "norman-designation",
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_epoch_end(epoch, logs):\n",
    "    seed_index = np.random.randint(len(X_train+X_test))\n",
    "    seed = (X_train + X_test)[seed_index]\n",
    "    \n",
    "    for diversity in [0.3, 0.4, 0.5, 0.6, 0.7]:\n",
    "        sentence = seed\n",
    "        examples_file.write('----- Diversity:' + str(diversity) + '\\n')\n",
    "        examples_file.write('----- Generating with seed:\\n\"' + ' '.join(sentence) + '\"\\n')\n",
    "        examples_file.write(' '.join(sentence))\n",
    "        \n",
    "        for i in range(50):\n",
    "            x_pred = np.zeros((1, MIN_SEQ))\n",
    "            for t, word in enumerate(sentence):\n",
    "                x_pred[0, t] = word_indices[word]\n",
    "            \n",
    "            preds = model.predict(x_pred, verbose=0)[0]\n",
    "            next_index = sample(preds, diversity)\n",
    "            next_word = indices_word[next_index]\n",
    "            \n",
    "            sentence = sentence[1:]\n",
    "            sentence.append(next_word)\n",
    "            examples_file.write(\" \" + next_word)\n",
    "        \n",
    "        examples_file.write('\\n')\n",
    "        \n",
    "    examples_file.write('='*25+'\\n')\n",
    "    examples_file.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "atlantic-parallel",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_model():\n",
    "    return tf.keras.models.Sequential([\n",
    "        tf.keras.layers.Embedding(\n",
    "            input_dim=len(words),\n",
    "            output_dim=1024\n",
    "        ),\n",
    "        tf.keras.layers.Bidirectional(\n",
    "            tf.keras.layers.LSTM(128)\n",
    "        ),\n",
    "        tf.keras.layers.Dense(\n",
    "            len(words)\n",
    "        ),\n",
    "        tf.keras.layers.Activation('softmax')]\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "binding-korean",
   "metadata": {},
   "source": [
    "#### Model compile section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "composite-performer",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = generate_model()\n",
    "model.compile(\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    optimizer='adam',\n",
    "    metrics=['acc']\n",
    ")\n",
    "file_path = \"./checkpoints/LSTM_LYRICS-epoch{epoch:03d}-words%d-sequence%d-minfreq%d-\" \\\n",
    "           \"loss{loss:.4f}-acc{accuracy:.4f}-val_loss{val_loss:.4f}-val_acc{val_accuracy:.4f}\" % \\\n",
    "           (len(words), MIN_SEQ, MIN_FREQ)\n",
    "\n",
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=file_path,\n",
    "    monitor='val_accuracy',\n",
    "    save_best_only=True\n",
    ")\n",
    "\n",
    "print_callback = tf.keras.callbacks.LambdaCallback(\n",
    "    on_epoch_end=on_epoch_end\n",
    ")\n",
    "\n",
    "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='val-accuracy',\n",
    "    patience=25\n",
    ")\n",
    "\n",
    "callbacks_list = [checkpoint, print_callback, early_stopping]\n",
    "\n",
    "examples_file = open('zs_example.md', \"w\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "mysterious-taylor",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "  413/43355 [..............................] - ETA: 2:43:34 - loss: 7.7531 - acc: 0.0945"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-ae74eee3dabc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m hist = model.fit(\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mgenerator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_seq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1098\u001b[0m                 _r=1):\n\u001b[1;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1101\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    853\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 855\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    856\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    857\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2940\u001b[0m       (graph_function,\n\u001b[1;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 2942\u001b[0;31m     return graph_function._call_flat(\n\u001b[0m\u001b[1;32m   2943\u001b[0m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[1;32m   2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1916\u001b[0m         and executing_eagerly):\n\u001b[1;32m   1917\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1918\u001b[0;31m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m   1919\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    553\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    554\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 555\u001b[0;31m           outputs = execute.execute(\n\u001b[0m\u001b[1;32m    556\u001b[0m               \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    557\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     57\u001b[0m   \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hist = model.fit(\n",
    "    generator(X_train, y_train, BATCH_SIZE),\n",
    "    steps_per_epoch=int(len(valid_seq)/BATCH_SIZE + 1),\n",
    "    epochs=25,\n",
    "    callbacks=callbacks_list,\n",
    "    validation_data=generator(X_test, y_test, BATCH_SIZE),\n",
    "    validation_steps=int(len(y_train)/BATCH_SIZE + 1)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "variable-keyboard",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-21-97ecbf874269>:1: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.config.list_physical_devices('GPU')` instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.test.is_gpu_available(\n",
    "    cuda_only=False, min_cuda_compute_capability=None\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "stretch-violation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "neutral-massage",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prescription-liabilities",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "discrete-confidentiality",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "language": "python",
   "name": "python38564bit9b13fc8a8c5449f8ad604cc2782fb3a5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}