train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import keras\n",
    "from keras.models import Sequential \n",
    "from keras.layers import Dense, LSTM, Dropout, Activation\n",
    "import string\n",
    "import sys\n",
    "import argparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_length = 60\n",
    "batch_size = 512\n",
    "dropout = 0.2\n",
    "layer_dim = 512\n",
    "nb_epochs = 30\n",
    "layer_count = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "characters = list(string.printable)\n",
    "characters.remove('\\x0b')\n",
    "characters.remove('\\x0c')\n",
    "\n",
    "vocab_size = len(characters)\n",
    "char_to_int = dict((c, i) for i, c in enumerate(characters))\n",
    "int_to_char = dict((i, c) for i, c in enumerate(characters))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Train():\n",
    "    text_train = open(\"dataset_train.txt\").read()\n",
    "    text_train_len = len(text_train)\n",
    "    text_val = open(\"dataset_val.txt\").read()\n",
    "    text_val_len = len(text_val)\n",
    "    \n",
    "    train_batch_count = (text_train_len - seq_length) // batch_size\n",
    "    val_batch_count = (text_val_len - seq_length) // batch_size\n",
    "    \n",
    "    def batch_generator(text, count):\n",
    "        while True:\n",
    "            for batch_idx in range(count):\n",
    "                X = np.zeros((batch_size, seq_length, vocab_size))\n",
    "                y = np.zeros((batch_size, vocab_size))\n",
    "        \n",
    "                batch_offset = batch_size * batch_idx\n",
    "                for sample_idx in range(batch_size):\n",
    "                    sample_start = batch_offset + sample_idx\n",
    "                    for s in range(seq_length):\n",
    "                        X[sample_idx, s, char_to_int[text[sample_start + s]]] = 1\n",
    "                    y[sample_idx, char_to_int[text[sample_start + s + 1]]] = 1\n",
    "        \n",
    "                yield X, y\n",
    "    \n",
    "    from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "    filepath = \"./model\" \n",
    "    \n",
    "    checkpoint = ModelCheckpoint(\n",
    "        filepath,\n",
    "        save_weights_only=True\n",
    "    )\n",
    "\n",
    "    early_stopping = EarlyStopping(monitor='val_loss', patience=5)\n",
    "    callbacks_list = [checkpoint, early_stopping]\n",
    "    \n",
    "    print(text_train_len)\n",
    "    print(text_val_len)\n",
    "    print(train_batch_count)\n",
    "    print(val_batch_count)\n",
    "    \n",
    "    def RNN():\n",
    "        model = Sequential()\n",
    "        model.add(LSTM(layer_dim, return_sequences = True, input_shape = (seq_length, vocab_size)))\n",
    "        model.add(Dropout (0.2))\n",
    "        model.add(LSTM(layer_dim, return_sequences = True))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(LSTM(layer_dim, return_sequences = True))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(LSTM(layer_dim, return_sequences = False))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(Dense(vocab_size, activation = 'softmax'))\n",
    "        model.compile(loss = 'categorical_crossentropy', optimizer = 'adam')\n",
    "        return model\n",
    "    \n",
    "    model = RNN()\n",
    "    \n",
    "    history = model.fit_generator(\n",
    "    batch_generator(text_train, count=train_batch_count),\n",
    "    train_batch_count,\n",
    "    max_queue_size=1,\n",
    "    epochs=nb_epochs,\n",
    "    callbacks=callbacks_list,\n",
    "    validation_data=batch_generator(text_val, count=val_batch_count),\n",
    "    validation_steps=val_batch_count,\n",
    "    initial_epoch = 0\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Generate(num_gen = 5, stri = \"It is\", cnt = 140):\n",
    "    \n",
    "    def RNN():\n",
    "        model = Sequential()\n",
    "        model.add(LSTM(layer_dim, return_sequences = True, batch_input_shape = (1, 1, vocab_size), stateful = True))\n",
    "        model.add(Dropout (0.2))\n",
    "        model.add(LSTM(layer_dim, return_sequences = True, stateful = True))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(LSTM(layer_dim, return_sequences = True, stateful = True))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(LSTM(layer_dim, return_sequences = False, stateful = True))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(Dense(vocab_size, activation = 'softmax'))\n",
    "        model.compile(loss = 'categorical_crossentropy', optimizer = 'adam')\n",
    "        return model\n",
    "    \n",
    "    model = RNN()\n",
    "    model.load_weights(\"./weights/weights\")\n",
    "    \n",
    "    def sample(preds, temperature=1.0):\n",
    "        preds = np.asarray(preds).astype('float64')\n",
    "        preds = np.log(preds) / temperature\n",
    "        exp_preds = np.exp(preds)\n",
    "        preds = exp_preds / np.sum(exp_preds)\n",
    "        probas = np.random.multinomial(1, preds, 1)\n",
    "        return np.argmax(probas)\n",
    "\n",
    "    def predict_next_char(model, current_char, diversity=1.0):\n",
    "        x = np.zeros((1, 1, vocab_size))\n",
    "        x[:,:,char_to_int[current_char]] = 1\n",
    "        y = model.predict(x, batch_size=1)\n",
    "        next_char_idx = sample(y[0,:], temperature=diversity)\n",
    "        next_char = characters[next_char_idx]\n",
    "        return next_char\n",
    "\n",
    "    def generate_text(model, seed=\"I am\", count=cnt):\n",
    "        model.reset_states()\n",
    "        for s in seed[:-1]:\n",
    "            next_char = predict_next_char(model, s)\n",
    "        current_char = seed[-1]\n",
    "\n",
    "        sys.stdout.write(\"[\"+seed+\"]\")\n",
    "        \n",
    "        for i in range(count - len(seed)):\n",
    "            next_char = predict_next_char(model, current_char, diversity=0.5)\n",
    "            current_char = next_char\n",
    "            sys.stdout.write(next_char)\n",
    "        print(\"...\\n\")\n",
    "    \n",
    "    for i in range(num_gen):\n",
    "        generate_text(\n",
    "            model,\n",
    "            seed=stri\n",
    "        )\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_args():\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument(\"--mode\", type = str)\n",
    "    parser.add_argument(\"--num_sentences\", type = int, default = 5)\n",
    "    parser.add_argument(\"--string\", type = str, default = \"It is\")\n",
    "    parser.add_argument(\"--num_chars\", type = int, default = 140)\n",
    "    args = parser.parse_args()\n",
    "    return args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "usage: ipykernel_launcher.py [-h] [--mode MODE]\n",
      "                             [--num_sentences NUM_SENTENCES] [--string STRING]\n",
      "                             [--num_chars NUM_CHARS]\n",
      "ipykernel_launcher.py: error: unrecognized arguments: -f /Users/soumil942/Library/Jupyter/runtime/kernel-38e3f840-1919-4bf2-afbc-34c6f2526126.json\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2971: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    args = get_args()\n",
    "    if(args.mode == \"train\"):\n",
    "        Train()\n",
    "    elif(args.mode == \"generate\"):\n",
    "        Generate(args.num_sentences, args.string, args.num_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}