char-rnn_trump_text_generation-TRAINING.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Char-RNN Trump-like text generation - TRAINING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import multi_gpu\n",
    "import string\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, Dropout, Activation, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N_GPU = 1 # you can experiment with more GPUs, it gets interesting with a high SEQUENCE_LEN\n",
    "SEQUENCE_LEN = 60\n",
    "BATCH_SIZE = 512\n",
    "EPOCHS = 20\n",
    "HIDDEN_LAYERS_DIM = 512\n",
    "LAYER_COUNT = 4\n",
    "DROPOUT = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocabulary len = 98\n",
      "['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~', ' ', '\\t', '\\n', '\\r']\n"
     ]
    }
   ],
   "source": [
    "# generic vocabulary\n",
    "characters = list(string.printable)\n",
    "characters.remove('\\x0b')\n",
    "characters.remove('\\x0c')\n",
    "\n",
    "VOCABULARY_SIZE = len(characters)\n",
    "characters_to_ix = {c:i for i,c in enumerate(characters)}\n",
    "print(\"vocabulary len = %d\" % VOCABULARY_SIZE)\n",
    "print(characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def describe_batch(X, y, samples=3):\n",
    "    \"\"\"Describe in a human-readable format some samples from a batch\"\"\"\n",
    "    for i in range(samples):\n",
    "        sentence = \"\"\n",
    "        for s in range(SEQUENCE_LEN):\n",
    "            sentence += characters[X[i,s,:].argmax()]\n",
    "        next_char = characters[y[i,:].argmax()]\n",
    "        \n",
    "        print(\"sample #%d: ...%s -> '%s'\" % (\n",
    "            i,\n",
    "            sentence[-20:],\n",
    "            next_char\n",
    "        ))\n",
    "\n",
    "def batch_generator(text, count):\n",
    "    \"\"\"Generate batches for training\"\"\"\n",
    "    while True: # keras wants that for reasons\n",
    "        for batch_ix in range(count):\n",
    "            X = np.zeros((BATCH_SIZE, SEQUENCE_LEN, VOCABULARY_SIZE))\n",
    "            y = np.zeros((BATCH_SIZE, VOCABULARY_SIZE))\n",
    "\n",
    "            batch_offset = BATCH_SIZE * batch_ix\n",
    "\n",
    "            for sample_ix in range(BATCH_SIZE):\n",
    "                sample_start = batch_offset + sample_ix\n",
    "                for s in range(SEQUENCE_LEN):\n",
    "                    X[sample_ix, s, characters_to_ix[text[sample_start+s]]] = 1\n",
    "                y[sample_ix, characters_to_ix[text[sample_start+s+1]]]=1\n",
    "\n",
    "            yield X, y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_model(gpu_count=1):\n",
    "    \"\"\"Build a Keras sequential model for training the char-rnn\"\"\"\n",
    "    model = Sequential()\n",
    "    for i in range(LAYER_COUNT):\n",
    "        model.add(\n",
    "            LSTM(\n",
    "                HIDDEN_LAYERS_DIM, \n",
    "                return_sequences=True if (i!=(LAYER_COUNT-1)) else False,\n",
    "                input_shape=(SEQUENCE_LEN, VOCABULARY_SIZE),\n",
    "            )\n",
    "        )\n",
    "        model.add(Dropout(DROPOUT))\n",
    "    \n",
    "    model.add(Dense(VOCABULARY_SIZE))\n",
    "    model.add(Activation('softmax'))\n",
    "    \n",
    "    if gpu_count>1:\n",
    "        model = multi_gpu.make_parallel(model, gpu_count)\n",
    "    \n",
    "    model.compile(loss='categorical_crossentropy', optimizer=\"adam\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total of 3448058 characters\n",
      "sample #0: ...ump view that an ind -> 'i'\n",
      "sample #1: ...mp view that an indi -> 'v'\n",
      "sample #2: ...p view that an indiv -> 'i'\n",
      "sample #3: ... view that an indivi -> 'd'\n",
      "sample #4: ...view that an individ -> 'u'\n"
     ]
    }
   ],
   "source": [
    "# loading the text\n",
    "with open(\"./dataset_train.txt\", \"r\") as f:\n",
    "    text_train = f.read()\n",
    "with open(\"./dataset_val.txt\", \"r\") as f:\n",
    "    text_val = f.read()\n",
    "\n",
    "text_train_len = len(text_train)\n",
    "text_val_len = len(text_val)\n",
    "print(\"Total of %d characters\" % (text_train_len + text_val_len))\n",
    "\n",
    "for ix, (X,y) in enumerate(batch_generator(text_train, count=1)):\n",
    "    # describe some samples from the first batch\n",
    "    describe_batch(X, y, samples=5)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training batch count: 6062\n",
      "validation batch count: 672\n"
     ]
    }
   ],
   "source": [
    "training_model = build_model(gpu_count=N_GPU)\n",
    "\n",
    "train_batch_count = (text_train_len - SEQUENCE_LEN) // BATCH_SIZE\n",
    "val_batch_count = (text_val_len - SEQUENCE_LEN) // BATCH_SIZE\n",
    "print(\"training batch count: %d\" % train_batch_count)\n",
    "print(\"validation batch count: %d\" % val_batch_count)\n",
    "\n",
    "# checkpoint\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "filepath = \"./%d-gpu_BS-%d_%d-%s_dp%.2f_%dS_epoch{epoch:02d}-loss{loss:.4f}-val-loss{val_loss:.4f}_weights\" % (\n",
    "    N_GPU,\n",
    "    BATCH_SIZE,\n",
    "    LAYER_COUNT,\n",
    "    HIDDEN_LAYERS_DIM,\n",
    "    DROPOUT,\n",
    "    SEQUENCE_LEN\n",
    ")\n",
    "checkpoint = ModelCheckpoint(\n",
    "    filepath,\n",
    "    save_weights_only=True\n",
    ")\n",
    "# early stopping\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)\n",
    "\n",
    "callbacks_list = [checkpoint, early_stopping]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = training_model.fit_generator(\n",
    "    batch_generator(text_train, count=train_batch_count),\n",
    "    train_batch_count,\n",
    "    max_queue_size=1, # no more than one queued batch in RAM\n",
    "    epochs=EPOCHS,\n",
    "    callbacks=callbacks_list,\n",
    "    validation_data=batch_generator(text_val, count=val_batch_count),\n",
    "    validation_steps=val_batch_count,\n",
    "    initial_epoch=0\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}