nomath_13_class_HAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup HAN prior running here:\n",
    "# $ git clone https://github.com/FlorisHoogenboom/keras-han-for-docla\n",
    "# $ cd keras-han-for-docla\n",
    "# $ pip install .\n",
    "\n",
    "import warnings\n",
    "import numpy as np\n",
    "import gc\n",
    "import json\n",
    "import h5py\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.utils import Sequence\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.sequence import _remove_long_seq\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, Input, Dense, Dropout\n",
    "from keras import metrics\n",
    "from keras import backend as K\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras_han.model import HAN\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "gpu_options = tf.GPUOptions(\n",
    "    per_process_gpu_memory_fraction=0.65, allow_growth=False)\n",
    "config = tf.ConfigProto(intra_op_parallelism_threads=16,\n",
    "                        inter_op_parallelism_threads=16, allow_soft_placement=True, gpu_options=gpu_options)\n",
    "session = tf.Session(config=config)\n",
    "K.set_session(session)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_vocab():\n",
    "    with open('data/word_nomath_index.json') as json_data:\n",
    "        return json.load(json_data)\n",
    "\n",
    "def load_glove():\n",
    "    glove = {}\n",
    "    with open('data/glove.model.nomath.txt') as glove_data:\n",
    "        for line in glove_data:\n",
    "            items = line.split()\n",
    "            key = items[0]\n",
    "            glove[key] = np.asarray(items[1:], dtype='float32')\n",
    "    return glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data-management before main training\n",
    "\n",
    "# Input data is obtained via batch loading from HDF5\n",
    "class DataGenerator(Sequence):\n",
    "    def __init__(self, batch_size=128, mode=\"train\", dim=480, max_sent_words = 20, max_sents = 24,\n",
    "             n_classes=13, x_hf = None, y_hf = None, shuffle=False, cap=None):\n",
    "        'Initialization'\n",
    "        self.x_hf = x_hf\n",
    "        self.y_hf = y_hf\n",
    "        self.mode = mode\n",
    "        self.max_sent_words = max_sent_words\n",
    "        self.max_sents = max_sents\n",
    "        if cap:\n",
    "            if cap>1:\n",
    "                self.total_len = cap\n",
    "            if cap>0 and cap<1:\n",
    "                self.total_len = int(cap * self.y_hf.shape[0])\n",
    "        else:\n",
    "            self.total_len = self.y_hf.shape[0]\n",
    "        self.validation_len = int(np.ceil(0.1 * self.total_len))\n",
    "        self.training_len = self.total_len - self.validation_len\n",
    "        if self.mode == \"validation\":\n",
    "            self.data_len = self.validation_len\n",
    "            self.list_IDs = np.arange(self.training_len, self.total_len)\n",
    "        else:\n",
    "            self.data_len = self.training_len\n",
    "            self.list_IDs = np.arange(0,self.training_len+1)\n",
    "            \n",
    "        self.dim = dim\n",
    "        self.batch_size = batch_size\n",
    "        self.n_classes = n_classes\n",
    "        self.shuffle = shuffle\n",
    "        self.on_epoch_end() \n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        'Updates indexes after each epoch'\n",
    "        self.indexes = np.arange(len(self.list_IDs))\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.indexes)\n",
    "            \n",
    "    def __len__(self):\n",
    "        'Denotes the number of batches per epoch'\n",
    "        return int(np.floor(len(self.list_IDs) / self.batch_size))\n",
    "    \n",
    "    def __data_generation(self, list_IDs_temp):\n",
    "        'Generates data containing batch_size samples' # X : (n_samples,  dim)\n",
    "        # Initialization\n",
    "        X = np.empty((self.batch_size, self.max_sents, self.max_sent_words), dtype=int)\n",
    "        y = np.empty((self.batch_size), dtype=int)\n",
    "\n",
    "        # Generate data\n",
    "        for i, ID in enumerate(list_IDs_temp):\n",
    "            # Store sample\n",
    "            # Also reshape x to a \"24 sentences of 20 words\" shape\n",
    "            X[i,] = self.x_hf[ID].reshape(self.max_sents, self.max_sent_words)\n",
    "            # Store class\n",
    "            y[i] = self.y_hf[ID]\n",
    "        \n",
    "        return X, y\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        'Generate one batch of data'\n",
    "        # Generate indexes of the batch\n",
    "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
    "\n",
    "        # Find list of IDs\n",
    "        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n",
    "\n",
    "        # Generate data\n",
    "        X, y = self.__data_generation(list_IDs_temp)\n",
    "\n",
    "        return X, y\n",
    "\n",
    "batch_size = 128\n",
    "# Parameters\n",
    "data_hf = h5py.File(\"data/confusion_free_nomath.hdf5\", 'r')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparing word embeddings\n",
    "# Takes 10 minutes to recompute, memoizing here for the v4 49 class variant:\n",
    "# class_weights = compute_class_weight('balanced', np.unique(training_generator.y_hf), training_generator.y_hf)\n",
    "# np.set_printoptions(precision=32, suppress=True)\n",
    "# print(class_weights)\n",
    "\n",
    "class_weights = [\n",
    "  0.7483095715566197, 4.754501247119297,  1.922516556926298,\n",
    "  1.1549558601260375, 2.668737662773207,  1.1207340910837413,\n",
    "  492.8275989186532,  0.3638972379781621, 0.20817024795949343,\n",
    "  13.798952339173741, 29.327446829514418,  1.205880584288014,\n",
    "  3.2184743461345864 ]\n",
    "\n",
    "n_classes = 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- loading word embeddings, this may take a little while...\n",
      "-- known dictionary items:  757203\n"
     ]
    }
   ],
   "source": [
    "def build_embedding_matrix(vocab_dim=300):\n",
    "    print(\"-- loading word embeddings, this may take a little while...\")\n",
    "    index_dict = load_vocab()\n",
    "    word_vectors = load_glove()\n",
    "    # adding 1 to account for 0th index (for masking)\n",
    "    n_symbols = len(index_dict) + 1\n",
    "    print(\"-- known dictionary items: \", n_symbols)\n",
    "    embedding_weights = np.zeros((n_symbols, vocab_dim))\n",
    "    embedding_weights[0] = 0\n",
    "    for word, index in index_dict.items():\n",
    "        embedding_weights[index, :] = word_vectors[word]\n",
    "    return embedding_weights\n",
    "\n",
    "embedding_matrix = build_embedding_matrix(vocab_dim=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- setting up model layout...\n",
      "WARNING:tensorflow:From /home/deyan/.local/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 8, 60)             0         \n",
      "_________________________________________________________________\n",
      "word_encoder (TimeDistribute (None, 8, 60, 300)        227566800 \n",
      "_________________________________________________________________\n",
      "word_attention (TimeDistribu (None, 8, 300)            30100     \n",
      "_________________________________________________________________\n",
      "sentence_encoder (Model)     (None, 8, 300)            405900    \n",
      "_________________________________________________________________\n",
      "sentence_attention (Attentio (None, 300)               30100     \n",
      "_________________________________________________________________\n",
      "class_prediction (Dense)     (None, 13)                3913      \n",
      "=================================================================\n",
      "Total params: 228,036,813\n",
      "Trainable params: 875,913\n",
      "Non-trainable params: 227,160,900\n",
      "_________________________________________________________________\n",
      "None\n",
      "-- training model...\n",
      "WARNING:tensorflow:From /home/deyan/.local/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/50\n",
      "56399/56399 [==============================] - 8902s 158ms/step - loss: 0.3540 - weighted_sparse_categorical_accuracy: 0.8878 - val_loss: 0.3578 - val_weighted_sparse_categorical_accuracy: 0.8863\n",
      "Epoch 2/50\n",
      "56399/56399 [==============================] - 9013s 160ms/step - loss: 0.3196 - weighted_sparse_categorical_accuracy: 0.8988 - val_loss: 0.3533 - val_weighted_sparse_categorical_accuracy: 0.8877\n",
      "Epoch 3/50\n",
      "56399/56399 [==============================] - 8914s 158ms/step - loss: 0.3245 - weighted_sparse_categorical_accuracy: 0.8971 - val_loss: 0.3833 - val_weighted_sparse_categorical_accuracy: 0.8775\n",
      "Epoch 4/50\n",
      "56399/56399 [==============================] - 8937s 158ms/step - loss: 0.3427 - weighted_sparse_categorical_accuracy: 0.8909 - val_loss: 0.3777 - val_weighted_sparse_categorical_accuracy: 0.8804\n",
      "Epoch 5/50\n",
      "56399/56399 [==============================] - 8549s 152ms/step - loss: 0.3435 - weighted_sparse_categorical_accuracy: 0.8907 - val_loss: 0.3885 - val_weighted_sparse_categorical_accuracy: 0.8758\n",
      "Epoch 00005: early stopping\n"
     ]
    }
   ],
   "source": [
    "# HAN Model Setup\n",
    "print(\"-- setting up model layout...\")\n",
    "model_file = \"nomath_han_batch%d_cat%d_gpu\" % (batch_size, n_classes)\n",
    "# stop early when metric stops improving\n",
    "earlystop = EarlyStopping(monitor='val_weighted_sparse_categorical_accuracy',\n",
    "                          min_delta=0.001,\n",
    "                          patience=3,\n",
    "                          verbose=1, mode='auto')\n",
    "\n",
    "def han_model(generator_params):\n",
    "    # Generators\n",
    "    training_generator = DataGenerator(**generator_params)\n",
    "    validation_generator = DataGenerator(mode=\"validation\", **generator_params)\n",
    "    # Prepare model\n",
    "    model = HAN(\n",
    "        generator_params['max_sent_words'], generator_params['max_sents'], n_classes, embedding_matrix,\n",
    "        word_encoding_dim=300, sentence_encoding_dim=300\n",
    "    )\n",
    "\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer=\"adam\",\n",
    "                  weighted_metrics=[metrics.sparse_categorical_accuracy])\n",
    "\n",
    "    # Print model summary\n",
    "    print(model.summary())\n",
    "\n",
    "    # Perform training\n",
    "    print('-- training model...')\n",
    "    out = model.fit_generator(\n",
    "        generator=training_generator,\n",
    "        validation_data=validation_generator,\n",
    "        workers = 1,\n",
    "        use_multiprocessing=False,\n",
    "        class_weight=class_weights,\n",
    "        epochs=50,\n",
    "        verbose=1,\n",
    "        callbacks=[earlystop])\n",
    "    return out, model\n",
    "\n",
    "# for (max_sents, max_sent_words) in [(5,96),(8,60),(10,48)]:\n",
    "max_sents = 8\n",
    "max_sent_words = 60\n",
    "generator_params = {\n",
    "    'batch_size': batch_size,\n",
    "    'n_classes': 13,\n",
    "#     'cap': 0.03,\n",
    "    'shuffle': True,\n",
    "    'max_sents': max_sents,\n",
    "    'max_sent_words': max_sent_words,\n",
    "    'x_hf': data_hf['x_train'],\n",
    "    'y_hf': data_hf['y_train']\n",
    "}\n",
    "(out, model) = han_model(generator_params)\n",
    "# print(\"(sents %d; words %d) result: \" % (max_sents, max_sent_words))\n",
    "# print(np.mean(out.history['val_weighted_sparse_categorical_accuracy']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- saving model weights to disk : nomath_han_batch128_cat13_gpu \n"
     ]
    }
   ],
   "source": [
    "# save model weights\n",
    "# CAUTION: a regular model .save, including a checkpoint, seemed to produce an error??\n",
    "#          .save_weights works better...\n",
    "print(\"-- saving model weights to disk : %s \" % model_file)\n",
    "model.save_weights(model_file+'_notebook.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Per-class test measures:\n",
      "2005303/2005303 [==============================] - 845s 421us/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.92      0.92    206138\n",
      "           1       0.99      0.99      0.99     32444\n",
      "           2       0.82      0.72      0.77     80235\n",
      "           3       0.87      0.83      0.85    133559\n",
      "           4       0.70      0.46      0.55     57800\n",
      "           5       0.82      0.89      0.86    137637\n",
      "           6       0.74      0.66      0.70       313\n",
      "           7       0.89      0.92      0.91    423895\n",
      "           8       0.93      0.97      0.95    740999\n",
      "           9       0.82      0.73      0.78     11178\n",
      "          10       0.66      0.67      0.66      5259\n",
      "          11       0.76      0.65      0.70    127919\n",
      "          12       0.76      0.72      0.74     47927\n",
      "\n",
      "   micro avg       0.89      0.89      0.89   2005303\n",
      "   macro avg       0.82      0.78      0.80   2005303\n",
      "weighted avg       0.88      0.89      0.88   2005303\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Per-class test measures:\")\n",
    "x_test = [x.reshape(8,60) for x in data_hf[\"x_test\"]]\n",
    "y_prob = model.predict(np.array(x_test), batch_size=batch_size, verbose=1)\n",
    "y_pred = y_prob.argmax(axis=-1)\n",
    "\n",
    "print(classification_report(data_hf[\"y_test\"], y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the data of 480 max-words per paragraph is reshaped to a fixed 8 sentences of 60 words each (padded with 0 for missing entries), and hence does *not* use the real sentence breaks from tokenization - a possible direction for improvement. The split was found by grid search on 3% of the final data.\n",
    "\n",
    "## Test HAN(8;60), 13-class task\n",
    "\n",
    "```\n",
    "Per-class test measures:\n",
    "2005303/2005303 [==============================] - 845s 421us/step\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "           0       0.92      0.92      0.92    206138\n",
    "           1       0.99      0.99      0.99     32444\n",
    "           2       0.82      0.72      0.77     80235\n",
    "           3       0.87      0.83      0.85    133559\n",
    "           4       0.70      0.46      0.55     57800\n",
    "           5       0.82      0.89      0.86    137637\n",
    "           6       0.74      0.66      0.70       313\n",
    "           7       0.89      0.92      0.91    423895\n",
    "           8       0.93      0.97      0.95    740999\n",
    "           9       0.82      0.73      0.78     11178\n",
    "          10       0.66      0.67      0.66      5259\n",
    "          11       0.76      0.65      0.70    127919\n",
    "          12       0.76      0.72      0.74     47927\n",
    "\n",
    "   micro avg       0.89      0.89      0.89   2005303\n",
    "   macro avg       0.82      0.78      0.80   2005303\n",
    "weighted avg       0.88      0.89      0.88   2005303\n",
    "```\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}