Untitled7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "Preprocessing the whole text\n",
      "Preprocessing done\n",
      "Nb of samples 2046\n",
      "1024/1024 [==============================] - 29s - loss: 8.9747    \n",
      "Epoch 2/1000\n",
      " 320/1024 [========>.....................] - ETA: 15s - loss: 8.7294sortie de boucle\n",
      " 384/1024 [==========>...................] - ETA: 14s - loss: 8.5378"
     ]
    },
    {
     "ename": "Exception",
     "evalue": "output of generator should be a tuple (x, y, sample_weight) or (x, y). Found: None",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-515a7a6708b0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     64\u001b[0m \u001b[0mreduce_lr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mReduceLROnPlateau\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'val_loss'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfactor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpatience\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_lr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.001\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_generator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_gen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msamples_per_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_data_size\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnb_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1000\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnb_worker\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpickle_safe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     67\u001b[0m \u001b[0mcompteur\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m \u001b[1;31m# training command\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, **kwargs)\u001b[0m\n\u001b[0;32m    872\u001b[0m                                         \u001b[0mmax_q_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_q_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    873\u001b[0m                                         \u001b[0mnb_worker\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnb_worker\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 874\u001b[1;33m                                         pickle_safe=pickle_safe)\n\u001b[0m\u001b[0;32m    875\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    876\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mevaluate_generator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_samples\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_q_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnb_worker\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpickle_safe\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe)\u001b[0m\n\u001b[0;32m   1415\u001b[0m                     raise Exception('output of generator should be a tuple '\n\u001b[0;32m   1416\u001b[0m                                     \u001b[1;34m'(x, y, sample_weight) '\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1417\u001b[1;33m                                     'or (x, y). Found: ' + str(generator_output))\n\u001b[0m\u001b[0;32m   1418\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1419\u001b[0m                     \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgenerator_output\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mException\u001b[0m: output of generator should be a tuple (x, y, sample_weight) or (x, y). Found: None"
     ]
    }
   ],
   "source": [
    "#### Imports\n",
    "import sys\n",
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras.utils import np_utils\n",
    "from glove import Glove\n",
    "import gensim\n",
    "import numpy as np\n",
    "from functions.generators import bidirectional_batch_generator\n",
    "from functions.callbacks import LossHistory, ReduceLROnPlateau\n",
    "\n",
    "#### Parameters\n",
    "# paths\n",
    "word_embedder_folder = \"/home/ubuntu/summarization_query_oriented/nn_models/word_vectorization_models/glove/\"\n",
    "word_embedder = \"rank1_alltxt_size_400_epoch_30.glove\"\n",
    "\n",
    "# path to the text we use for training, take the corpus or sub corpus of the one you have trained glove \n",
    "txt_file = \"/home/ubuntu/summarization_query_oriented/data/wikipedia/txt/td_qfs_rank_1_all.txt\"\n",
    "\n",
    "# model folder\n",
    "model_folder = \"/home/ubuntu/summarization_query_oriented/nn_models/language_models/RNN/\"\n",
    "\n",
    "# size\n",
    "train_data_size = 1034\n",
    "val_data_size = 1034\n",
    "batch_size = 64\n",
    "seq_length = 10\n",
    "\n",
    "#### Implementation\n",
    "\n",
    "# glove model\n",
    "glove = Glove.load(word_embedder_folder+word_embedder)\n",
    "corpus = glove.dictionary\n",
    "corpus_size = len(corpus.keys())\n",
    "\n",
    "# dictionnaries\n",
    "word_to_int = corpus\n",
    "int_to_word = dict((corpus[key], key) for key in corpus.keys())\n",
    "\n",
    "# generator instance (this perform bidirectionnal batchs)\n",
    "train_gen = bidirectional_batch_generator(txt_file, data_size = train_data_size,glove_model = glove,  batch_size = batch_size, seq_length=10, verbose=True)\n",
    "\n",
    "val_gen  =  bidirectional_batch_generator(txt_file, data_size=val_data_size ,glove_model = glove,  batch_size = batch_size, seq_length=10, verbose=True, starting_idx=train_data_size)\n",
    "\n",
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(400, input_shape=(seq_length, glove.no_components), return_sequences=False, name=\"lstm_1\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(corpus_size, activation='softmax', name=\"dense_1\"))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adagrad')\n",
    "\n",
    "# Implementing callbacks\n",
    "\n",
    "model_name = \"bidirectional_LSTM_glove_\"\n",
    "model_name = model_name +\"epoch_{epoch:02d}_valloss_{val_loss:.2f}.hdf5\"\n",
    "\n",
    "history = LossHistory()\n",
    "checkpointer = ModelCheckpoint(filepath=model_folder+model_name, verbose=1, save_best_only=True)\n",
    "earlystopper = EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='auto')\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=3, min_lr=0.001)\n",
    "\n",
    "model.fit_generator(train_gen, samples_per_epoch = train_data_size-seq_length, nb_epoch = 1000, verbose = 1, nb_worker=1,pickle_safe = True)\n",
    "compteur = 0\n",
    "# training command\n",
    "while compteur < 2*train_data_size:\n",
    "    compteur+=1\n",
    "    x,y = train_gen.next()\n",
    "    model.train_on_batch(x,y)\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "#callbacks = [history, checkpointer, earlystopper, reduce_lr], validation_data = val_gen, nb_val_samples = 10000,  pickle_safe=True)\n",
    "                    #class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=True)\n",
    "\n",
    "# saving the model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "W1 = model.get_layer(\"lstm_1\").get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400, 400)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W1[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_gen' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-45e572e9db19>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_gen\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'train_gen' is not defined"
     ]
    }
   ],
   "source": [
    "X,y = train_gen.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "StopIteration",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-878d3780249b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_gen\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mStopIteration\u001b[0m: "
     ]
    }
   ],
   "source": [
    "type(train_gen.next())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}