Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "# load ascii text and covert to lowercase\n",
    "from glove import Glove\n",
    "import gensim\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# paths\n",
    "word_embedder_folder = \"/home/ubuntu/summarization_query_oriented/nn_models/word_vectorization_models/glove/\"\n",
    "word_embedder = \"rank1_alltxt_size_400_epoch_30.glove\"\n",
    "\n",
    "# glove model\n",
    "glove = Glove.load(word_embedder_folder+word_embedder)\n",
    "corpus = glove.dictionary\n",
    "corpus_size = len(corpus.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create mapping of unique chars to integers, and a reverse mapping\n",
    "#words = sorted(list(set(raw_text.split(\" \"))))\n",
    "word_to_int = corpus\n",
    "int_to_word = dict((corpus[key], key) for key in corpus.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "txt_file = \"/home/ubuntu/summarization_query_oriented/data/wikipedia/txt/td_qfs_rank_1_all.txt\"\n",
    "with open(txt_file, 'r') as f:\n",
    "    raw_text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = gensim.utils.simple_preprocess(raw_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3782348"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words:\n",
      "3782348\n",
      "Total Vocab: \n",
      "84486\n"
     ]
    }
   ],
   "source": [
    "# summarize the loaded data\n",
    "n_words = len(text)\n",
    "n_vocab = len(corpus.keys())\n",
    "print(\"Total words:\")\n",
    "print(n_words)\n",
    "print(\"Total Vocab: \")\n",
    "print(n_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare the dataset of input to output pairs encoded as integers\n",
    "seq_length = 10\n",
    "dataX = []\n",
    "dataY = []\n",
    "#for i in range(0, n_chars - seq_length, 1):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one_hot_vector(word ,corpus):\n",
    "    \"\"\"\n",
    "    given a corpus of (key,value) = (word,index) , this function return a one hot vector corresponding the input word\n",
    "    \"\"\"\n",
    "    size = len(corpus.values())\n",
    "    y = np.zeros(size)\n",
    "    y[corpus[word]] = 1.0\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "for i in range(0, batch_size, 1):\n",
    "    seq_in = text[i:i + seq_length]\n",
    "    seq_out = text[i + seq_length]\n",
    "    dataX.append([glove.word_vectors[word_to_int[word]] for word in seq_in])\n",
    "    dataY.append(one_hot_vector(seq_out,corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "X = np.asarray(dataX)\n",
    "y = np.asarray(dataY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 10, 400)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "glove.no_components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X[0,:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(400, input_shape=(X.shape[1], X.shape[2]), return_sequences=False, name=\"lstm_1\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(y.shape[1], activation='softmax', name=\"dense_1\"))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64, 84486)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from functions.generators import bidirectional_batch_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "train_gen = bidirectional_batch_generator(txt_file, train_data_size=1000, glove_model=glove, batch_size = 128, seq_length=10, verbose=True, val_data_size = 1000, validation = False)\n",
    "val_gen = bidirectional_batch_generator(txt_file, train_data_size=1000, glove_model=glove, batch_size = 128, seq_length=10, verbose=True, val_data_size = 1000, validation = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X,y = train_gen.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X, y = val_gen.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 10, 400)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 84486)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "Preprocessing the whole text\n",
      "Preprocessing done\n",
      "Nb of samples3782337\n",
      "100096/100000 [==============================] - 1955s - loss: 7.9725  \n",
      "Epoch 2/3\n",
      "100096/100000 [==============================] - 1942s - loss: 7.6664  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.py:1462: UserWarning: Epoch comprised more than `samples_per_epoch` samples, which might affect learning results. Set `samples_per_epoch` correctly to avoid this warning.\n",
      "  warnings.warn('Epoch comprised more than '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f94107d6d50>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(gen, samples_per_epoch = 100000 , nb_epoch = 3, verbose=1, callbacks=[], validation_data=None, nb_val_samples=None, class_weight=None, max_q_size=10, nb_worker=4, pickle_safe=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = gen.next()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pick a random seed\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print(\"Seed:\")\n",
    "print(''.join([int_to_word[value] for value in pattern]))\n",
    "# generate characters\n",
    "for i in range(1000):\n",
    "\tx = numpy.reshape(pattern, (1, len(pattern), 1))\n",
    "\tx = x / float(n_vocab)\n",
    "\tprediction = model.predict(x, verbose=0)\n",
    "\tindex = numpy.argmax(prediction)\n",
    "\tresult = int_to_char[index]\n",
    "\tseq_in = [int_to_char[value] for value in pattern]\n",
    "\tsys.stdout.write(result)\n",
    "\tpattern.append(index)\n",
    "\tpattern = pattern[1:len(pattern)]\n",
    "print(\"\\nDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}