lstm_get_hidden_layer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#from __future__ import print_function\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Activation, Dropout\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.utils.data_utils import get_file\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "import sys\n",
    "import os    \n",
    "os.environ['THEANO_FLAGS'] = \"device=gpu, floatX=float32\"\n",
    "import gensim\n",
    "from functions.words_chars import vocabulary_from_json_corpus\n",
    "\n",
    "json_corpus_path = \"/home/ubuntu/summarization_query_oriented/data/wikipedia/json/td_qfs_rank_1/\"\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('word_indices', <type 'dict'>, 'length:', 83161)\n",
      "('indices_words', <type 'dict'>, 'length', 83161)\n",
      "Build model...\n",
      "Model built...\n"
     ]
    }
   ],
   "source": [
    "# building vocabulary of the corpus\n",
    "words = vocabulary_from_json_corpus(json_corpus_path)\n",
    "word_indices = dict((c, i) for i, c in enumerate(words))\n",
    "indices_word = dict((i, c) for i, c in enumerate(words))\n",
    "print(\"word_indices\", type(word_indices), \"length:\",len(word_indices))\n",
    "print(\"indices_words\", type(indices_word), \"length\", len(indices_word))\n",
    "\n",
    "maxlen = 10\n",
    "\n",
    "#defining the lstm model\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "model.add(LSTM(400, return_sequences=True, input_shape=(maxlen, len(word_indices))))\n",
    "model.add(Dropout(0.6))\n",
    "model.add(LSTM(400, return_sequences=False))\n",
    "model.add(Dropout(0.6))\n",
    "model.add(Dense(len(words)))\n",
    "#model.add(Dense(1000))\n",
    "model.add(Activation('softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "print('Model built...')\n",
    "\n",
    "# naming \n",
    "model_folder = \"/home/ubuntu/summarization_query_oriented/nn_models/language_models/RNN/wider_30102016/\"\n",
    "model_name = \"tdqfs_lstm_wider_corpus_last.hdf5\"\n",
    "\n",
    "if os.path.isfile('/home/ubuntu/summarization_query_oriented/nn_models/language_models/RNN/wider_30102016/tdqfs_lstm_wider_corpus_last.hdf5'):\n",
    "    model.load_weights('/home/ubuntu/summarization_query_oriented/nn_models/language_models/RNN/wider_30102016/tdqfs_lstm_wider_corpus_last.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "lstm_1 (LSTM)                    (None, 10, 400)       133699200   lstm_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 10, 400)       0           lstm_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                    (None, 400)           1281600     dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 400)           0           lstm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 83161)         33347561    dropout_2[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 83161)         0           dense_1[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 168328361\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lstm_1\n",
      "lstm_2\n"
     ]
    }
   ],
   "source": [
    "print(model.layers[0].name)\n",
    "print(model.layers[2].name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build model to get the hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "Model built...\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "model2 = Sequential()\n",
    "model2.add(LSTM(400, return_sequences=True, batch_input_shape=(1,maxlen, len(word_indices)),weights=model.layers[0].get_weights(),stateful=True))\n",
    "model2.add(LSTM(400, return_sequences=False,weights=model.layers[2].get_weights(),stateful=True))\n",
    "print('Model built...')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# predict vector from sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "()\n",
      "----- Example:\n",
      "('----- Generating with seed: \"', [u'diabetes', u'mellitus', u'dm', u'commonly', u'referred', u'to', u'as', u'diabetes', u'is', u'group', u'of', u'metabolic', u'diseases', u'in', u'which', u'there', u'are', u'high', u'blood', u'sugar', u'levels', u'over', u'prolonged', u'period'], '\"')\n",
      "()\n"
     ]
    },
    {
     "ename": "Exception",
     "evalue": "Error when checking : expected lstm_input_1 to have shape (None, 10, 83161) but got array with shape (1, 24, 83161)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-03a0a737a3bb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     21\u001b[0m         \u001b[0mx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mword_indices\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[0mnext_index\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[0;32m    669\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    670\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 671\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    672\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    673\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict_on_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[0;32m   1159\u001b[0m         x = standardize_input_data(x, self.input_names,\n\u001b[0;32m   1160\u001b[0m                                    \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minternal_input_shapes\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1161\u001b[1;33m                                    check_batch_dim=False)\n\u001b[0m\u001b[0;32m   1162\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstateful\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1163\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[1;34m(data, names, shapes, check_batch_dim, exception_prefix)\u001b[0m\n\u001b[0;32m    106\u001b[0m                                         \u001b[1;34m' to have shape '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshapes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m                                         \u001b[1;34m' but got array with shape '\u001b[0m \u001b[1;33m+\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 108\u001b[1;33m                                         str(array.shape))\n\u001b[0m\u001b[0;32m    109\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0marrays\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mException\u001b[0m: Error when checking : expected lstm_input_1 to have shape (None, 10, 83161) but got array with shape (1, 24, 83161)"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import numpy as np\n",
    "\n",
    "sentence_str = \"Diabetes mellitus (DM), commonly referred to as diabetes, is a group of metabolic diseases in which there are high blood sugar levels over a prolonged period.\"\n",
    "sentence = gensim.utils.simple_preprocess(sentence_str)\n",
    "starting_idx = 0\n",
    "#sentence = sentence[starting_idx :maxlen + starting_idx]\n",
    "\n",
    "#start_index = random.randint(0, len(list_words) - maxlen - 1)\n",
    "\n",
    "print()\n",
    "print('----- Example:')\n",
    "generated = ''\n",
    "generated += ' '.join(sentence)\n",
    "print('----- Generating with seed: \"' , sentence , '\"')\n",
    "print()\n",
    "\n",
    "for i in range(3):\n",
    "    x = np.zeros((1, maxlen, len(words)))\n",
    "    for t, word in enumerate(sentence):\n",
    "        x[0, t, word_indices[word]] = 1.\n",
    "\n",
    "    preds = model.predict(x, verbose=0)[0]\n",
    "\n",
    "    next_index = np.argmax(preds)\n",
    "    next_word = indices_word[next_index]\n",
    "    generated += \" \" + next_word\n",
    "    print(generated)\n",
    "    del sentence[0]\n",
    "    sentence.append(next_word)\n",
    "    \n",
    "    tic = time.time()\n",
    "    preds = model2.predict(x, verbose=0)[0]\n",
    "    toc = time.time()\n",
    "    print(\"Time : \", toc-tic)\n",
    "    print(preds.shape)\n",
    "    print(preds[:5])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Quadruplets \n",
    "\n",
    "# training file , I remove the concept of validation loss\n",
    "\n",
    "# In this script we perform the training of the fully connected model\n",
    "\n",
    "# Import \n",
    "import gensim\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "#%matplotlib inline\n",
    "import pyrouge\n",
    "from pyrouge import Rouge155\n",
    "\n",
    "from hard_coded import data_json_dir, data_txt_dir, lang_model_dir, model_dir, nn_summarizers_dir, summary_system_super_dir, tdqfs_folder\n",
    "from hard_coded import non_selected_keys,tdqfs_themes\n",
    "from functions.training_functions import *\n",
    "\n",
    "# paths to folder \n",
    "data_json = data_json_dir\n",
    "data_txt = data_txt_dir\n",
    "lang_model_folder = lang_model_dir\n",
    "nn_summarizers_folder = nn_summarizers_dir\n",
    "summary_system_super_folder = summary_system_super_dir\n",
    "themes = tdqfs_themes\n",
    "\n",
    "#title_file = \"/home/ubuntu/summarization_query_oriented/data/DUC/duc2005_topics.sgml\"\n",
    "#titles_folder = \"/home/ubuntu/summarization_query_oriented/data/DUC/duc2005_docs/\"\n",
    "\n",
    "\n",
    "# training parameters\n",
    "\n",
    "patience_limit = 25\n",
    "\n",
    "## loading a d2vmodel (to be a shifted LSTM next ...)\n",
    "\n",
    "# parameters of doc2vec\n",
    "dm = 0\n",
    "min_count = 5\n",
    "window = 10\n",
    "size = 400\n",
    "sample = 1e-4\n",
    "negative = 5\n",
    "workers = 4\n",
    "epoch = 100\n",
    "\n",
    "# Initialize the model ( IMPORTANT )\n",
    "d2v_model = gensim.models.doc2vec.Doc2Vec(dm=dm,min_count=min_count, window=window, size=size, sample=sample, negative=negative, workers=workers,iter = epoch)\n",
    "\n",
    "# load model\n",
    "model_name =\"dm_\"+str(dm)+\"_mc_\"+str(min_count)+\"_w_\"+str(window)+\"_size_\"+str(size)+\"_neg_\"+str(negative)+\"_ep_\"+str(epoch)+\"_wosw\"\n",
    "try :\n",
    "    d2v_model = d2v_model.load(lang_model_folder+model_name+\".d2v\")\n",
    "except :\n",
    "    print \"try a model in : \", os.listdir(lang_model_folder)\n",
    "print(\"model loaded\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "txt = \"they have a common name diabetes mellitus and diabetes insipidus\"\n",
    "prep = gensim.utils.simple_preprocess(txt, deacc=True)\n",
    "tic = time.time()\n",
    "vector = d2v_model.infer_vector(txt)\n",
    "toc = time.time()\n",
    "print(\"time : \",toc-tic)\n",
    "print(vector.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "()\n",
      "----- Example:\n",
      "('----- Generating with seed: \"', [u'diabetes', u'mellitus', u'dm', u'commonly', u'referred', u'to', u'as', u'diabetes', u'is', u'group'], '\"')\n",
      "()\n",
      "('Time : ', 0.2493610382080078)\n",
      "(1, 400)\n",
      "[[  0.00000000e+00  -1.60113745e-03   0.00000000e+00   5.92325255e-02\n",
      "   -2.90173152e-03  -1.00483885e-02   0.00000000e+00  -1.47134922e-02\n",
      "   -0.00000000e+00  -1.10678934e-02  -3.17822257e-03  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   1.06023783e-02  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   5.68611221e-03  -0.00000000e+00\n",
      "   -2.05752694e-05   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   1.21447882e-02   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   1.05356807e-02   4.54170287e-01  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    4.03374881e-02  -0.00000000e+00   4.50232401e-02   0.00000000e+00\n",
      "    0.00000000e+00  -2.10595094e-02   0.00000000e+00   0.00000000e+00\n",
      "    5.87070826e-03   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   4.05173935e-03\n",
      "    9.38000381e-02   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   4.25078999e-03   0.00000000e+00\n",
      "    0.00000000e+00   8.17145407e-02   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -5.46167698e-03\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00  -1.48412779e-01\n",
      "   -0.00000000e+00   0.00000000e+00   1.92058042e-01   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   2.16715932e-02\n",
      "    1.64407834e-01   0.00000000e+00  -2.51287855e-02   0.00000000e+00\n",
      "   -7.36058829e-03   6.01446703e-02   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    1.35131106e-02   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -6.09611766e-03\n",
      "   -5.83479181e-04   4.67265069e-01   0.00000000e+00  -0.00000000e+00\n",
      "   -9.03361514e-02  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -1.44817077e-05  -4.67999693e-04   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -1.73157468e-01  -4.53947298e-03   1.68736875e-01  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -2.83118505e-02   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   7.90729970e-02   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   2.91295326e-03  -2.79085841e-02\n",
      "   -2.88646971e-03   3.88440257e-03   0.00000000e+00   0.00000000e+00\n",
      "   -2.78316438e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -2.16307398e-02  -0.00000000e+00  -6.05993345e-02  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    1.55895567e-02   0.00000000e+00  -0.00000000e+00  -1.23457052e-03\n",
      "   -4.53711525e-02   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -1.37707428e-03\n",
      "    2.55479403e-02   0.00000000e+00  -7.79058435e-04   8.67900532e-03\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -1.35602790e-03\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   6.50264788e-03\n",
      "   -1.24314905e-03   1.84821844e-01   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   2.59472145e-05   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   4.54501919e-02  -1.35597987e-02\n",
      "    0.00000000e+00  -2.37547909e-03   8.14663872e-05   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   2.05859942e-06  -0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00  -1.11731231e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   1.50237843e-01  -1.23780236e-01  -3.81472111e-02\n",
      "    0.00000000e+00   0.00000000e+00   2.56875646e-03   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00  -6.33096928e-03   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   6.47066845e-05   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   8.59703170e-04   3.39838771e-05\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   3.55326780e-03   0.00000000e+00\n",
      "    0.00000000e+00  -2.76075065e-01   1.14139766e-01  -1.10482387e-01\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   1.76145993e-02\n",
      "    0.00000000e+00  -1.54650938e-02   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   1.84466073e-03   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -5.66656049e-03\n",
      "    1.18369674e-02   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   1.68962181e-02\n",
      "    0.00000000e+00   0.00000000e+00   1.67892575e-01   2.13296828e-03\n",
      "    0.00000000e+00   1.95025623e-01   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    2.44019181e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -4.62518595e-02   1.50367757e-02   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -5.91274817e-03   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -6.52191741e-03\n",
      "   -8.82582754e-05   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    1.14662321e-02  -6.75105155e-02  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   1.36610687e-01   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00  -9.14508360e-04   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -6.93801465e-03  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -1.84775859e-01  -0.00000000e+00   0.00000000e+00   0.00000000e+00]]\n",
      "('Time : ', 0.25579094886779785)\n",
      "(1, 400)\n",
      "[[  0.00000000e+00  -1.03723551e-05   0.00000000e+00   7.01672286e-02\n",
      "   -0.00000000e+00  -2.90491618e-03   0.00000000e+00  -3.16928017e-05\n",
      "    0.00000000e+00  -1.32068549e-03  -5.14616584e-03  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   4.68415601e-06  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   3.46891629e-03  -0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   4.07816563e-03   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -6.60340767e-04   5.64792871e-01  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   3.30860093e-02   0.00000000e+00\n",
      "    0.00000000e+00  -9.95111000e-03   0.00000000e+00   0.00000000e+00\n",
      "   -4.57954739e-04   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    9.65557545e-02   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   4.63641854e-03  -0.00000000e+00\n",
      "   -0.00000000e+00   5.21250218e-02   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -9.70844440e-07   0.00000000e+00  -1.60648018e-01\n",
      "   -0.00000000e+00   0.00000000e+00   2.14726567e-01   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   3.36252409e-03\n",
      "    1.64747477e-01   0.00000000e+00  -6.31876197e-03   0.00000000e+00\n",
      "   -5.67294518e-03   6.37249798e-02   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    5.83084114e-02   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    7.07580475e-03   5.88783860e-01   0.00000000e+00  -0.00000000e+00\n",
      "   -1.22805141e-01  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -2.13903412e-01  -1.81258051e-03   2.03830734e-01   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -2.18405314e-02   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   9.60703269e-02  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   1.18873452e-07  -8.17030221e-02\n",
      "    6.70524081e-04   2.49672960e-03   0.00000000e+00   0.00000000e+00\n",
      "   -3.60163867e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "   -2.26595718e-02   0.00000000e+00  -5.98397814e-02  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -2.44092867e-02   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -1.93990394e-02   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    1.52312247e-02   0.00000000e+00  -3.12980410e-04  -3.97912692e-03\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   3.01478256e-04\n",
      "    3.38805583e-03   1.75281346e-01   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   4.74185236e-02  -1.58999227e-02\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   7.03236321e-03  -0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00  -1.29289076e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   1.70973942e-01  -2.76064396e-01  -2.91616991e-02\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00  -3.66778602e-03   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -5.91685675e-05   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   1.95389657e-05\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   4.48925933e-03   0.00000000e+00\n",
      "    0.00000000e+00  -2.74567127e-01   1.37514725e-01  -3.60422507e-02\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   2.64326856e-02\n",
      "    0.00000000e+00  -1.35270944e-02   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -8.59490596e-04\n",
      "    7.06664100e-03   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -4.91116303e-09   0.00000000e+00   0.00000000e+00   1.46420760e-04\n",
      "    0.00000000e+00   0.00000000e+00   1.32727578e-01   3.58242338e-04\n",
      "    0.00000000e+00   3.56952578e-01   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    2.59704322e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -4.32546400e-02   1.67325083e-02   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -1.00197864e-03\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    1.13047333e-02  -4.66939807e-02  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   1.71848416e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -3.99963290e-04   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -4.26920224e-03  -0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -1.04261443e-01  -0.00000000e+00   0.00000000e+00   0.00000000e+00]]\n",
      "('Time : ', 0.24881196022033691)\n",
      "(1, 400)\n",
      "[[  0.00000000e+00  -0.00000000e+00   0.00000000e+00   2.22029492e-01\n",
      "   -0.00000000e+00  -3.47663918e-05   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00  -0.00000000e+00   3.37279914e-03  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   1.48518132e-02  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   7.84611225e-01   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   1.19188726e-02   0.00000000e+00\n",
      "    0.00000000e+00  -2.03782972e-02   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    1.59836918e-01   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -1.22563785e-03  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00  -3.01711619e-01\n",
      "   -0.00000000e+00   0.00000000e+00   4.17892069e-01   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -2.87783821e-03\n",
      "    4.62055236e-01   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "   -5.49776852e-03   1.25872672e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    1.18167326e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    2.01433329e-04   8.43527198e-01   0.00000000e+00  -0.00000000e+00\n",
      "   -8.53167623e-02  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -3.55033576e-01  -2.50798953e-03   2.54169502e-03   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -2.30812542e-02   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   2.62770295e-01   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00  -6.35043755e-02\n",
      "    0.00000000e+00   2.67867523e-04   0.00000000e+00   0.00000000e+00\n",
      "   -7.98880398e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "   -5.92447584e-03   0.00000000e+00  -1.57245338e-01  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "   -5.99138532e-03   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -1.15430495e-03   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    5.05006723e-02   0.00000000e+00  -1.22762471e-03  -8.25707320e-05\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    1.53021584e-03   3.16408545e-01   0.00000000e+00  -0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   8.72864574e-02  -2.11415011e-02\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   2.50212122e-02  -0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -0.00000000e+00  -1.95812702e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   3.41513276e-01  -2.32428148e-01  -1.07846402e-01\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   1.30392673e-05\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   1.74736716e-02   0.00000000e+00\n",
      "    0.00000000e+00  -5.73982179e-01   3.30277056e-01  -4.71663289e-03\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   4.12581898e-02\n",
      "    0.00000000e+00  -1.97707862e-02   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    3.07872593e-02   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -1.33817666e-03\n",
      "   -0.00000000e+00  -0.00000000e+00   2.73743629e-01   0.00000000e+00\n",
      "    0.00000000e+00   5.01265526e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    5.78041732e-01   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00  -1.20691378e-02   4.33936082e-02  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "    2.51757703e-03  -1.13232888e-01  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   3.70023072e-01   0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   -3.10234702e-03  -0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   -5.86915277e-02   0.00000000e+00  -0.00000000e+00   0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import numpy as np\n",
    "\n",
    "sentence_str = \"Diabetes mellitus (DM), commonly referred to as diabetes, is a group of metabolic diseases in which there are high blood sugar levels over a prolonged period.\"\n",
    "sentence_X = gensim.utils.simple_preprocess(sentence_str)\n",
    "starting_idx = 0\n",
    "sentence = sentence_X[starting_idx :maxlen + starting_idx]\n",
    "\n",
    "#start_index = random.randint(0, len(list_words) - maxlen - 1)\n",
    "\n",
    "print()\n",
    "print('----- Example:')\n",
    "generated = ''\n",
    "generated += ' '.join(sentence)\n",
    "print('----- Generating with seed: \"' , sentence , '\"')\n",
    "print()\n",
    "\n",
    "for i in range(3):\n",
    "    x = np.zeros((1, maxlen, len(words)))\n",
    "    for t, word in enumerate(sentence):\n",
    "        x[0, t, word_indices[word]] = 1.\n",
    "\n",
    "    preds = model.predict(x, verbose=0)[0]\n",
    "\n",
    "    del sentence[0]\n",
    "    sentence.append(sentence_X[maxlen+starting_idx+i])\n",
    "    \n",
    "    tic = time.time()\n",
    "    preds = model2.predict(x, verbose=0)[0]\n",
    "    toc = time.time()\n",
    "    print(\"Time : \", toc-tic)\n",
    "    print(preds.shape)\n",
    "    print(preds[:5])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#[ 0.00528679  0.00453413  0.00248272  0.01435476 -0.01339659]\n",
    "#[ 0.02352135  0.03000085  0.01512095 -0.02398492 -0.01188593]\n",
    "model2.reset_states() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'p\\xc3\\xa9p\\xc3\\xa9'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-45-6c29d21b45cb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mtry\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mword_indices\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"pépé\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"hello\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'p\\xc3\\xa9p\\xc3\\xa9'"
     ]
    }
   ],
   "source": [
    "try : \n",
    "    word_indices[\"pépé\"]\n",
    "except KeyError:\n",
    "    pass\n",
    "print(\"hello\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_unknown_words(txt_prep, word_indices):\n",
    "    txt_wo_uw = []\n",
    "    for word in txt_prep:\n",
    "        if word in word_indices.keys():\n",
    "            txt_wo_uw.append(word)\n",
    "    return txt_wo_uw\n",
    "\n",
    "def lstm_infer_vector(lstm_model, txt, stopwords,word_indices, maxlen=10) :\n",
    "    \"\"\"\n",
    "    d2v.infer_vector equivalent for a lstm language model\n",
    "    \"\"\"\n",
    "    \n",
    "    txt_prep = gensim.utils.simple_preprocess(txt, deacc=True)\n",
    "    txt_wo_uw = remove_unknown_words(txt_prep, word_indices)\n",
    "    txt_wo_ws = remove_stopwords(txt_wo_uw, stopwords)\n",
    "    \n",
    "    if len(txt_wo_ws)<maxlen :\n",
    "        #cas du texte trop court\n",
    "        sentence = txt_wo_ws\n",
    "        X = np.zeros((1, maxlen, len(word_indices)), dtype=np.bool)\n",
    "        y = np.zeros((1, len(word_indices)), dtype=np.bool)\n",
    "        for t, word in enumerate(sentence):\n",
    "            X[1, t, word_indices[word]] = 1\n",
    "        preds = model2.predict(X, verbose=0)[0]\n",
    "    else :\n",
    "        \n",
    "        for current_part in range(len(txt_wo_ws)/maxlen):\n",
    "            sentence = txt_wo_ws[current_part*maxlen:(current_part+1)*maxlen]\n",
    "            X = np.zeros((1, maxlen, len(word_indices)), dtype=np.bool)\n",
    "            y = np.zeros((1, len(word_indices)), dtype=np.bool)\n",
    "            for t, word in enumerate(sentence):\n",
    "                X[0, t, word_indices[word]] = 1\n",
    "            preds = model2.predict(X, verbose=0)[0]\n",
    "            \n",
    "\n",
    "    return preds\n",
    "\n",
    "def create_triplets_lstm(lstm_model, article_names, article_weights,stopwords,word_indices, nb_triplets=20, triplets_per_file=5, neg_ratio=0.5, str_mode = False, with_txt_vect = False) :\n",
    "    \"\"\"\n",
    "    inputs :    \n",
    "        - lstm_model : lstm language model \n",
    "        - article_names : ndarray containing the names of the json files (absolute path !)\n",
    "        - article_weights: ndarray normalized of the weight of each files \n",
    "        - nb_triplets : nb of triplets to generate\n",
    "        - triplets_per_file : number of triplet built for each selected file\n",
    "        - neg_ratio : ratio of positives / negative examples. Negative examples are taken inside the article !\n",
    "        \n",
    "    output : \n",
    "        - triplets : nd_array of triplets of shape (nb_triplets+ , embed_dim)\n",
    "        - labels : nd_array of labels of shape (nb_triplets+ ,)\n",
    "\n",
    "    \"\"\"\n",
    "    triplets = []\n",
    "    labels = []\n",
    "    \n",
    "    assert nb_triplets>=triplets_per_file, \"you should have nb_triplets > triplets_per_file\"\n",
    "    \n",
    "    # nb of pos / neg triplets per file\n",
    "    neg_per_file = np.floor(triplets_per_file*neg_ratio) #number of negative triplets to generate given(query + partial summary)\n",
    "    assert neg_per_file >= 1, \"you have to increase your neg_ratio\"\n",
    "    \n",
    "    nb_files = nb_triplets / triplets_per_file\n",
    "    selected_files_array = np.random.choice(article_names, size=nb_files, p=article_weights, replace = False)\n",
    "    \n",
    "    for full_name in selected_files_array :\n",
    "        with open(full_name) as f :\n",
    "            file_as_dict = json.load(f)\n",
    "        \n",
    "        counter = 0\n",
    "        while counter < triplets_per_file :\n",
    "            \n",
    "            # select a key for positive examples\n",
    "            key_pos = select_key(file_as_dict)\n",
    "            \n",
    "            triplet = build_triplet_lstm(lstm_model, file_as_dict, key_pos, positive = True, str_mode = str_mode, with_txt_vect=with_txt_vect)\n",
    "            label = 1\n",
    "            \n",
    "            triplets.append(triplet)\n",
    "            labels.append(label)\n",
    "            counter += 1 \n",
    "            \n",
    "            if neg_ratio < 1 : \n",
    "                \n",
    "                if np.random.rand() < neg_ratio :\n",
    "                    \n",
    "                    triplet = build_triplet_lstm(lstm_model, file_as_dict, key_pos, positive = False, str_mode = str_mode, with_txt_vect=with_txt_vect)\n",
    "                    label = 0\n",
    "                    \n",
    "                    triplets.append(triplet)\n",
    "                    labels.append(label)\n",
    "                    counter += 1 \n",
    "\n",
    "            else :\n",
    "                \n",
    "                for n in range(int(np.floor(neg_ratio))):\n",
    "                    \n",
    "                    triplet = build_triplet_lstm(lstm_model, file_as_dict, key_pos, positive = False, str_mode = str_mode,with_txt_vect=with_txt_vect)\n",
    "                    label = 0\n",
    "                    \n",
    "                    triplets.append(triplet)\n",
    "                    labels.append(label)\n",
    "                    counter += 1 \n",
    "\n",
    "            \n",
    "    triplets = np.asarray(triplets)[:nb_triplets]\n",
    "    labels = np.asarray(labels)[:nb_triplets]\n",
    "    \n",
    "    return triplets, labels\n",
    "\n",
    "def build_triplet_lstm(lstm_model, file_as_dict, key_pos,stopwords,word_indices, positive = True, str_mode = False, remove_stop_words=True, with_txt_vect = False):\n",
    "    if remove_stop_words : \n",
    "        stopwords = stop_words()\n",
    "    else :\n",
    "        stopwords = []\n",
    "        \n",
    "    if with_txt_vect :\n",
    "        text_str = \"\"\n",
    "        for key in file_as_dict.keys():\n",
    "            if key not in non_selected_keys :\n",
    "                text_str += file_as_dict[key]\n",
    "\n",
    "        text_vector = lstm_infer_vector(lstm_model,text_str,stopwords,word_indices)\n",
    "            \n",
    "    query_str = key_pos\n",
    "    query_vector = lstm_infer_vector(lstm_model,query_str,stopwords,word_indices)\n",
    "    \n",
    "    summary_str = file_as_dict[key_pos]\n",
    "    sentences = summary_str.split(\".\")\n",
    "    \n",
    "    partial_summary = []\n",
    "    candidates = []\n",
    "    \n",
    "    size_partial_summary = np.random.rand()\n",
    "    \n",
    "    for sentence in sentences: \n",
    "        if np.random.rand() < size_partial_summary :\n",
    "            partial_summary.append(sentence)\n",
    "        else :\n",
    "            candidates.append(sentence)\n",
    "    \n",
    "    candidate = \"\"\n",
    "    counter_candidate = 0\n",
    "    while (candidate == \"\" or partial_summary == \"\") and counter_candidate < 10:\n",
    "        counter_candidate += 1\n",
    "        \n",
    "        if positive : \n",
    "            if len(candidates) > 0:\n",
    "                random_candidate_index = np.random.randint(0,len(candidates))\n",
    "                candidate = candidates[random_candidate_index]\n",
    "            else :\n",
    "                random_candidate_index = np.random.randint(0,len(partial_summary))\n",
    "                candidate = partial_summary[random_candidate_index]\n",
    "                partial_summary[random_candidate_index] = \"\"\n",
    "\n",
    "\n",
    "            candidate_prep = gensim.utils.simple_preprocess(candidate, deacc=True)\n",
    "            candidate_vector = lstm_infer_vector(lstm_model,candidate,stopwords,word_indices)\n",
    "\n",
    "        else :\n",
    "\n",
    "            key_neg = select_key(file_as_dict)\n",
    "            counter = 0\n",
    "\n",
    "            while key_neg == key_pos and counter<10 : # the counter is for the preproduction code \n",
    "                counter += 1\n",
    "                key_neg = select_key(file_as_dict)\n",
    "\n",
    "            summary_str = file_as_dict[key_neg]\n",
    "\n",
    "            sentences = summary_str.split('.')\n",
    "            random_candidate_index = np.random.randint(0,len(sentences))\n",
    "            candidate = sentences[random_candidate_index]\n",
    "            candidate_vector = lstm_infer_vector(lstm_model,candidate,stopwords,word_indices)\n",
    "        \n",
    "        partial_summary_str = \"\".join(partial_summary)\n",
    "        partial_summary_vector = lstm_infer_vector(lstm_model, partial_summary_str,stopwords,word_indices)\n",
    "    \n",
    "    if str_mode :\n",
    "        return query_str, partial_summary_str, candidate\n",
    "    elif with_txt_vect:\n",
    "        return np.hstack( [query_vector, partial_summary_vector, candidate_vector, text_vector])\n",
    "    else :\n",
    "        return np.hstack( [query_vector, partial_summary_vector, candidate_vector] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('TIme', 1.5339951515197754)\n"
     ]
    }
   ],
   "source": [
    "from functions.training_functions import remove_stopwords, stop_words\n",
    "stopwords= stop_words()\n",
    "tic = time.time()\n",
    "txt = \"Diabetes mellitus (DM), commonly referred to as diabetes, is a group of metabolic diseases in which there are high blood sugar levels over a prolonged period.[2] Symptoms of high blood sugar include frequent urination, increased thirst, and increased hunger. If left untreated, diabetes can cause many complications.[3] Acute complications can include diabetic ketoacidosis, nonketotic hyperosmolar coma, or death.[4] Serious long-term complications include heart disease, stroke, chronic kidney failure, foot ulcers, and damage to the eyes.\"\n",
    "lstm_infer_vector(model2, txt, stopwords,word_indices=word_indices)\n",
    "toc = time.time()\n",
    "print(\"TIme\",toc-tic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def summarize_lstm(text, query, lstm_model,  nn_model, stopwords, word_indices, limit = 250, remove_stop_words = True,with_txt_vect=False):\n",
    "    \"\"\"\n",
    "    Perform summarization on text given query,\n",
    "    \"\"\"\n",
    "    if remove_stop_words : \n",
    "        stopwords = stop_words()\n",
    "    else :\n",
    "        stopwords = []\n",
    "    \n",
    "    if with_txt_vect :\n",
    "        text_vector = lstm_infer_vector(lstm_model, text, stopwords,word_indices)\n",
    "        \n",
    "    query_vector = lstm_infer_vector(lstm_model, query, stopwords,word_indices)\n",
    "    \n",
    "    summary  = \"\"\n",
    "    summary_vector = lstm_infer_vector(lstm_model, [\"\"], stopwords,word_indices)\n",
    "    summary_idx = []\n",
    "    \n",
    "    sentences = text.split('.')\n",
    "    sentences = np.asarray(sentences)\n",
    "    \n",
    "    remaining_sentences = copy.copy(sentences)\n",
    "    \n",
    "    size = 0\n",
    "    counter = 0\n",
    "    while size < limit and len(remaining_sentences)>0 :\n",
    "        counter = counter+1\n",
    "        scores = []\n",
    "        for sentence in remaining_sentences :\n",
    "            sentence_vector = lstm_infer_vector(lstm_model, sentence, stopwords,word_indices)\n",
    "            if with_txt_vect :\n",
    "                nn_input = np.hstack([query_vector, summary_vector, sentence_vector, text_vector])\n",
    "            else:\n",
    "                nn_input = np.hstack([query_vector, summary_vector, sentence_vector])\n",
    "            nn_input = np.asarray([nn_input]) # weird but it is important to do it\n",
    "            score = nn_model.predict(nn_input) \n",
    "            scores.append(score)\n",
    "        #print(scores)\n",
    "        max_idx_rem = int(np.argmax(scores))\n",
    "        idx_selected_sentence = np.arange(len(sentences))[sentences == remaining_sentences[max_idx_rem]]\n",
    "        idx_selected_sentence = int(idx_selected_sentence[0])\n",
    "        size += len(remaining_sentences[max_idx_rem].split())\n",
    "        \n",
    "        remaining_sentences = list(remaining_sentences)\n",
    "        del remaining_sentences[max_idx_rem]\n",
    "        bisect.insort_left(summary_idx,idx_selected_sentence)\n",
    "\n",
    "        summary  = \"\"\n",
    "\n",
    "        for idx in summary_idx:\n",
    "            summary = summary + \" \" + sentences[idx]\n",
    "\n",
    "        summary_vector = lstm_infer_vector(lstm_model, summary, stopwords,word_indices)\n",
    "\n",
    "    return summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[40, 41, 42, 43, 44, 45, 46, 47, 48, 49]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "range(50)[-10:]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}