SentimentAnalysis/jupyternotebook/Doc2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\scott\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "# gensim modules\n",
    "from gensim import utils\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "from gensim.models import Doc2Vec\n",
    "import pandas as pd\n",
    "# numpy\n",
    "import numpy as np\n",
    "# random\n",
    "from random import shuffle\n",
    "# classifier\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# inspecting data size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "NegTrainDataPath = \"..\\\\CleanData\\\\train-neg.txt\"\n",
    "PosTrainDataPath = \"..\\\\CleanData\\\\train-pos.txt\"\n",
    "UnLabelTrainDataPath = \"..\\\\CleanData\\\\train-unsup.txt\"\n",
    "testDataPath = \"..\\\\CleanData\\\\test.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "NegLabelTrainDataFrame = pd.read_csv(NegTrainDataPath,header=None)\n",
    "PosLabelTrainDataFrame = pd.read_csv(PosTrainDataPath,header=None)\n",
    "UnLabelTrainDataFrame = pd.read_csv(UnLabelTrainDataPath,header=None)\n",
    "TestDataFrame = pd.read_csv(testDataPath,header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NegTrainData =  12500\n",
      "PosTrainData =  12500\n",
      "UnLabelTrainData =  50000\n",
      "TestData =  25000\n"
     ]
    }
   ],
   "source": [
    "print(\"NegTrainData = \",NegLabelTrainDataFrame.shape[0])\n",
    "print(\"PosTrainData = \",PosLabelTrainDataFrame.shape[0])\n",
    "print(\"UnLabelTrainData = \",UnLabelTrainDataFrame.shape[0])\n",
    "print(\"TestData = \",TestDataFrame.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train model and save to disk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attention: here I omit some very important step, such as how to choose super parameter, you can check ``Doc2Vec.py`` to see my settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for example, ``mini cout`` is a subtle parameter, I suggest it should be 30 to 50, since each movie occurs 30 times"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "``size`` is another importatne paramter, More features result in longer runtimes, and often, but not always, result in better models. Reasonable values can be in the tens to hundreds; we used 400."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "more tech to select `Doc2vec` parameter can be found in here:https://www.kaggle.com/c/word2vec-nlp-tutorial#part-2-word-vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = {'CleanData\\\\test.txt': 'TEST', 'CleanData\\\\train-neg.txt': 'TRAIN_NEG',\n",
    "            'CleanData\\\\train-pos.txt': 'TRAIN_POS', 'CleanData\\\\train-unsup.txt': 'TRAIN_UNS'}\n",
    "test = Doc2Vec.Train(sources)\n",
    "test.process()\n",
    "test.save('.\\\\Model\\\\imdb.d2v')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## loadmodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Doc2Vec.load('../Persistence/Model/imdb.d2v')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  a sample vector "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.3791196 , -0.9334    ,  1.0988426 , -0.94970757, -1.8186786 ,\n",
       "       -1.6068125 , -0.8571677 , -0.71717787,  0.07672118,  0.3074862 ,\n",
       "       -1.2175689 ,  0.5106428 , -0.1872679 ,  1.1860161 , -2.0082247 ,\n",
       "       -1.118874  ,  0.5909139 , -0.27789843,  0.5407293 ,  1.4340234 ,\n",
       "        0.36014515, -0.56291395, -0.7012361 , -0.8333109 , -1.3679545 ,\n",
       "       -0.50061065, -0.11031017,  1.1482604 , -0.10441066, -2.6325662 ,\n",
       "        0.5159839 , -1.5029888 ,  0.525387  , -0.8092677 ,  1.0439361 ,\n",
       "       -1.8843384 , -1.6593565 , -3.3668585 ,  1.1564244 ,  2.047468  ,\n",
       "       -0.80236375,  2.2020767 ,  1.1480349 , -0.65203184,  0.85826904,\n",
       "        0.3046908 ,  2.2514524 ,  2.5198383 ,  1.4709095 , -0.7628401 ,\n",
       "       -0.8293866 ,  0.4131753 ,  2.7910335 , -0.9634716 ,  0.47208372,\n",
       "        1.3176486 , -0.454331  ,  0.8937898 ,  0.8185652 , -1.3318874 ,\n",
       "        0.00654519,  3.2685728 ,  1.5720541 ,  1.6603485 , -0.29937273,\n",
       "       -1.1018872 ,  1.2067581 ,  1.50389   , -0.5250043 ,  1.2135458 ,\n",
       "       -0.364794  , -0.92752147,  1.0204291 , -0.2616081 ,  2.0649765 ,\n",
       "       -1.0027922 , -0.28190055, -0.5973105 ,  0.9983063 , -2.7704053 ,\n",
       "        0.19924769, -2.167757  , -3.1771643 , -0.43510383,  1.5078566 ,\n",
       "       -0.3577755 ,  2.0284762 ,  1.3232058 ,  0.35584742, -2.569906  ,\n",
       "       -1.4898775 , -1.6096723 , -1.485084  ,  2.7474072 , -2.198397  ,\n",
       "       -1.1801085 ,  2.543942  , -1.3986951 , -1.4014239 , -2.5267432 ,\n",
       "        3.0916524 ,  2.0659893 ,  1.4344174 ,  0.08690822,  1.1604383 ,\n",
       "       -0.86459345, -0.76441264,  1.9546925 , -0.44301403, -1.3457329 ,\n",
       "       -1.2183568 , -1.3937595 , -0.4233679 ,  0.3788009 , -1.9568337 ,\n",
       "        1.3506478 ,  1.190381  , -0.81284195, -0.66690814,  1.9173471 ,\n",
       "       -1.2565852 ,  0.6100571 , -0.264243  ,  2.1727402 , -0.08879408,\n",
       "       -0.5872505 ,  0.08308972,  1.3827897 , -2.2509773 ,  1.5287042 ,\n",
       "        3.1392598 , -0.06602587,  2.5731761 , -0.9457149 ,  0.05050765,\n",
       "       -0.76676077, -0.51984257, -0.31648698,  0.13072895, -0.70872146,\n",
       "        0.04510385,  0.6782712 , -1.421535  ,  0.53453606, -1.2934453 ,\n",
       "        1.1208673 , -1.0428891 , -3.1155915 ,  0.44040138,  0.1394977 ,\n",
       "       -1.6017733 , -0.18511191,  1.2417628 , -1.7474003 ,  1.3398858 ,\n",
       "        2.413028  , -1.2628695 , -0.2378459 , -2.6008668 ,  0.24473956,\n",
       "       -1.4391084 , -0.30668652, -0.0045456 , -0.76526916,  0.47802952,\n",
       "        2.2809858 , -0.2048242 ,  1.0081532 ,  1.225347  , -1.0643945 ,\n",
       "       -1.7719983 ,  1.0387142 ,  1.0881276 ,  0.62940276,  1.1767128 ,\n",
       "       -0.54817176, -1.8500389 ,  0.9411553 ,  0.9532948 ,  1.6827029 ,\n",
       "        0.55970573,  2.9660923 , -1.4253227 ,  0.29344216,  0.6443064 ,\n",
       "       -1.1197181 ,  0.4028988 ,  1.8817662 , -0.16575207,  1.3108958 ,\n",
       "       -0.7576621 , -0.10731664,  0.67218673,  2.897797  , -1.1077874 ,\n",
       "       -0.8653067 , -1.14485   ,  0.2973909 ,  1.1639211 , -0.26173764,\n",
       "        0.27158803,  0.44247824, -1.3524957 ,  1.4308692 ,  3.0302463 ,\n",
       "        0.2881249 ,  0.12186033,  0.89399344, -2.0473688 ,  0.780323  ,\n",
       "       -2.0990815 , -0.30266398, -0.07020292, -3.137376  , -0.49733862,\n",
       "       -0.8198958 , -0.16402504,  0.40699053,  1.3701477 , -0.29300544,\n",
       "       -0.86621034,  0.7763802 , -1.2379733 , -2.9606435 , -0.9752233 ,\n",
       "        0.6211673 , -0.7024692 , -0.7032941 , -0.15036218,  1.7898546 ,\n",
       "        0.7534321 ,  0.78699464,  0.9634438 ,  0.43122733, -0.36484733,\n",
       "       -1.2064921 , -0.08924502,  1.5973486 , -0.07193011, -0.12310762,\n",
       "        1.060969  ,  0.37843725, -1.4110681 ,  2.2446704 , -1.9215816 ,\n",
       "       -1.6958077 , -1.7469727 ,  0.8324075 ,  1.6314638 ,  0.6250182 ,\n",
       "        1.9343518 , -1.9346405 , -0.09110817,  1.0350789 ,  2.8271952 ,\n",
       "        0.08730811, -0.08243645, -0.4660558 , -0.19050173,  1.0007923 ,\n",
       "       -1.0247525 , -2.700342  , -0.1792636 ,  1.4172086 ,  0.9786776 ,\n",
       "        1.1069558 ,  2.1510496 ,  1.6349391 ,  0.63199407, -1.9541458 ,\n",
       "        0.2673156 , -2.8789375 , -2.2139044 ,  0.16654171, -2.085294  ,\n",
       "        0.6249318 , -0.26173973,  0.54492104,  2.0745878 ,  0.19890492,\n",
       "       -2.1377254 , -0.3198285 ,  0.6214506 ,  2.4802177 ,  0.42567712,\n",
       "       -1.0611762 , -0.05543269, -0.2649233 ,  1.3826969 ,  2.713165  ,\n",
       "       -0.69977194, -0.56113243, -1.4970077 , -0.973306  ,  0.80354017,\n",
       "       -3.097287  ,  1.8772416 , -0.84815145, -0.17692314, -0.65490395],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model['TRAIN_NEG_0']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspecting the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### similar words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "check sentiment words, like ``good``,``suck``,find the most similar words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('great', 0.5707298517227173),\n",
       " ('decent', 0.5701993703842163),\n",
       " ('bad', 0.4825226664543152),\n",
       " ('fine', 0.4680075943470001),\n",
       " ('nice', 0.4640767574310303),\n",
       " ('terrific', 0.40220221877098083),\n",
       " ('cool', 0.3792990744113922),\n",
       " ('solid', 0.375804603099823),\n",
       " ('impressive', 0.3720175623893738),\n",
       " ('weak', 0.37139537930488586)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar('good')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('terrible', 0.5165358781814575),\n",
       " ('horrible', 0.5109846591949463),\n",
       " ('awful', 0.48885127902030945),\n",
       " ('good', 0.4825226664543152),\n",
       " ('poor', 0.42794835567474365),\n",
       " ('stupid', 0.4101775288581848),\n",
       " ('lousy', 0.39594411849975586),\n",
       " ('inept', 0.37407028675079346),\n",
       " ('horrid', 0.36204835772514343),\n",
       " ('atrocious', 0.35902562737464905)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar('bad')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "seem pretty good, since our analysis depends on the sentiment words, so their relation is rather important"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### match words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "to check model robust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'kitchen'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.doesnt_match(\"man woman child kitchen\".split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('girl', 0.639438271522522),\n",
       " ('sister', 0.3268493413925171),\n",
       " ('daughter', 0.32179757952690125),\n",
       " ('female', 0.3057842254638672),\n",
       " ('child', 0.30202263593673706),\n",
       " ('boys', 0.28385084867477417),\n",
       " ('girls', 0.27956512570381165),\n",
       " ('kid', 0.27465373277664185),\n",
       " ('babe', 0.26127684116363525),\n",
       " ('sexual', 0.2572239339351654)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['woman','boy'],negative=['man'],topn=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### model corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19312"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.wv.vectors_norm )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run `tensorboard --logdir=..\\Model\\visual` to run visualize result on tensorboard\n"
     ]
    }
   ],
   "source": [
    "import sys, os\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')\n",
    "from gensim.models import Word2Vec\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.contrib.tensorboard.plugins import projector\n",
    "\n",
    "def visualize(model, output_path):\n",
    "    meta_file = \"w2x_metadata.tsv\"\n",
    "    placeholder = np.zeros((len(model.wv.index2word), model.wv.vector_size))\n",
    "\n",
    "    with open(os.path.join(output_path,meta_file), 'wb') as file_metadata:\n",
    "        for i, word in enumerate(model.wv.index2word):\n",
    "            placeholder[i] = model[word]\n",
    "            # temporary solution for https://github.com/tensorflow/tensorflow/issues/9094\n",
    "            if word == '':\n",
    "                print(\"Emply Line, should replecaed by any thing else, or will cause a bug of tensorboard\")\n",
    "                file_metadata.write(\"{0}\".format('<Empty Line>').encode('utf-8') + b'\\n')\n",
    "            else:\n",
    "                file_metadata.write(\"{0}\".format(word).encode('utf-8') + b'\\n')\n",
    "\n",
    "    # define the model without training\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    embedding = tf.Variable(placeholder, trainable = False, name = 'w2x_metadata')\n",
    "    tf.global_variables_initializer().run()\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    writer = tf.summary.FileWriter(output_path, sess.graph)\n",
    "\n",
    "    # adding into projector\n",
    "    config = projector.ProjectorConfig()\n",
    "    embed = config.embeddings.add()\n",
    "    embed.tensor_name = 'w2x_metadata'\n",
    "    embed.metadata_path = meta_file\n",
    "\n",
    "    # Specify the width and height of a single thumbnail.\n",
    "    projector.visualize_embeddings(writer, config)\n",
    "    saver.save(sess, os.path.join(output_path,'w2x_metadata.ckpt'))\n",
    "    print('Run `tensorboard --logdir={0}` to run visualize result on tensorboard'.format(output_path))\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    model = Word2Vec.load(\"..\\\\Model\\\\imdb.d2v\")\n",
    "    visualize(model,\"..\\\\Model\\\\visual\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run `tensorboard --logdir=..\\Model\\visual` to run visualize result on tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![52776363609](http://wx4.sinaimg.cn/mw690/0060lm7Tly1fs1iq0ghwzj31gq0pt12q.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### review \"suck\" using euclidean  distance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![52776398234](http://wx1.sinaimg.cn/mw690/0060lm7Tly1fs1iozlqgej30v10m4mym.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# save vector for LSTM "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for LSTM use, we need to convert document to a list of vector, I choose to vectorize firt `1000` words as their feature, so every document has `1000` feature, each feature is a `400` length vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SaveDataForRNN(self, feature=500):\n",
    "        \"\"\"\n",
    "        save X_train,Y_train,X_test to ``./Persistence/``\\n\n",
    "        X_train.shape = (25000,feature_size,model.vector.size)\\n\n",
    "        Y_train.shape = (25000,2)  [1,0] for ``positive`` [0,1] for ``negative``\\n\n",
    "        too big to return, read data from ``./Persistence``\n",
    "        \"\"\"\n",
    "        sources = {'CleanData\\\\test.txt': 'TEST', 'CleanData\\\\train-neg.txt': 'TRAIN_NEG',\n",
    "                   'CleanData\\\\train-pos.txt': 'TRAIN_POS', 'CleanData\\\\train-unsup.txt': 'TRAIN_UNS'}\n",
    "\n",
    "        # dict sentences_dict[\"TRAIN_NEG_0\"] = [\"a\",\"b\",...\"last word\"]\n",
    "        sentences = LabeledLineSentence(sources)\n",
    "        sentences_array = sentences.to_array()\n",
    "        sentences_dict = {}\n",
    "        for i in range(0, len(sentences_array)):\n",
    "            sentences_dict[sentences_array[i][1][0]] = sentences_array[i][0]\n",
    "\n",
    "        # Index2word is a list that contains the names of the words in\n",
    "        # the model's vocabulary. Convert it to a set, for speed\n",
    "        index2word_set = set(self.model.wv.index2word)\n",
    "\n",
    "        # when short of word, using zero instead\n",
    "        empty_word = np.zeros(self.model.vector_size,dtype='float16')\n",
    "\n",
    "\n",
    "        X_train = np.zeros((25000, feature, self.model.vector_size),dtype='float16')\n",
    "        Y_train = np.zeros(25000)\n",
    "\n",
    "\n",
    "\n",
    "        # get first 1000 feature vectors from 1000 words\n",
    "        for i in range(12500):\n",
    "            prefix_train_pos = 'TRAIN_POS_' + str(i)\n",
    "            prefix_train_neg = 'TRAIN_NEG_' + str(i)\n",
    "            # length of document\n",
    "            len1 = len(sentences_dict[prefix_train_pos])\n",
    "            len2 = len(sentences_dict[prefix_train_neg])\n",
    "            cout = j = 0\n",
    "            # for pos document\n",
    "            while cout < feature:\n",
    "                if j < len1:\n",
    "                    word = sentences_dict[prefix_train_pos][j]\n",
    "                    if word in index2word_set:\n",
    "                        X_train[i, cout, :] = self.model[word]\n",
    "                        cout += 1\n",
    "                else:\n",
    "                    X_train[i, cout, :] = empty_word\n",
    "                    cout += 1\n",
    "                j += 1\n",
    "\n",
    "            # reset j/cout, for neg document\n",
    "            cout = j = 0\n",
    "            while cout < feature:\n",
    "                if j < len2:\n",
    "                    word = sentences_dict[prefix_train_neg][j]\n",
    "                    if word in index2word_set:\n",
    "                        X_train[12500+i, cout, :] = self.model[word]\n",
    "                        cout += 1\n",
    "                else:\n",
    "                    X_train[12500+i, cout, :] = empty_word\n",
    "                    cout += 1\n",
    "                j += 1\n",
    "\n",
    "            Y_train[i] = 1\n",
    "            Y_train[12500 + i] = 0\n",
    "\n",
    "        with open('./Persistence/LSTM/X_train.pickle', 'wb') as f:\n",
    "            pickle.dump(X_train, f, protocol=4)\n",
    "            f.close()\n",
    "\n",
    "        with open('./Persistence/LSTM/Y_train.pickle', 'wb') as f:\n",
    "            pickle.dump(Y_train, f, protocol=4)\n",
    "            f.close()\n",
    "\n",
    "        X_test =np.zeros((25000, feature, self.model.vector_size),dtype='float16')\n",
    "\n",
    "        for i in range(25000):\n",
    "            prefix_test = 'TEST_' + str(i)\n",
    "            len1 = len(sentences_dict[prefix_test])\n",
    "            cout = j = 0\n",
    "            while cout < feature:\n",
    "                if j < len1:\n",
    "                    word = sentences_dict[prefix_test][j]\n",
    "                    if word in index2word_set:\n",
    "                        X_test[i, cout, :] = self.model[word]\n",
    "                        cout += 1\n",
    "                else:\n",
    "                    X_test[i, cout, :] = empty_word\n",
    "                    cout += 1\n",
    "                j += 1\n",
    "\n",
    "        with open('./Persistence/LSTM/X_test.pickle', 'wb') as f:\n",
    "            pickle.dump(X_test, f, protocol=4)\n",
    "            f.close()\n",
    "\n",
    "        print(\"save narray success to ./Persistence\")\n",
    "\n",
    "        # return X_train,Y_train,X_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}