Proj3_Data_Gen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Sentiment Analysis v2: Include Sum/Average/WeightSum/Max to replace Flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic RNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keras 2.0.8\n",
      "tensorflow 1.3.0\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "## To get helper functions from ano`ther folder\n",
    "# sys.path.insert(0, '../') # if _helper_basics_ is in previous folder\n",
    "now_i_am_at = 'home' # home dso test\n",
    "if now_i_am_at=='home':\n",
    "    sys.path.insert(0, 'E:/Leonard HDD/Dropbox/DSO/Tasks/')\n",
    "elif now_i_am_at=='dso':\n",
    "    sys.path.insert(0, 'D:/Dropbox/DSO/Tasks')\n",
    "\n",
    "from _helper_basics_ import *\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['savefig.dpi'] = 100\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "print('keras', keras.__version__)\n",
    "print('tensorflow', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# imdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train_imdb (25000,)\n",
      "y_train_imdb (25000,)\n",
      "x_test_imdb  (25000,)\n",
      "y_test_imdb  (25000,)\n",
      "len(x_train_imdb[0]) 218\n",
      "len(x_train_imdb[1]) 189\n",
      "\n",
      "Pad sequences: (samples x time)\n",
      "X_train_imdb shape: (25000, 80)\n",
      "Pad sequences: (samples x time)\n",
      "Y_train_imdb shape: (25000, 2)\n",
      "\n",
      "X_train_imdb (25000, 80)\n",
      "Y_train_imdb (25000, 2)\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import imdb\n",
    "\n",
    "num_words=20000 # max is 88586\n",
    "skip_top=0\n",
    "maxlen = 80 # in padding\n",
    "\n",
    "(x_train_imdb, y_train_imdb), (x_test_imdb, y_test_imdb) = imdb.load_data(path=\"imdb.npz\",\n",
    "                                                      num_words=num_words,\n",
    "                                                      skip_top=skip_top,\n",
    "                                                      maxlen=None,\n",
    "                                                      seed=113,\n",
    "                                                      start_char=1,\n",
    "                                                      oov_char=2,\n",
    "                                                      index_from=3)\n",
    "print('x_train_imdb', x_train_imdb.shape); print('y_train_imdb', y_train_imdb.shape)\n",
    "print('x_test_imdb ', x_test_imdb.shape);  print('y_test_imdb ', y_test_imdb.shape)\n",
    "print('len(x_train_imdb[0])', len(x_train_imdb[0]))\n",
    "print('len(x_train_imdb[1])', len(x_train_imdb[1]))\n",
    "\n",
    "## Get word indices\n",
    "word_index_imdb = imdb.get_word_index(path=\"word_index_imdb.json\")\n",
    "\n",
    "\n",
    "################## Padding sequences ##################\n",
    "print('\\nPad sequences: (samples x time)')\n",
    "X_train_imdb = pad_sequences(x_train_imdb, maxlen=maxlen)\n",
    "# X_test_imdb  = pad_sequences(x_test_imdb, maxlen=maxlen)\n",
    "print('X_train_imdb shape:', X_train_imdb.shape)\n",
    "# print('X_test_imdb shape: ', X_test_imdb.shape)\n",
    "\n",
    "print('Pad sequences: (samples x time)')\n",
    "Y_train_imdb = to_categorical(y_train_imdb, 2)\n",
    "# Y_test_imdb  = to_categorical(y_test_imdb, 46)\n",
    "print('Y_train_imdb shape:', Y_train_imdb.shape)\n",
    "# print('Y_test_imdb shape: ', Y_test_imdb.shape)\n",
    "\n",
    "\n",
    "################## Saving the XY_train ##################\n",
    "print('\\nX_train_imdb', X_train_imdb.shape)\n",
    "print('Y_train_imdb', Y_train_imdb.shape)\n",
    "# print('X_test_imdb ', X_test_imdb.shape)\n",
    "# print('Y_test_imdb ', Y_test_imdb.shape)\n",
    "data_mode = 'imdb'\n",
    "Data_dir = 'Data_'+data_mode+'/'\n",
    "if not os.path.exists(Data_dir): os.mkdir(Data_dir)\n",
    "np.save(Data_dir+'X_train_imdb',X_train_imdb)\n",
    "np.save(Data_dir+'Y_train_imdb',Y_train_imdb)\n",
    "# np.save(Data_dir+'X_test_imdb',X_test_imdb)\n",
    "# np.save(Data_dir+'Y_test_imdb',Y_test_imdb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'<oov_char>'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-103-c8be1d39160a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mword_index_imdb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'<oov_char>'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m: '<oov_char>'"
     ]
    }
   ],
   "source": [
    "word_index_imdb['<oov_char>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Make dictionary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<start_char>'"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num2word_imdb[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 out of 88584\n",
      "5000 out of 88584\n",
      "10000 out of 88584\n",
      "15000 out of 88584\n",
      "20000 out of 88584\n",
      "25000 out of 88584\n",
      "30000 out of 88584\n",
      "35000 out of 88584\n",
      "40000 out of 88584\n",
      "45000 out of 88584\n",
      "50000 out of 88584\n",
      "55000 out of 88584\n",
      "60000 out of 88584\n",
      "65000 out of 88584\n",
      "70000 out of 88584\n",
      "75000 out of 88584\n",
      "80000 out of 88584\n",
      "85000 out of 88584\n"
     ]
    }
   ],
   "source": [
    "num2word_imdb = ['','<start_char>','<oov_char>']\n",
    "for idx in range(len(word_index_imdb)):\n",
    "    if idx%5000==0: print(idx,'out of',len(word_index_imdb))\n",
    "    for x in word_index:\n",
    "        if not word_index[x]==(idx+1): \n",
    "            continue\n",
    "        else:\n",
    "            num2word_imdb.append(x)\n",
    "            break\n",
    "num2word_imdb = num2word_imdb[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30981"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(num2word_imdb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "14\n",
      "22\n",
      "16\n",
      "43\n",
      "530\n",
      "973\n",
      "1622\n",
      "1385\n",
      "65\n",
      "458\n",
      "4468\n",
      "66\n",
      "3941\n",
      "4\n",
      "173\n",
      "36\n",
      "256\n",
      "5\n",
      "25\n",
      "100\n",
      "43\n",
      "838\n",
      "112\n",
      "50\n",
      "670\n",
      "22665\n",
      "9\n",
      "35\n",
      "480\n",
      "284\n",
      "5\n",
      "150\n",
      "4\n",
      "172\n",
      "112\n",
      "167\n",
      "21631\n",
      "336\n",
      "385\n",
      "39\n",
      "4\n",
      "172\n",
      "4536\n",
      "1111\n",
      "17\n",
      "546\n",
      "38\n",
      "13\n",
      "447\n",
      "4\n",
      "192\n",
      "50\n",
      "16\n",
      "6\n",
      "147\n",
      "2025\n",
      "19\n",
      "14\n",
      "22\n",
      "4\n",
      "1920\n",
      "4613\n",
      "469\n",
      "4\n",
      "22\n",
      "71\n",
      "87\n",
      "12\n",
      "16\n",
      "43\n",
      "530\n",
      "38\n",
      "76\n",
      "15\n",
      "13\n",
      "1247\n",
      "4\n",
      "22\n",
      "17\n",
      "515\n",
      "17\n",
      "12\n",
      "16\n",
      "626\n",
      "18\n",
      "19193\n",
      "5\n",
      "62\n",
      "386\n",
      "12\n",
      "8\n",
      "316\n",
      "8\n",
      "106\n",
      "5\n",
      "4\n",
      "2223\n",
      "5244\n",
      "16\n",
      "480\n",
      "66\n",
      "3785\n",
      "33\n",
      "4\n",
      "130\n",
      "12\n",
      "16\n",
      "38\n",
      "619\n",
      "5\n",
      "25\n",
      "124\n",
      "51\n",
      "36\n",
      "135\n",
      "48\n",
      "25\n",
      "1415\n",
      "33\n",
      "6\n",
      "22\n",
      "12\n",
      "215\n",
      "28\n",
      "77\n",
      "52\n",
      "5\n",
      "14\n",
      "407\n",
      "16\n",
      "82\n",
      "10311\n",
      "8\n",
      "4\n",
      "107\n",
      "117\n",
      "5952\n",
      "15\n",
      "256\n",
      "4\n",
      "31050\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-87-454e3256ea9d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx_i\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mx_train_imdb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_i\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0msent\u001b[0m\u001b[1;33m+=\u001b[0m\u001b[1;34m' '\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mnum2word_imdb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mx_i\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m \u001b[0msent\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m \u001b[0mx_train_imdb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "idx = 0\n",
    "sent = ''\n",
    "for x_i in x_train_imdb[idx]:\n",
    "    print(x_i)\n",
    "    sent+=' '+num2word_imdb[x_i-2]\n",
    "print( sent )\n",
    "print( x_train_imdb[0] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_word_index = 1\n",
    "for x in x_train_imdb:\n",
    "    for xx in x:\n",
    "        if xx > max_word_index: \n",
    "            max_word_index = xx + 0\n",
    "max_word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[    1   778   128    74    12   630   163    15     4  1766  7982  1051\n",
      " 43222    32    85   156    45    40   148   139   121   664   665    10\n",
      "    10  1361   173     4   749 86588    16  3804     8     4   226    65\n",
      "    12    43   127    24 15344    10    10]\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "print(np.asarray(x_train[0]).T)\n",
    "print(y_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pad sequences: (samples x time)\n",
      "X_train_imdb shape: (25000, 80)\n",
      "Pad sequences: (samples x time)\n",
      "Y_train_imdb shape: (25000, 2)\n"
     ]
    }
   ],
   "source": [
    "print('Pad sequences: (samples x time)')\n",
    "maxlen = 80\n",
    "X_train_imdb = pad_sequences(x_train_imdb, maxlen=maxlen)\n",
    "# X_test_imdb  = pad_sequences(x_test_imdb, maxlen=maxlen)\n",
    "print('X_train_imdb shape:', X_train_imdb.shape)\n",
    "# print('X_test_imdb shape: ', X_test_imdb.shape)\n",
    "\n",
    "print('Pad sequences: (samples x time)')\n",
    "Y_train_imdb = to_categorical(y_train_imdb, 2)\n",
    "# Y_test_imdb  = to_categorical(y_test_imdb, 46)\n",
    "print('Y_train_imdb shape:', Y_train_imdb.shape)\n",
    "# print('Y_test_imdb shape: ', Y_test_imdb.shape)\n",
    "\n",
    "print('X_train_imdb', X_train_imdb.shape)\n",
    "print('Y_train_imdb', Y_train_imdb.shape)\n",
    "# print('X_test_imdb ', X_test_imdb.shape)\n",
    "# print('Y_test_imdb ', Y_test_imdb.shape)\n",
    "data_mode = 'imdb'\n",
    "Data_dir = 'Data_'+data_mode+'/'\n",
    "if not os.path.exists(Data_dir): os.mkdir(Data_dir)\n",
    "np.save(Data_dir+'X_train_imdb',X_train_imdb)\n",
    "np.save(Data_dir+'Y_train_imdb',Y_train_imdb)\n",
    "# np.save(Data_dir+'X_test_imdb',X_test_imdb)\n",
    "# np.save(Data_dir+'Y_test_imdb',Y_test_imdb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_imdb (25000, 80)\n",
      "Y_train_imdb (25000, 2)\n"
     ]
    }
   ],
   "source": [
    "print('X_train_imdb', X_train_imdb.shape)\n",
    "print('Y_train_imdb', Y_train_imdb.shape)\n",
    "# print('X_test_imdb ', X_test_imdb.shape)\n",
    "# print('Y_test_imdb ', Y_test_imdb.shape)\n",
    "data_mode = 'imdb'\n",
    "Data_dir = 'Data_'+data_mode+'/'\n",
    "if not os.path.exists(Data_dir): os.mkdir(Data_dir)\n",
    "np.save(Data_dir+'X_train_imdb',X_train_imdb)\n",
    "np.save(Data_dir+'Y_train_imdb',Y_train_imdb)\n",
    "# np.save(Data_dir+'X_test_imdb',X_test_imdb)\n",
    "# np.save(Data_dir+'Y_test_imdb',Y_test_imdb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12500 25000 0.5\n"
     ]
    }
   ],
   "source": [
    "## Percentage of the class 3\n",
    "count_3 = 0\n",
    "for y in y_train_imdb:\n",
    "    if y==0: count_3+=1\n",
    "        \n",
    "print(count_3, len(y_train_imdb), count_3/len(y_train_imdb))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.datasets import reuters\n",
    "\n",
    "num_words=None # 100\n",
    "skip_top=0\n",
    "maxlen=None # 1000 2375 None\n",
    "test_split=0.2\n",
    "(x_train_reu, y_train_reu), (x_test_reu, y_test_reu) = reuters.load_data(path=\"reuters.npz\",\n",
    "                                                         num_words=num_words,\n",
    "                                                         skip_top=skip_top,\n",
    "                                                         maxlen=maxlen,\n",
    "                                                         test_split=test_split,\n",
    "                                                         seed=113,\n",
    "                                                         start_char=1,\n",
    "                                                         oov_char=2,\n",
    "                                                         index_from=3)\n",
    "word_index_reuters = reuters.get_word_index(path=\"word_index_reuters.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3159 8982 0.3517034068136273\n"
     ]
    }
   ],
   "source": [
    "## Percentage of the class 3\n",
    "count_3 = 0\n",
    "for y in y_train_reu:\n",
    "    if y==3: count_3+=1\n",
    "        \n",
    "print(count_3, len(y_train_reu), count_3/len(y_train_reu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train_reu (8982,)\n",
      "y_train_reu (8982,)\n",
      "x_test_reu  (2246,)\n",
      "y_test_reu  (2246,)\n",
      "\n",
      "len(x_train_reu[0]) 87\n",
      "len(x_train_reu[1]) 56\n",
      "y_train_reu[0] 3\n",
      "y_train_reu[1] 4\n",
      "\n",
      "len(x_test_reu[0]) 145\n",
      "len(x_test_reu[1]) 745\n",
      "y_test_reu[0] 3\n",
      "y_test_reu[1] 10\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'france': 775,\n",
       " 'beaumont': 13126,\n",
       " 'reopened': 6560,\n",
       " 'edict': 24509,\n",
       " 'ellemann': 27703,\n",
       " \"cairo's\": 23603,\n",
       " 'irna': 5395,\n",
       " 'arbb': 22325,\n",
       " 'voutilainen': 23095,\n",
       " 'piers': 23809,\n",
       " 'subjective': 23087,\n",
       " 'denotes': 22852,\n",
       " 'pipework': 21401,\n",
       " \"sosnoff's\": 6717,\n",
       " \"lisbon's\": 15270,\n",
       " 'possess': 29841,\n",
       " 'umar': 13305,\n",
       " 'stereotype': 25218,\n",
       " 'oglebay': 27052,\n",
       " 'walkout': 14598,\n",
       " '911': 5507,\n",
       " 'mulroney': 5542,\n",
       " 'paraguayan': 28343,\n",
       " 'anxiously': 17168,\n",
       " 'monitoring': 4628,\n",
       " 'among': 647,\n",
       " \"clevite's\": 10739,\n",
       " '158': 3317,\n",
       " '176': 3256,\n",
       " 'quilon': 23633,\n",
       " 'smithson': 7742,\n",
       " 'investcorp': 25282,\n",
       " 'danzar': 15813,\n",
       " 'diversify': 4426,\n",
       " 'gill': 11259,\n",
       " 'middling': 11321,\n",
       " 'anonima': 18061,\n",
       " 'luigi': 26027,\n",
       " 'genuinely': 18027,\n",
       " 'procedures': 2917,\n",
       " 'subdue': 17814,\n",
       " 'mainenance': 29638,\n",
       " 'gobbled': 19894,\n",
       " \"tomorrow's\": 5924,\n",
       " '790': 4826,\n",
       " 'vishay': 29245,\n",
       " 'postmaster': 20781,\n",
       " 'francisco': 4168,\n",
       " 'gasoil': 16626,\n",
       " 'sign': 1712,\n",
       " 'profile': 8028,\n",
       " 'reaching': 3036,\n",
       " 'him': 1823,\n",
       " 'verdal': 18627,\n",
       " 'feeling': 5115,\n",
       " 'oral': 13190,\n",
       " 'a1': 23481,\n",
       " 'camps': 17709,\n",
       " 'merrill': 1408,\n",
       " 'if': 104,\n",
       " 'housewives': 10540,\n",
       " 'macedo': 26778,\n",
       " 'tenera': 19147,\n",
       " 'humphreys': 29986,\n",
       " 'smooth': 8487,\n",
       " 'mosar': 17235,\n",
       " 'cuts': 877,\n",
       " 'disallowed': 15442,\n",
       " 'shuttle': 14895,\n",
       " 'norgold': 11849,\n",
       " \"exportfinans'\": 25010,\n",
       " 'assumptrion': 30664,\n",
       " 'legended': 20297,\n",
       " 'reactivation': 17941,\n",
       " \"khashoggi's\": 15901,\n",
       " 'liquor': 7768,\n",
       " 'typhi': 26936,\n",
       " \"ncnb's\": 30830,\n",
       " 'minimised': 17984,\n",
       " 'groupe': 9047,\n",
       " 'conflict': 3520,\n",
       " 'chamorro': 19960,\n",
       " \"deal's\": 11770,\n",
       " 'franciso': 25665,\n",
       " 'nikolay': 21038,\n",
       " 'beans': 2707,\n",
       " 'wendt': 16841,\n",
       " 'boosts': 18447,\n",
       " 'orchards': 18360,\n",
       " 'multimillion': 19451,\n",
       " 'picked': 6863,\n",
       " 'conversative': 23907,\n",
       " 'catchup': 26039,\n",
       " 'sese': 24050,\n",
       " 'quite': 2620,\n",
       " 'goerdeler': 19531,\n",
       " 'imprints': 25057,\n",
       " '1978': 3147,\n",
       " 'slump': 3676,\n",
       " 'selvage': 26343,\n",
       " 'pfennig': 20323,\n",
       " 'flies': 9690,\n",
       " 'grande': 7957,\n",
       " 'trifles': 30741,\n",
       " 'saad': 30111,\n",
       " 'kandeel': 27404,\n",
       " 'bauer': 16895,\n",
       " 'alcasa': 14429,\n",
       " 'prostate': 22409,\n",
       " 'thorugh': 29056,\n",
       " \"customers'\": 14157,\n",
       " 'units': 668,\n",
       " 'initial': 940,\n",
       " '1983': 1597,\n",
       " 'inadequate': 3280,\n",
       " 'vacant': 11313,\n",
       " 'grenada': 17150,\n",
       " 'decoration': 25771,\n",
       " 'alcoa': 8260,\n",
       " 'escort': 4806,\n",
       " 'oilfied': 19909,\n",
       " 'low': 453,\n",
       " '413p': 30693,\n",
       " 'komatsu': 8076,\n",
       " 'expiration': 3502,\n",
       " 'p': 1211,\n",
       " 'semi': 2167,\n",
       " 'grantes': 22808,\n",
       " 'vienot': 16791,\n",
       " 'pagecorp': 13334,\n",
       " 'slightly': 835,\n",
       " 'represent': 2743,\n",
       " 'ayll': 18291,\n",
       " '2267': 17143,\n",
       " 'knives': 24683,\n",
       " 'significant': 919,\n",
       " 'patching': 22655,\n",
       " 'callebaut': 9522,\n",
       " 'horse': 10155,\n",
       " 'rotting': 16258,\n",
       " '21950': 22957,\n",
       " 'seidman': 9111,\n",
       " 'gauge': 8464,\n",
       " 'porsche': 6016,\n",
       " 'unionized': 9463,\n",
       " 'reenter': 12854,\n",
       " 'springfield': 19427,\n",
       " 'jonahi': 24252,\n",
       " \"xebec's\": 25943,\n",
       " 'incomex': 23559,\n",
       " 'preserved': 12333,\n",
       " 'christians': 7488,\n",
       " 'norris': 11148,\n",
       " 'treading': 25310,\n",
       " 'described': 1849,\n",
       " \"o'clock\": 16638,\n",
       " 'erc': 13953,\n",
       " 'fortin': 20774,\n",
       " 'segment': 3514,\n",
       " 'ccc': 1638,\n",
       " 'justifiably': 23040,\n",
       " 'includes': 232,\n",
       " 'entreprise': 28711,\n",
       " 'shells': 7828,\n",
       " 'andf': 22984,\n",
       " 'tell': 3211,\n",
       " 'fred': 6409,\n",
       " 'practical': 7934,\n",
       " 'informations': 16192,\n",
       " 'dixie': 14227,\n",
       " 'johsen': 16318,\n",
       " 'dietz': 17112,\n",
       " 'divided': 3338,\n",
       " 'kvil': 25786,\n",
       " 'input': 8705,\n",
       " 'incident': 6849,\n",
       " 'guillonneau': 18897,\n",
       " 'digesting': 23122,\n",
       " 'nominees': 9112,\n",
       " 'underscored': 10056,\n",
       " 'yohai': 25655,\n",
       " 'mcdowell': 9681,\n",
       " 'varieties': 6624,\n",
       " 'garcia': 5218,\n",
       " 'classification': 13365,\n",
       " 'lightning': 10895,\n",
       " 'markus': 20255,\n",
       " 'yet': 833,\n",
       " 'ope': 12745,\n",
       " 'can': 327,\n",
       " 'gaons': 22304,\n",
       " 'improves': 11850,\n",
       " \"cetus'\": 15449,\n",
       " 'improperly': 12552,\n",
       " 'sperry': 7178,\n",
       " 'peup': 20709,\n",
       " 'caobisco': 20075,\n",
       " \"plains'\": 22472,\n",
       " 'ost': 12114,\n",
       " 'bastian': 10488,\n",
       " 'marketer': 9144,\n",
       " 'soyabeans': 22793,\n",
       " 'ploughing': 14420,\n",
       " 'injects': 27011,\n",
       " 'lipper': 13320,\n",
       " 'overheads': 12963,\n",
       " 'varian': 15462,\n",
       " 'atkinson': 28349,\n",
       " 'coronet': 20279,\n",
       " 'lucinda': 27589,\n",
       " 'mpany': 26583,\n",
       " 'permission': 4056,\n",
       " 'earnest': 16698,\n",
       " 'intersected': 9486,\n",
       " 'ether': 20191,\n",
       " 'foundered': 14967,\n",
       " 'mcdade': 19054,\n",
       " 'calculated': 2675,\n",
       " '892': 4919,\n",
       " 'minnesota': 4035,\n",
       " 'investigations': 6276,\n",
       " 'febaruary': 30316,\n",
       " 'architectural': 11080,\n",
       " 'hawley': 6100,\n",
       " 'stocks': 410,\n",
       " 'elders': 3480,\n",
       " 'cheetham': 11918,\n",
       " 'synfuels': 14262,\n",
       " 'gilliam': 27007,\n",
       " 'humidity': 10806,\n",
       " 'nhy': 14278,\n",
       " \"exporters'\": 7986,\n",
       " \"platinum's\": 21334,\n",
       " 'apartment': 11247,\n",
       " 'platforms': 3461,\n",
       " \"callebaut's\": 14880,\n",
       " 'montoya': 20235,\n",
       " \"doumeng's\": 28821,\n",
       " 'deliberate': 15239,\n",
       " 'drops': 7726,\n",
       " 'sul': 7669,\n",
       " 'hodel': 4687,\n",
       " 'matsunaga': 11025,\n",
       " 'curtis': 16728,\n",
       " 'fbt': 25536,\n",
       " 'entail': 12514,\n",
       " 'rigged': 23741,\n",
       " 'job': 2614,\n",
       " 'simmons': 7558,\n",
       " 'elgin': 13726,\n",
       " 'fervor': 26425,\n",
       " '431': 6197,\n",
       " 'sfp': 22447,\n",
       " 'vogel': 16615,\n",
       " 'incentives': 2244,\n",
       " 'phil': 8825,\n",
       " 'council': 587,\n",
       " 'crops': 1094,\n",
       " 'oilmeal': 16794,\n",
       " 'atp': 17313,\n",
       " 'indo': 16809,\n",
       " 'naif': 22545,\n",
       " 'vongarlem': 17741,\n",
       " 'ale': 28031,\n",
       " 'surprisingly': 12596,\n",
       " '707': 7134,\n",
       " 'nuggets': 21558,\n",
       " 'specialising': 12601,\n",
       " 'dainik': 29273,\n",
       " 'poultry': 2530,\n",
       " 'nwp': 22527,\n",
       " 'ernest': 9848,\n",
       " 'modality': 22136,\n",
       " 'salon': 26438,\n",
       " '82': 993,\n",
       " 'definite': 7027,\n",
       " 'gisela': 14356,\n",
       " 'styx': 14377,\n",
       " 'rank': 3770,\n",
       " 'intw': 21121,\n",
       " 'paringa': 21025,\n",
       " '34p': 14260,\n",
       " 'unsettle': 26959,\n",
       " 'xianglin': 25648,\n",
       " 'reaffirmation': 9628,\n",
       " 'persons': 7492,\n",
       " 'squeeze': 8252,\n",
       " 'escalated': 14322,\n",
       " 'underwriter': 6510,\n",
       " 'dilution': 7539,\n",
       " 'abdulla': 14806,\n",
       " \"officer's\": 29535,\n",
       " 'predicting': 8282,\n",
       " 'czechoslovakia': 8777,\n",
       " 'bread': 5658,\n",
       " 'lives': 8867,\n",
       " 'rios': 11525,\n",
       " 'investigate': 6104,\n",
       " 'contractors': 5745,\n",
       " 'morenci': 29137,\n",
       " '0667': 29069,\n",
       " 'explorer': 19014,\n",
       " '344': 4573,\n",
       " '141': 2177,\n",
       " 'implications': 3639,\n",
       " 'improved': 1004,\n",
       " 'kuroda': 7566,\n",
       " 'andersson': 29405,\n",
       " 'scientist': 30691,\n",
       " 'vik': 10779,\n",
       " 'irs': 7598,\n",
       " 'disincentive': 9869,\n",
       " 'nebr': 23410,\n",
       " \"sony's\": 21353,\n",
       " '698': 9901,\n",
       " 'responds': 23115,\n",
       " \"ntpc's\": 27465,\n",
       " '957': 5415,\n",
       " 'clever': 17355,\n",
       " 'grassgreen': 27878,\n",
       " 'cyclical': 9708,\n",
       " '7665': 23258,\n",
       " 'wednesbury': 27479,\n",
       " '7730': 17473,\n",
       " 'gesellschaft': 25078,\n",
       " 'apples': 11089,\n",
       " 'prepare': 4599,\n",
       " 'findley': 23605,\n",
       " '95p': 15194,\n",
       " 'soothe': 23325,\n",
       " 'pages': 19708,\n",
       " 'analyzing': 10844,\n",
       " 'acknowledgement': 19873,\n",
       " 'witte': 7506,\n",
       " 'paues': 13184,\n",
       " 'flexi': 9984,\n",
       " \"mlx's\": 26665,\n",
       " 'bahia': 6680,\n",
       " 'suppositions': 20828,\n",
       " 'which': 47,\n",
       " 'unreasonable': 8192,\n",
       " 'mlnm': 20115,\n",
       " 'question': 1199,\n",
       " 'ghanaian': 9403,\n",
       " 'usdc': 28088,\n",
       " 'create': 1908,\n",
       " 'hidden': 14438,\n",
       " 'certifying': 28670,\n",
       " '5195': 27513,\n",
       " '8498': 29694,\n",
       " \"li's\": 30041,\n",
       " 'alitalia': 11947,\n",
       " 'hung': 10709,\n",
       " 'bltfx': 26055,\n",
       " 'thompson': 11015,\n",
       " 'centralia': 23805,\n",
       " 'atom': 28840,\n",
       " 'soros': 6026,\n",
       " 'dental': 7685,\n",
       " 'preview': 27737,\n",
       " 'beome': 23117,\n",
       " 'devices': 6551,\n",
       " 'pollinates': 27145,\n",
       " \"ldp's\": 19390,\n",
       " \"associated's\": 16394,\n",
       " 'indebted': 6461,\n",
       " 'norf': 20974,\n",
       " \"tobacco's\": 18001,\n",
       " 'fathers': 14132,\n",
       " 'dyson': 16667,\n",
       " 'ffcop': 20318,\n",
       " 'adherence': 6299,\n",
       " 'wires': 20921,\n",
       " 'page': 7813,\n",
       " \"searle's\": 11059,\n",
       " 'ichan': 20588,\n",
       " 'grazing': 12944,\n",
       " 'bronstein': 17977,\n",
       " 'evolution': 14742,\n",
       " 'oponents': 24992,\n",
       " 'aspects': 4358,\n",
       " 'persists': 12666,\n",
       " 'favored': 6201,\n",
       " 'cloudy': 22694,\n",
       " 'ordered': 2694,\n",
       " 'fending': 30078,\n",
       " 'reinsured': 27856,\n",
       " 'katsuhiko': 24980,\n",
       " 'heart': 3762,\n",
       " 'repeat': 8378,\n",
       " 'leader': 1877,\n",
       " 'carry': 1953,\n",
       " 'confectionery': 8718,\n",
       " 'surpassed': 8392,\n",
       " 'length': 6185,\n",
       " 'unwound': 30147,\n",
       " \"they've\": 4797,\n",
       " 'fiberglass': 24331,\n",
       " 'erekson': 23359,\n",
       " 'gloeilampenfabriek': 28524,\n",
       " \"ncb's\": 28898,\n",
       " 'preventive': 22573,\n",
       " '978825': 26938,\n",
       " 'electorate': 18290,\n",
       " 'eaux': 15172,\n",
       " '222': 2931,\n",
       " 'margarine': 9691,\n",
       " 'followthrough': 28504,\n",
       " 'rjr': 6720,\n",
       " 'conroe': 30757,\n",
       " 'valencia': 12655,\n",
       " 'watanabe': 3783,\n",
       " 'gibbs': 11061,\n",
       " \"ross's\": 21585,\n",
       " 'mortner': 14478,\n",
       " 'bolivia': 4804,\n",
       " 'ni': 25241,\n",
       " 'coinciding': 11827,\n",
       " 'robertson': 11123,\n",
       " 'costlier': 29463,\n",
       " 'republic': 2604,\n",
       " 'tv': 4755,\n",
       " 'crit': 27264,\n",
       " 'reinstating': 24823,\n",
       " '019': 6354,\n",
       " 'estimation': 16825,\n",
       " \"owen's\": 20947,\n",
       " 'signs': 1605,\n",
       " 'templeton': 19794,\n",
       " 'convern': 21751,\n",
       " 'ranasinghe': 22185,\n",
       " 'mohawk': 26005,\n",
       " 'kfhg': 27261,\n",
       " 'enter': 2153,\n",
       " 'oxy': 8505,\n",
       " 'dimensions': 13214,\n",
       " 'electricity': 4232,\n",
       " 'terribly': 13522,\n",
       " 'dn61': 28410,\n",
       " 'issuer': 9063,\n",
       " 'frank': 4472,\n",
       " '795': 8361,\n",
       " 'and': 6,\n",
       " '489': 4217,\n",
       " 'culminates': 21365,\n",
       " 'downpours': 22779,\n",
       " 'bargain': 7093,\n",
       " 'rebuilding': 11889,\n",
       " '824': 5573,\n",
       " 'ladenburg': 24993,\n",
       " 'journal': 5050,\n",
       " 'novak': 29413,\n",
       " 'apolonio': 23884,\n",
       " 'redice': 26652,\n",
       " 'sobeys': 18729,\n",
       " '783': 7141,\n",
       " 'gooseberry': 23841,\n",
       " 'en': 11370,\n",
       " 'electrostatic': 21380,\n",
       " 'impunity': 29886,\n",
       " \"30's\": 24219,\n",
       " 'signficant': 15977,\n",
       " 'bno': 18557,\n",
       " 'fernando': 6990,\n",
       " '420': 2709,\n",
       " 'locomotives': 22474,\n",
       " 'bamberger': 13219,\n",
       " 'highs': 4218,\n",
       " 'plowing': 21027,\n",
       " 'phlx': 8317,\n",
       " 'harleysville': 17120,\n",
       " 'corridor': 17040,\n",
       " 'anel': 25346,\n",
       " 'debs': 25855,\n",
       " 'certificated': 20026,\n",
       " 'q': 15985,\n",
       " 'ionterest': 25019,\n",
       " 'convicted': 28806,\n",
       " 'nisker': 12499,\n",
       " 'spectradyne': 21201,\n",
       " 'vermilion': 22064,\n",
       " '870p': 22018,\n",
       " 'bartlett': 15620,\n",
       " 'belive': 26564,\n",
       " 'executived': 28982,\n",
       " '555': 3760,\n",
       " 'generations': 30916,\n",
       " 'agreemats': 28601,\n",
       " '537p': 26847,\n",
       " \"iran's\": 2846,\n",
       " \"bojangles'\": 10347,\n",
       " 'gearing': 9769,\n",
       " 'determines': 11325,\n",
       " 'explainable': 21100,\n",
       " 'inflows': 4140,\n",
       " 'uddevalla': 22365,\n",
       " 'mistake': 8329,\n",
       " 'convincing': 8721,\n",
       " 'programming': 15723,\n",
       " 'distant': 12265,\n",
       " 'currrent': 23529,\n",
       " 'stanlow': 22516,\n",
       " 'elkhorn': 16355,\n",
       " '02': 1474,\n",
       " 'wifo': 8918,\n",
       " 'shocks': 20687,\n",
       " 'cerier': 27485,\n",
       " 'derivatives': 6058,\n",
       " 'stamped': 28245,\n",
       " 'improbable': 18150,\n",
       " 'adult': 13144,\n",
       " \"'ali\": 21900,\n",
       " 'lks': 27408,\n",
       " 'bartering': 19498,\n",
       " 'tracked': 17999,\n",
       " 'arabica': 9449,\n",
       " 'erasable': 14512,\n",
       " 'lakeland': 28354,\n",
       " 'flooded': 7609,\n",
       " 'satisfacotry': 16371,\n",
       " 'reintensification': 26423,\n",
       " 'pivot': 12131,\n",
       " 'forceful': 14937,\n",
       " 'saks': 22082,\n",
       " 'switzerland': 2606,\n",
       " 'charmglow': 16598,\n",
       " 'childhood': 19436,\n",
       " 'staging': 13500,\n",
       " 'zimmer': 15201,\n",
       " 'petroleum': 424,\n",
       " 'yoot': 13779,\n",
       " 'fashions': 13420,\n",
       " '956': 6852,\n",
       " 'converting': 10859,\n",
       " 'soyproduct': 14634,\n",
       " 'libres': 29114,\n",
       " 'recommendations': 3006,\n",
       " 'tully': 16799,\n",
       " 'representations': 14304,\n",
       " 'gm': 3049,\n",
       " 'foiled': 28605,\n",
       " 'jerusalem': 22206,\n",
       " 'munitions': 23654,\n",
       " 'years': 203,\n",
       " 'ciego': 24723,\n",
       " 'overhang': 9191,\n",
       " 'offenders': 19734,\n",
       " 'message': 4265,\n",
       " 'nikonov': 19979,\n",
       " 'gottesman': 19634,\n",
       " 'teapa': 20986,\n",
       " 'eight': 350,\n",
       " 'resolving': 6986,\n",
       " 'earnings': 183,\n",
       " 'larry': 7132,\n",
       " 'domestic': 257,\n",
       " 'premises': 17380,\n",
       " 'hermes': 24825,\n",
       " 'cranes': 8199,\n",
       " 'matsuda': 26092,\n",
       " 'iringa': 19900,\n",
       " 'occured': 14403,\n",
       " 'absorbed': 8682,\n",
       " 'opposite': 13862,\n",
       " 'follows': 1237,\n",
       " 'cheng': 8750,\n",
       " 'contradicted': 14341,\n",
       " 'contravening': 24952,\n",
       " 'metal': 1230,\n",
       " 'liban': 21221,\n",
       " 'archer': 7061,\n",
       " 'primary': 1131,\n",
       " 'babcock': 10118,\n",
       " 'boardwalk': 25735,\n",
       " 'aquiring': 28984,\n",
       " 'agree': 1538,\n",
       " 'bulk': 1771,\n",
       " 'molybdenum': 22735,\n",
       " 'aspirant': 24803,\n",
       " \"twa's\": 5892,\n",
       " 'excluding': 1691,\n",
       " 'quantech': 15196,\n",
       " 'landscape': 29666,\n",
       " 'ripe': 14583,\n",
       " 'renegotiate': 13098,\n",
       " \"thursday's\": 9338,\n",
       " 'petcord': 21349,\n",
       " 'slc': 20196,\n",
       " 'papandreou': 6543,\n",
       " 'closed': 810,\n",
       " 'satisfactorily': 10711,\n",
       " 'piloted': 29833,\n",
       " 'asca': 26799,\n",
       " 'transshipment': 20001,\n",
       " 'level': 317,\n",
       " 'macroeconomic': 9260,\n",
       " 'pit': 8327,\n",
       " 'forestalled': 30812,\n",
       " '24': 293,\n",
       " 'retrieving': 23619,\n",
       " 'britain': 799,\n",
       " 'revalued': 7607,\n",
       " 'unpredictable': 18455,\n",
       " 'impoundments': 21116,\n",
       " 'encouragement': 19598,\n",
       " 'pessimists': 24596,\n",
       " \"nigeria's\": 6448,\n",
       " 'pyongyang': 11424,\n",
       " 'adedeji': 14380,\n",
       " 'unneccessarily': 21514,\n",
       " 'demonstrations': 18163,\n",
       " 'greyhound': 5821,\n",
       " 'hubei': 20211,\n",
       " 'rgc': 9490,\n",
       " 'videodiscs': 24266,\n",
       " 'tito': 24225,\n",
       " 'crowded': 9822,\n",
       " 'stockbuilding': 10080,\n",
       " 'profitability': 2139,\n",
       " 'forests': 10805,\n",
       " 'denwa': 6800,\n",
       " 'adulterated': 24862,\n",
       " 'padding': 12535,\n",
       " 'freed': 11244,\n",
       " 'palates': 26269,\n",
       " 'brooms': 24931,\n",
       " 'chrome': 23285,\n",
       " 'flexibility': 2977,\n",
       " 'flotilla': 15385,\n",
       " 'karin': 25763,\n",
       " 'irvin': 26426,\n",
       " \"ibp's\": 14623,\n",
       " 'rightholders': 17587,\n",
       " 'final': 581,\n",
       " '395': 6423,\n",
       " 'doubted': 5307,\n",
       " 'studying': 2860,\n",
       " 'output': 308,\n",
       " '130p': 15162,\n",
       " 'shaky': 15432,\n",
       " 'tenant': 17369,\n",
       " 'policing': 18949,\n",
       " 'paul': 1534,\n",
       " 'dbt': 23956,\n",
       " 'natl': 12706,\n",
       " 'suffield': 10427,\n",
       " 'reminded': 12043,\n",
       " 'propsective': 30055,\n",
       " 'lyons': 4660,\n",
       " 'perelman': 6684,\n",
       " 'rigorous': 19895,\n",
       " 'financier': 7888,\n",
       " 'purhcase': 29318,\n",
       " \"crd's\": 26657,\n",
       " 'electron': 12907,\n",
       " 'evidently': 28557,\n",
       " 'injection': 4563,\n",
       " 'smuggled': 11226,\n",
       " 'advertising': 4839,\n",
       " 'phaseout': 24067,\n",
       " 'alleviate': 7284,\n",
       " '971': 5757,\n",
       " 'blockade': 15155,\n",
       " 'snow': 3694,\n",
       " 'disrupting': 18493,\n",
       " \"grenfell's\": 22156,\n",
       " 'classifications': 12784,\n",
       " 'welfare': 11582,\n",
       " 'hendrik': 18929,\n",
       " 'amp': 7627,\n",
       " 'uproar': 12746,\n",
       " 'parsley': 21998,\n",
       " \"strategic's\": 22909,\n",
       " 'enetrprises': 29955,\n",
       " 'kaske': 9579,\n",
       " 'statement': 238,\n",
       " 'activating': 20711,\n",
       " 'suggesting': 5358,\n",
       " 'respect': 2828,\n",
       " 'escape': 11520,\n",
       " 'pendleton': 20591,\n",
       " '600p': 25680,\n",
       " \"columbia's\": 15097,\n",
       " 'hui': 14964,\n",
       " 'groupement': 28032,\n",
       " 'density': 10167,\n",
       " 'supplied': 3651,\n",
       " '490': 3777,\n",
       " 'sunrise': 20492,\n",
       " 'cyrus': 16903,\n",
       " 'quel': 11105,\n",
       " 'qsv': 24721,\n",
       " 'wiggins': 10685,\n",
       " 'dyke': 11287,\n",
       " 'reassemble': 18303,\n",
       " \"'teishoku'\": 30605,\n",
       " \"december's\": 4702,\n",
       " 'surplus': 285,\n",
       " 'surmount': 27225,\n",
       " 'youghal': 9488,\n",
       " 'standardised': 28600,\n",
       " 'universal': 4106,\n",
       " 'lies': 6928,\n",
       " 'shops': 8784,\n",
       " 'adopts': 17638,\n",
       " 'removal': 5031,\n",
       " '777': 5231,\n",
       " 'whitehead': 13335,\n",
       " 'alternates': 24742,\n",
       " 'changes': 753,\n",
       " 'unconditional': 9049,\n",
       " 'hecht': 24728,\n",
       " 'sabotage': 19030,\n",
       " 'increas': 26607,\n",
       " 'grown': 3017,\n",
       " '743': 6200,\n",
       " 'assaults': 12556,\n",
       " 'baer': 15425,\n",
       " 'wielding': 23427,\n",
       " 'seasons': 6154,\n",
       " 'cracker': 16112,\n",
       " 'cents': 511,\n",
       " 'baht': 3988,\n",
       " 'receiveable': 25450,\n",
       " 'bfg': 8562,\n",
       " 'qtelc': 17382,\n",
       " 'enrique': 28232,\n",
       " 'eiaj': 19700,\n",
       " 'trades': 3235,\n",
       " 'intervened': 1733,\n",
       " 'copperworkers': 30919,\n",
       " 'griffin': 29340,\n",
       " 'slackened': 22080,\n",
       " 'send': 3258,\n",
       " 'landeszentralbanken': 30262,\n",
       " '5870': 17983,\n",
       " 'selectcare': 16823,\n",
       " 'petro': 3412,\n",
       " 'bevis': 16691,\n",
       " 'roll': 4419,\n",
       " 'ta': 10543,\n",
       " \"dna's\": 24974,\n",
       " 'harima': 16310,\n",
       " 'resutls': 27325,\n",
       " 'opffer': 30235,\n",
       " 'sentry': 25440,\n",
       " 'stearine': 28216,\n",
       " 'chris': 7832,\n",
       " 'comparison': 3393,\n",
       " 'sporadic': 13701,\n",
       " 'selm': 21039,\n",
       " 'forecasters': 15418,\n",
       " '007025': 29893,\n",
       " 'destroying': 9272,\n",
       " 'tomography': 14715,\n",
       " \"microtel's\": 29550,\n",
       " 'electrinical': 21500,\n",
       " 'multiuser': 30794,\n",
       " 'labels': 8226,\n",
       " 'daniel': 3351,\n",
       " 'profits': 451,\n",
       " 'spence': 24413,\n",
       " 'network': 1669,\n",
       " 'flink': 30940,\n",
       " '10th': 12349,\n",
       " 'influx': 9396,\n",
       " 'underlying': 1663,\n",
       " 'lip': 20051,\n",
       " 'evanston': 17710,\n",
       " 'chaper': 29014,\n",
       " 'suprised': 29182,\n",
       " 'unclear': 6051,\n",
       " 'pockets': 14176,\n",
       " '080': 7307,\n",
       " 'endevco': 20179,\n",
       " 'mineral': 2458,\n",
       " 'inference': 30786,\n",
       " 'cheques': 16208,\n",
       " 'outlooks': 23646,\n",
       " 'matters': 3270,\n",
       " 'ofconcern': 29188,\n",
       " '814': 7882,\n",
       " 'consignment': 16150,\n",
       " 'koppabergs': 21619,\n",
       " 'compass': 13906,\n",
       " 'colorants': 20928,\n",
       " 'cyclop': 24529,\n",
       " 'given': 627,\n",
       " 'restraint': 3424,\n",
       " 'transfered': 19224,\n",
       " \"boards'\": 27616,\n",
       " 'applicant': 23375,\n",
       " 'establishment': 6118,\n",
       " 'treadmills': 14756,\n",
       " 'pentron': 22286,\n",
       " 'handelsblad': 17357,\n",
       " 'apt': 8924,\n",
       " 'uninfected': 24238,\n",
       " 'cleaned': 16198,\n",
       " \"industries'\": 20515,\n",
       " 'midlevel': 23695,\n",
       " 'cosby': 24997,\n",
       " 'resells': 23715,\n",
       " 'spacelab': 20944,\n",
       " 'thermoset': 23722,\n",
       " \"standex's\": 28564,\n",
       " '7619': 29424,\n",
       " 'cfas': 16604,\n",
       " 'samsung': 12279,\n",
       " 'oceans': 25746,\n",
       " 'counterbore': 21132,\n",
       " 'wheats': 12085,\n",
       " 'eddie': 2819,\n",
       " \"msu's\": 30420,\n",
       " 'homebuilder': 26998,\n",
       " 'woods': 8803,\n",
       " 'officer': 1077,\n",
       " 'febraury': 20289,\n",
       " 'ahc': 14305,\n",
       " 'sbp': 27444,\n",
       " 'profitable': 1784,\n",
       " 'fixed': 800,\n",
       " 'polyphenyl': 12605,\n",
       " \"ireland's\": 17803,\n",
       " 'decline': 430,\n",
       " 'flonetics': 27438,\n",
       " 'critiera': 23955,\n",
       " 'response': 804,\n",
       " 'kassenobligation': 29026,\n",
       " 'terra': 5823,\n",
       " 'trusting': 26299,\n",
       " 'winston': 11691,\n",
       " 'siess': 20549,\n",
       " 'toal': 19407,\n",
       " 'shadowed': 25200,\n",
       " \"foods'\": 18708,\n",
       " 'canefields': 28000,\n",
       " 'khashoggi': 16454,\n",
       " 'vancouver': 4629,\n",
       " 'capozza': 23863,\n",
       " 'pca': 14192,\n",
       " 'deploring': 15467,\n",
       " 'stimulating': 5475,\n",
       " \"philippine's\": 28021,\n",
       " 'iris': 18757,\n",
       " 'backlog': 2254,\n",
       " 'chronic': 12009,\n",
       " 'slice': 20536,\n",
       " 'judge': 4509,\n",
       " 'delimiting': 21947,\n",
       " 'michelin': 22450,\n",
       " 'shed': 8298,\n",
       " 'leao': 13782,\n",
       " 'yom': 26448,\n",
       " 'strained': 6919,\n",
       " 'nicolls': 20517,\n",
       " 'napa': 13529,\n",
       " 'constitucion': 21278,\n",
       " 'unanswered': 17411,\n",
       " 'obvious': 5442,\n",
       " 'imc': 17155,\n",
       " 'succeeds': 8514,\n",
       " 'faq': 20550,\n",
       " 'suspiciously': 23137,\n",
       " '091': 6366,\n",
       " 'automakers': 14155,\n",
       " 'totaling': 4746,\n",
       " 'postal': 9400,\n",
       " 'weekday': 27567,\n",
       " \"france's\": 3232,\n",
       " 'deterioration': 3448,\n",
       " 'clarifies': 11007,\n",
       " 'structuring': 20850,\n",
       " 'hartford': 9179,\n",
       " 'residues': 15278,\n",
       " 'revision': 2663,\n",
       " '625': 3118,\n",
       " 'imprecise': 21735,\n",
       " 'shelled': 11524,\n",
       " 'chips': 2031,\n",
       " 'tariffs': 722,\n",
       " 'merry': 21648,\n",
       " 'orange': 2380,\n",
       " 'circulate': 14732,\n",
       " 'fintech': 11804,\n",
       " 'telephonques': 22929,\n",
       " 'gove': 23895,\n",
       " 'sr': 18592,\n",
       " 'wyttenbach': 6845,\n",
       " 'rawl': 8862,\n",
       " 'spindle': 25669,\n",
       " 'barcoo': 19506,\n",
       " 'bars': 8340,\n",
       " 'commerical': 7728,\n",
       " 'interemdiate': 29168,\n",
       " 'becker': 22861,\n",
       " 'susumu': 13648,\n",
       " 'seram': 23791,\n",
       " 'flourish': 30890,\n",
       " 'inundated': 24641,\n",
       " 'cancelling': 12895,\n",
       " 'brent': 4776,\n",
       " 'jorge': 6081,\n",
       " 'csbc': 11111,\n",
       " 'sustained': 3289,\n",
       " 'akiman': 14297,\n",
       " 'restaurants': 3489,\n",
       " 'southern': 1030,\n",
       " 'proportion': 3312,\n",
       " 'overpayments': 20362,\n",
       " 'southampton': 30045,\n",
       " 'below': 417,\n",
       " 'agressively': 26646,\n",
       " 'misunderstandings': 12339,\n",
       " 'picul': 20816,\n",
       " 'flag': 2812,\n",
       " 'keeps': 9547,\n",
       " 'inspire': 18263,\n",
       " 'negotiate': 1766,\n",
       " '1949': 30472,\n",
       " 'demands': 2006,\n",
       " \"botswana's\": 28776,\n",
       " 'plantings': 2498,\n",
       " 'acceptances': 3045,\n",
       " 'wichita': 10963,\n",
       " 'researcher': 22589,\n",
       " 'dura': 20397,\n",
       " 'transfers': 4738,\n",
       " 'isle': 10038,\n",
       " 'coercive': 15797,\n",
       " 'orchid': 29139,\n",
       " 'bcom': 9645,\n",
       " 'counts': 12720,\n",
       " 'takever': 28519,\n",
       " 'hoppers': 26111,\n",
       " 'fsis': 12193,\n",
       " 'formation': 3547,\n",
       " 'february': 101,\n",
       " 'submits': 25073,\n",
       " 'floatation': 28921,\n",
       " 'notimex': 28626,\n",
       " 'forsakrings': 19677,\n",
       " 'dinner': 8180,\n",
       " 'kalo': 26520,\n",
       " 'noteworthy': 26964,\n",
       " 'strevig': 24520,\n",
       " 'myra': 13313,\n",
       " 'sur': 14350,\n",
       " 'danish': 5155,\n",
       " 'scoffed': 18558,\n",
       " 'prudency': 24239,\n",
       " 'drunkenness': 25495,\n",
       " 'ragsdale': 24580,\n",
       " 'clampdown': 30970,\n",
       " 'vetoing': 17325,\n",
       " 'edd': 28198,\n",
       " 'expeditiously': 13630,\n",
       " 'resales': 19267,\n",
       " \"bowater's\": 29227,\n",
       " 'cooperative': 2744,\n",
       " 'africa': 1197,\n",
       " 'declaratory': 12877,\n",
       " 'disturbed': 8810,\n",
       " 'montary': 20476,\n",
       " 'westpac': 4744,\n",
       " 'meru': 25411,\n",
       " 'pigs': 6373,\n",
       " 'dollars': 912,\n",
       " 'potencial': 26287,\n",
       " 'rang': 27775,\n",
       " 'destrehan': 10322,\n",
       " 'cashflow': 18773,\n",
       " 'citation': 27361,\n",
       " 'osprey': 14143,\n",
       " 'amendend': 23278,\n",
       " 'earners': 8663,\n",
       " 'morales': 17785,\n",
       " 'golden': 4007,\n",
       " '3185': 22483,\n",
       " 'stockbroker': 9906,\n",
       " 'aa': 3475,\n",
       " 'kerosene': 8047,\n",
       " \"leiner's\": 22574,\n",
       " \"pool's\": 22121,\n",
       " 'newark': 20332,\n",
       " 'inherited': 16816,\n",
       " 'urged': 1579,\n",
       " 'liable': 8949,\n",
       " \"transamerican's\": 8897,\n",
       " 'depreciated': 11205,\n",
       " 'marks': 347,\n",
       " 'shelard': 12862,\n",
       " 'infringe': 17703,\n",
       " 'matr': 15506,\n",
       " '464p': 30948,\n",
       " 'batteries': 9889,\n",
       " 'instructing': 16753,\n",
       " 'califoirnia': 20883,\n",
       " 'suharto': 5812,\n",
       " 'lopez': 14815,\n",
       " ...}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('x_train_reu', x_train_reu.shape)\n",
    "print('y_train_reu', y_train_reu.shape)\n",
    "print('x_test_reu ', x_test_reu.shape)\n",
    "print('y_test_reu ', y_test_reu.shape)\n",
    "\n",
    "print('\\nlen(x_train_reu[0])', len(x_train_reu[0]))\n",
    "print('len(x_train_reu[1])', len(x_train_reu[1]))\n",
    "# print('x_train_reu[0]', x_train_reu[0])\n",
    "# print('x_train_reu[1]', x_train_reu[1])\n",
    "print('y_train_reu[0]', y_train_reu[0])\n",
    "print('y_train_reu[1]', y_train_reu[1])\n",
    "\n",
    "print('\\nlen(x_test_reu[0])', len(x_test_reu[0]))\n",
    "print('len(x_test_reu[1])', len(x_test_reu[1]))\n",
    "# print('x_test_reu[0]', x_test_reu[0])\n",
    "# print('x_test_reu[1]', x_test_reu[1])\n",
    "print('y_test_reu[0]', y_test_reu[0])\n",
    "print('y_test_reu[1]', y_test_reu[1])\n",
    "\n",
    "## Dictionary\n",
    "word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 out of 30979\n",
      "5000 out of 30979\n",
      "10000 out of 30979\n",
      "15000 out of 30979\n",
      "20000 out of 30979\n",
      "25000 out of 30979\n",
      "30000 out of 30979\n"
     ]
    }
   ],
   "source": [
    "num2word = ['']\n",
    "for idx in range(len(word_index)):\n",
    "    if idx%5000==0: print(idx,'out of',len(word_index))\n",
    "    for x in word_index:\n",
    "        if not word_index[x]==(idx+1): \n",
    "            continue\n",
    "        else:\n",
    "            num2word.append(x)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_index['dlrs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " the wattie nondiscriminatory mln loss for plc said at only ended said commonwealth could 1 traders now april 0 a after said from 1985 and from foreign 000 april 0 prices its account year a but in this mln home an states earlier and rise and revs vs 000 its 16 vs 000 a but 3 psbr oils several and shareholders and dividend vs 000 its all 4 vs 000 1 mln agreed largely april 0 are 2 states will billion total and against 000 pct dlrs\n",
      "[1, 27595, 28842, 8, 43, 10, 447, 5, 25, 207, 270, 5, 3095, 111, 16, 369, 186, 90, 67, 7, 89, 5, 19, 102, 6, 19, 124, 15, 90, 67, 84, 22, 482, 26, 7, 48, 4, 49, 8, 864, 39, 209, 154, 6, 151, 6, 83, 11, 15, 22, 155, 11, 15, 7, 48, 9, 4579, 1005, 504, 6, 258, 6, 272, 11, 15, 22, 134, 44, 11, 15, 16, 8, 197, 1245, 90, 67, 52, 29, 209, 30, 32, 132, 6, 109, 15, 17, 12]\n"
     ]
    }
   ],
   "source": [
    "sent = ''\n",
    "for x_i in x_train_reu[0]:\n",
    "    sent+=' '+num2word[x_i]\n",
    "print( sent )\n",
    "print( x_train_reu[0] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30979"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pad sequences: (samples x time)\n",
      "X_train_reu shape: (8982, 50)\n",
      "X_test_reu shape:  (2246, 50)\n"
     ]
    }
   ],
   "source": [
    "print('Pad sequences: (samples x time)')\n",
    "maxlen = 50\n",
    "X_train_reu = pad_sequences(x_train_reu, maxlen=maxlen)\n",
    "X_test_reu  = pad_sequences(x_test_reu, maxlen=maxlen)\n",
    "print('X_train_reu shape:', X_train_reu.shape)\n",
    "print('X_test_reu shape: ', X_test_reu.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pad sequences: (samples x time)\n",
      "Y_train_reu shape: (8982, 46)\n",
      "Y_test_reu shape:  (2246, 46)\n"
     ]
    }
   ],
   "source": [
    "print('Pad sequences: (samples x time)')\n",
    "Y_train_reu = to_categorical(y_train_reu, 46)\n",
    "Y_test_reu  = to_categorical(y_test_reu, 46)\n",
    "print('Y_train_reu shape:', Y_train_reu.shape)\n",
    "print('Y_test_reu shape: ', Y_test_reu.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_reu (8982, 50)\n",
      "Y_train_reu (8982, 46)\n",
      "X_test_reu  (2246, 50)\n",
      "Y_test_reu  (2246, 46)\n"
     ]
    }
   ],
   "source": [
    "print('X_train_reu', X_train_reu.shape)\n",
    "print('Y_train_reu', Y_train_reu.shape)\n",
    "print('X_test_reu ', X_test_reu.shape)\n",
    "print('Y_test_reu ', Y_test_reu.shape)\n",
    "data_mode = 'reuters'\n",
    "Data_dir = 'Data_'+data_mode+'/'\n",
    "np.save(Data_dir+'X_train_reu',X_train_reu)\n",
    "np.save(Data_dir+'Y_train_reu',Y_train_reu)\n",
    "np.save(Data_dir+'X_test_reu',X_test_reu)\n",
    "np.save(Data_dir+'Y_test_reu',Y_test_reu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_mode = 'reuters'\n",
    "if \n",
    "    X_train_reu = pad_sequences(x_train_reu, maxlen=maxlen)\n",
    "    X_test_reu  = pad_sequences(x_test_reu, maxlen=maxlen)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}