Encoder_Decoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INPUT\n",
    "\n",
    "#shape of input vector (20, 100)\n",
    "#each word is represented as a vector of 100 features\n",
    "#sentences with more than 20 words are clipped\n",
    "#and one with less than 20 words are padded with zero vectors\n",
    "#target -> onehot vectors of dimension = length of vocabulary\n",
    "\n",
    "#OUTPUT\n",
    "#softmax output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import gensim\n",
    "import json\n",
    "import numpy as np\n",
    "import csv\n",
    "import nltk\n",
    "print('success')\n",
    "#hello"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success\n"
     ]
    }
   ],
   "source": [
    "#loading data\n",
    "f = open(\"consolidated.csv\", \"r\")\n",
    "data = csv.reader(f, delimiter = ',')\n",
    "next(data)\n",
    "corrected = []\n",
    "original = []\n",
    "\n",
    "#tokenizing\n",
    "for row in data :\n",
    "    if (row[-1] == \"1\") :\n",
    "        original.append(nltk.word_tokenize(row[1]))\n",
    "        corrected.append(nltk.word_tokenize(row[2]))\n",
    "        \n",
    "print('success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9340\n",
      "success\n"
     ]
    }
   ],
   "source": [
    "#preprocessing original text\n",
    "d = {}\n",
    "for i in original :\n",
    "    for j in i :\n",
    "        d[j] = d.get(j, 0) + 1\n",
    "for i in range(len(original)) :\n",
    "    for j in range(len(original[i])) :\n",
    "        original[i][j] = original[i][j].lower()\n",
    "print('success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of corrected vocabulary =  2209\n",
      "success\n"
     ]
    }
   ],
   "source": [
    "#making onehot vectors and index to word mappings\n",
    "from collections import Counter, defaultdict\n",
    "flat = [\" \".join(x) for x in corrected]\n",
    "new_list = []\n",
    "for item in flat :\n",
    "    new_list += item.split(' ')\n",
    "for i in range(len(new_list)) :\n",
    "    new_list[i] = new_list[i].lower()\n",
    "c = Counter(new_list)\n",
    "del new_list\n",
    "new_list = []\n",
    "for i, j in c.items() :\n",
    "    if j > 2 :\n",
    "        new_list.append(i)\n",
    "flat = list(set(new_list))\n",
    "print(\"Length of corrected vocabulary = \", len(flat))\n",
    "\n",
    "out_feature = len(flat)\n",
    "\n",
    "one_hot_corrected = {}\n",
    "index_to_word = {}\n",
    "k = [0 for x in range(len(flat)+1)]\n",
    "for i in range(len(flat)) :\n",
    "    temp = k[:]\n",
    "    temp[i]= 1\n",
    "    one_hot_corrected[flat[i]] = temp[:]\n",
    "    index_to_word[i] = flat[i]\n",
    "del flat\n",
    "k[-1] = 1\n",
    "one_hot_corrected = defaultdict(lambda : k[:], one_hot_corrected) #take __unk__\n",
    "index_to_word = defaultdict(lambda : \"__unk__\", index_to_word)\n",
    "\n",
    "print('success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success\n"
     ]
    }
   ],
   "source": [
    "#word2vec\n",
    "IN_FEATURES = 100\n",
    "word2vec_original = gensim.models.Word2Vec(original + [[\"__unk__\"]], min_count=3, size = IN_FEATURES)\n",
    "print('success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success\n"
     ]
    }
   ],
   "source": [
    "#creating x and y data\n",
    "NB_WORDS = 20\n",
    "master_x = []\n",
    "fifty_zeroes = IN_FEATURES * [0]\n",
    "for i in original :\n",
    "    tmp = []\n",
    "    for j in i :\n",
    "        if (j in word2vec_original and len(tmp) < NB_WORDS) :\n",
    "            tmp.append(word2vec_original[j])\n",
    "        elif len(tmp) < NB_WORDS :\n",
    "            tmp.append(100 * [0])\n",
    "    while (len(tmp) < NB_WORDS) :\n",
    "        tmp.append(fifty_zeroes)\n",
    "    master_x.append(tmp)\n",
    "\n",
    "master_y = []\n",
    "for sentence in corrected :\n",
    "    new = []\n",
    "    for word in sentence :\n",
    "        new.append(one_hot_corrected[word])\n",
    "    new = new[:NB_WORDS]\n",
    "    while(len(new) < NB_WORDS) :\n",
    "        new.append(one_hot_corrected[\"__unk__\"])\n",
    "    master_y.append(new[:])\n",
    "print('success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4500, 20, 100) (4500, 20, 2210) 450\n",
      "success\n"
     ]
    }
   ],
   "source": [
    "#splitting data into training and validation\n",
    "INPUT_SAMPLE_SIZE = 4500\n",
    "\n",
    "VALIDATION_SPLIT = 0.1\n",
    "master_x = np.array(master_x)\n",
    "master_y = np.array(master_y)\n",
    "indices = np.arange(master_x.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "master_x = master_x[indices]\n",
    "master_y = master_y[indices]\n",
    "\n",
    "x_data = master_x[:INPUT_SAMPLE_SIZE]\n",
    "y_data = master_y[:INPUT_SAMPLE_SIZE]\n",
    "\n",
    "nb_validation_samples = int(VALIDATION_SPLIT * x_data.shape[0])\n",
    "\n",
    "print(x_data.shape, y_data.shape, nb_validation_samples)\n",
    "\n",
    "x_train = x_data[:-nb_validation_samples]\n",
    "y_train = y_data[:-nb_validation_samples]\n",
    "x_val = x_data[-nb_validation_samples:]\n",
    "y_val = y_data[-nb_validation_samples:]\n",
    "\n",
    "print('success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "450 20 100\n",
      "Train on 4050 samples, validate on 450 samples\n",
      "Epoch 1/2\n",
      "4050/4050 [==============================] - 21s 5ms/step - loss: 6.6527 - acc: 0.3656 - val_loss: 5.1519 - val_acc: 0.3742\n",
      "Epoch 2/2\n",
      "4050/4050 [==============================] - 16s 4ms/step - loss: 4.5428 - acc: 0.3819 - val_loss: 4.2979 - val_acc: 0.3742\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x184264371d0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Dense\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers import RepeatVector\n",
    "\n",
    "print(len(x_val), len(x_val[0]), len(x_val[0][0]))\n",
    "\n",
    "n_features = IN_FEATURES\n",
    "n_timesteps_in = NB_WORDS\n",
    "batch_size = 100\n",
    "model = Sequential()\n",
    "model.add(LSTM(50, input_shape=(n_timesteps_in, n_features)))\n",
    "model.add(RepeatVector(n_timesteps_in))\n",
    "model.add(LSTM(50, return_sequences=True))\n",
    "model.add(TimeDistributed(Dense(out_feature + 1, activation='softmax')))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])\n",
    "\n",
    "# train LSTM\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          validation_data=[x_val, y_val],\n",
    "          epochs=2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}