Seq2Seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import json\n",
    "import os,time\n",
    "from faker import Faker\n",
    "import babel\n",
    "from babel.dates import format_date\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.legacy_seq2seq as seq2seq\n",
    "from os.path import isfile, isdir, getsize\n",
    "from tqdm import tqdm\n",
    "import zipfile\n",
    "from urllib import urlretrieve\n",
    "from IPython.display import clear_output, Image, display, HTML\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This cell contains helper methods..\n",
    "class DLProgress(tqdm):\n",
    "    last_block = 0\n",
    "\n",
    "    def hook(self, block_num=1, block_size=1, total_size=None):\n",
    "        self.total = total_size\n",
    "        self.update((block_num - self.last_block) * block_size)\n",
    "        self.last_block = block_num\n",
    "\n",
    "\n",
    "def downloadData(file, url):        \n",
    "    if not isfile(file):\n",
    "        with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Fake News Dataset') as pbar:\n",
    "            urlretrieve(url, file, pbar.hook)\n",
    "\n",
    "    with zipfile.ZipFile(file) as f:\n",
    "        f.extractall('./data/')\n",
    "        \n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = \"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fake = Faker()\n",
    "fake.seed(42)\n",
    "random.seed(42)\n",
    "FORMATS = ['short',\n",
    "           'medium',\n",
    "           'long',\n",
    "           'full',\n",
    "           'd MMM YYY',\n",
    "           'd MMMM YYY',\n",
    "           'dd MMM YYY',\n",
    "           'd MMM, YYY',\n",
    "           'd MMMM, YYY',\n",
    "           'dd, MMM YYY',\n",
    "           'd MM YY',\n",
    "           'd MMMM YYY',\n",
    "           'MMMM d YYY',\n",
    "           'MMMM d, YYY',\n",
    "           'dd.MM.YY',\n",
    "           ]\n",
    "\n",
    "# change this if you want it to work with only a single language\n",
    "LOCALES = babel.localedata.locale_identifiers()\n",
    "LOCALES = [lang for lang in LOCALES if 'en' in str(lang)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_date():\n",
    "    \"\"\"\n",
    "        Creates some fake dates \n",
    "        :returns: tuple containing \n",
    "                  1. human formatted string\n",
    "                  2. machine formatted string\n",
    "                  3. date object.\n",
    "    \"\"\"\n",
    "    dt = fake.date_object()\n",
    "    try:\n",
    "        human = format_date(dt,\n",
    "                            format=random.choice(FORMATS),\n",
    "                            locale=random.choice(LOCALES))\n",
    "\n",
    "        case_change = random.randint(0,3) # 1/2 chance of case change\n",
    "        if case_change == 1:\n",
    "            human = human.upper()\n",
    "        elif case_change == 2:\n",
    "            human = human.lower()\n",
    "        machine = dt.isoformat()\n",
    "    except AttributeError as e:\n",
    "        return None, None, None\n",
    "\n",
    "    return human, machine #, dt\n",
    "\n",
    "data = [create_date() for _ in range(50000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'22, OCT 2000', '2000-10-22'),\n",
       " (u'wednesday, 17 march 1971', '1971-03-17'),\n",
       " (u'APRIL 3, 1983', '1983-04-03'),\n",
       " (u'03/10/1980', '1980-10-03'),\n",
       " (u'26/06/2005', '2005-06-26')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Now extract the src and targets into seperate lists\n",
    "\n",
    "source_list = [tpl1 for tpl1,tpl_2 in  data]\n",
    "target_list = [tpl_2 for tpl1,tpl_2 in  data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now we will find the unique characters coming in the date format\n",
    "#1. To get lookup for src -> num \n",
    "unique_chars_src = set(' '.join(source_list))\n",
    "char_num_dict_src = dict(zip(unique_chars_src,range(len(unique_chars_src))))\n",
    "char_num_dict_src['<PAD_VAR>']=len(char_num_dict_src)\n",
    "# Reverse lookup\n",
    "num_char_dict_src= dict((v, k) for k, v in char_num_dict_src.iteritems())\n",
    "\n",
    "\n",
    "#2. To get lookup for dest -> num\n",
    "unique_chars_dest = set(' '.join(target_list))\n",
    "char_num_dict_dest = dict(zip(unique_chars_dest,range(len(unique_chars_dest))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now we have to do static padding.. We will also demo the same with dynamic padding and bucket later..\n",
    "max_src_len = max([len(date) for date in source_list])\n",
    "x = [[char_num_dict_src['<PAD_VAR>']]* (max_src_len -len(date)) + [char_num_dict_src[cur_char] for cur_char in date] for date in source_list]\n",
    "x = np.array(x)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Do the STOP pad for target also\n",
    "char_num_dict_dest['<STOP>'] = len(char_num_dict_dest)\n",
    "# Reverse lookup\n",
    "num_char_dict_dest = dict((v, k) for k, v in char_num_dict_dest.iteritems())\n",
    "y = [[char_num_dict_dest['<STOP>']]+[char_num_dict_dest[cur_char] for cur_char in date] for date in target_list]\n",
    "y=np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_seq_length = len(x[0])\n",
    "y_seq_length  = len(y[0])-1 # Stop  pad is added"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def batch_data(x,y,batch_size):\n",
    "    start = 0\n",
    "    shuffle = np.random.permutation(len(x))\n",
    "    x = x[shuffle]\n",
    "    y=y[shuffle]\n",
    "    while start+batch_size <= len(x):\n",
    "        yield x[start:start+batch_size], y[start:start+batch_size]\n",
    "        start += batch_size\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 2\n",
    "batch_size = 128\n",
    "nodes = 32\n",
    "embed_size = 10\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "learning_rate = 1e-3\n",
    "\n",
    "# The data feeder placeholders..\n",
    "inputs = tf.placeholder(tf.int32,shape = (None,x_seq_length),name='inputs')\n",
    "outputs = tf.placeholder(tf.int32,shape=(None,None),name='outputs')\n",
    "targets = tf.placeholder(tf.int32,shape=(None,None),name='targets')\n",
    "\n",
    "\n",
    "#Embedding layers\n",
    "input_embedding = tf.Variable(tf.random_uniform((len(char_num_dict_src),embed_size),-1.0,1.0),name='enc_embedding')\n",
    "output_embedding = tf.Variable(tf.random_uniform((len(char_num_dict_dest),embed_size),-1.0,1.0),name='dec_embedding')\n",
    "\n",
    "# Now look ups\n",
    "\n",
    "date_input_embed = tf.nn.embedding_lookup(input_embedding,inputs)\n",
    "date_output_embed = tf.nn.embedding_lookup(output_embedding,outputs)\n",
    "\n",
    "with tf.variable_scope(\"encoding\") as encoding_scope:\n",
    "    lstm_enc = tf.nn.rnn_cell.BasicLSTMCell(nodes)\n",
    "    _,last_state = tf.nn.dynamic_rnn(lstm_enc,inputs=date_input_embed,dtype=tf.float32)\n",
    "\n",
    "with tf.variable_scope(\"decoding\") as decoding_scope:\n",
    "    lstm_dec = tf.nn.rnn_cell.BasicLSTMCell(nodes)\n",
    "    dec_outputs,_ = tf.nn.dynamic_rnn(lstm_dec,inputs=date_output_embed,initial_state=last_state)\n",
    "    \n",
    "logits = tf.contrib.layers.fully_connected(dec_outputs,num_outputs=len(char_num_dict_dest),activation_fn=None)\n",
    "\n",
    "with tf.name_scope(\"optimization\"):\n",
    "    loss = tf.contrib.seq2seq.sequence_loss(logits,targets,tf.ones([batch_size,y_seq_length]))\n",
    "    optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None, 32]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dec_outputs.get_shape().as_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   0 Loss:  1.215 Accuracy: 0.5813 Epoch duration:  7.343s\n",
      "Epoch   1 Loss:  0.793 Accuracy: 0.7219 Epoch duration:  6.724s\n",
      "Epoch   2 Loss:  0.608 Accuracy: 0.7883 Epoch duration:  7.500s\n",
      "Epoch   3 Loss:  0.458 Accuracy: 0.8477 Epoch duration:  8.178s\n",
      "Epoch   4 Loss:  0.393 Accuracy: 0.8688 Epoch duration:  7.882s\n",
      "Epoch   5 Loss:  0.328 Accuracy: 0.8898 Epoch duration:  7.267s\n",
      "Epoch   6 Loss:  0.290 Accuracy: 0.9008 Epoch duration:  7.177s\n",
      "Epoch   7 Loss:  0.206 Accuracy: 0.9344 Epoch duration:  7.132s\n",
      "Epoch   8 Loss:  0.186 Accuracy: 0.9437 Epoch duration:  7.123s\n",
      "Epoch   9 Loss:  0.149 Accuracy: 0.9531 Epoch duration:  7.277s\n"
     ]
    }
   ],
   "source": [
    "sess.run(tf.global_variables_initializer())\n",
    "epochs = 10\n",
    "for epoch_i in range(epochs):\n",
    "    start_time = time.time()\n",
    "    for batch_i, (source_batch, target_batch) in enumerate(batch_data(X_train, y_train, batch_size)):\n",
    "        _, batch_loss, batch_logits = sess.run([optimizer, loss, logits],\n",
    "            feed_dict = {inputs: source_batch,\n",
    "             outputs: target_batch[:, :-1],\n",
    "             targets: target_batch[:, 1:]})\n",
    "    accuracy = np.mean(batch_logits.argmax(axis=-1) == target_batch[:,1:])\n",
    "    print('Epoch {:3} Loss: {:>6.3f} Accuracy: {:>6.4f} Epoch duration: {:>6.3f}s'.format(epoch_i, batch_loss, \n",
    "                                                                      accuracy, time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test set is:  0.929\n",
      "25 APR 1983 => 1983-04-25\n",
      "16 08 02 => 2002-08-16\n"
     ]
    }
   ],
   "source": [
    "source_batch, target_batch = next(batch_data(X_test, y_test, batch_size))\n",
    "dec_input = np.zeros((len(source_batch), 1)) + char_num_dict_dest['<STOP>']\n",
    "for i in range(y_seq_length):\n",
    "    batch_logits = sess.run(logits,\n",
    "                feed_dict = {inputs: source_batch,\n",
    "                 outputs: dec_input})\n",
    "    prediction = batch_logits[:,-1].argmax(axis=-1)\n",
    "    dec_input = np.hstack([dec_input, prediction[:,None]])\n",
    "    \n",
    "print('Accuracy on test set is: {:>6.3f}'.format(np.mean(dec_input == target_batch)))\n",
    "\n",
    "\n",
    "\n",
    "num_preds = 2\n",
    "source_chars = [[num_char_dict_src[l] for l in sent if num_char_dict_src[l]!=\"<PAD_VAR>\"] for sent in source_batch[:num_preds]]\n",
    "dest_chars = [[num_char_dict_dest[l] for l in sent] for sent in dec_input[:num_preds, 1:]]\n",
    "\n",
    "for date_in, date_out in zip(source_chars, dest_chars):\n",
    "    print(''.join(date_in)+' => '+''.join(date_out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}