notebooks/language_model_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import pandas as pd\n",
    "import random\n",
    "from collections import Counter\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read files and put all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.45 s, sys: 185 ms, total: 1.63 s\n",
      "Wall time: 1.63 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data_path = '../data/external/joke-dataset/'\n",
    "reddit_jokes = pd.read_json(data_path + 'reddit_jokes.json')\n",
    "wocka_jokes = pd.read_json(data_path + 'wocka.json' )\n",
    "stupid_jokes = pd.read_json(data_path + 'stupidstuff.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "208345"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reddit_jokes['body'] = reddit_jokes['title'] + '\\n' + reddit_jokes['body']\n",
    "del reddit_jokes['title']\n",
    "jokes = pd.concat([reddit_jokes['body'], wocka_jokes['body'], stupid_jokes['body']], axis = 0, ignore_index=True)\n",
    "len(jokes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing: data selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove empty texts\n",
    "jokes = jokes[~(jokes=='')]\n",
    "\n",
    "# Ensure no nulls and no empty\n",
    "assert sum(jokes=='')==0\n",
    "assert sum(jokes.isnull())==0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f2c7a214cc0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of jokes: 152493\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGZ5JREFUeJzt3X+Q3PV93/HnyxJgDbKRCGRHI6mR\nXJR0ZG4i0A0oEztzwok4yWmEU5dCGRA2tpKxmNqTa4uIJxXlx4xoK3vCGJPKQUVKHJ8pNoMGpCqq\noivDHwIhLHMSmOgs5LFuhDRBsuQzFPfou3/s59LlPnu3e3u3u1/E6zGzc7vv7+f73fd+b3df9/2x\nt4oIzMzMKn2o3Q2YmVnxOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPL\nTG93A4267LLLYsGCBXWP/8UvfsHFF1/cvIYaVNS+wL01oqh9gXtrRFH7gsZ7O3DgwD9ExOU1B0bE\n+/KydOnSmIi9e/dOaHyrFLWvCPfWiKL2FeHeGlHUviIa7w14Mep4j/VuJTMzyzgczMws43AwM7OM\nw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzy7xv/32GvT8sWP9MU5bb0zHM7TWWfWzjp5ty\n32YfBN5yMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwy\nNcNB0oclvSDph5IOS/qPqf6YpNclHUyXJakuSQ9JGpD0sqSrK5a1RtKRdFlTUV8qqT/N85AkNePB\nmplZfer530rvANdFxJCkC4DnJO1M0/5dRDwxavxKYFG6XAs8Alwr6VJgA9AJBHBA0vaIOJPGfBF4\nHtgBdAM7MTOztqi55RBlQ+nmBekS48yyGtiW5tsHzJI0B7ge2B0Rp1Mg7Aa607SPRsS+iAhgG3DD\nJB6TmZlNksrvxzUGSdOAA8AVwMMRcZekx4DforxlsQdYHxHvSHoa2BgRz6V59wB3AV3AhyPi/lT/\nM+BtoC+N/91U/yRwV0T8fpU+1gJrAUql0tLe3t66H+jQ0BAzZ86se3yrFLUvmJre+gfPTlE371Wa\nASffHn9Mx9xLmnLf4znff5/NUtTeitoXNN7b8uXLD0REZ61xdf3L7oh4F1giaRbwpKQrgbuBN4AL\ngc2UA+DeCXc6ARGxOd0XnZ2d0dXVVfe8fX19TGR8qxS1L5ia3mr9W+1G9XQMs6l//KfvsVu6mnLf\n4znff5/NUtTeitoXNL+3CZ2tFBE/A/YC3RFxIu06egf4b8A1adggML9itnmpNl59XpW6mZm1ST1n\nK12ethiQNAP4PeBH6VgB6cyiG4BDaZbtwG3prKVlwNmIOAHsAlZImi1pNrAC2JWmnZO0LC3rNuCp\nqX2YZmY2EfXsVpoDbE3HHT4EPB4RT0v6O0mXAwIOAn+cxu8AVgEDwFvA5wAi4rSk+4D9ady9EXE6\nXf8S8Bgwg/JZSj5TycysjWqGQ0S8DFxVpX7dGOMDWDfGtC3Alir1F4Era/ViZmat4U9Im5lZxuFg\nZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4\nmJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmaZml8TKunDwLPARWn8ExGxQdJCoBf4FeAAcGtE\n/FLSRcA2YCnwJvCvIuJYWtbdwB3Au8C/iYhdqd4N/DkwDfjLiNg4pY/SWLD+mQnP09MxzO0NzGdm\n73/1bDm8A1wXEb8JLAG6JS0DHgS+HhFXAGcov+mTfp5J9a+ncUhaDNwEfBzoBr4paZqkacDDwEpg\nMXBzGmtmZm1Sc8shIgIYSjcvSJcArgP+dapvBe4BHgFWp+sATwDfkKRU742Id4DXJQ0A16RxAxFx\nFEBSbxr7ymQemFkjW0uT1dMxTFfL79Vs6tV1zCH9hX8QOAXsBn4M/CwihtOQ48DcdH0u8FOANP0s\n5V1P/1gfNc9YdTMza5OaWw4AEfEusETSLOBJ4J81tasxSFoLrAUolUr09fXVPe/Q0NCExrdKq/rq\n6RiuPWiU0ozG5muFovZWmkEhn2dQ3NcAFLe3ovYFze+trnAYERE/k7QX+C1glqTpaetgHjCYhg0C\n84HjkqYDl1A+MD1SH1E5z1j10fe/GdgM0NnZGV1dXXX33tfXx0TGt0qr+mrkwHJPxzCb+if0FGmZ\novbW0zHMjQV8nkFxXwNQ3N6K2hc0v7eau5UkXZ62GJA0A/g94FVgL/DZNGwN8FS6vj3dJk3/u3Tc\nYjtwk6SL0plOi4AXgP3AIkkLJV1I+aD19ql4cGZm1ph6/vSaA2xNZxV9CHg8Ip6W9ArQK+l+4AfA\no2n8o8BfpQPOpym/2RMRhyU9TvlA8zCwLu2uQtKdwC7Kp7JuiYjDU/YIzcxswuo5W+ll4Koq9aP8\n/7ONKuv/G/iXYyzrAeCBKvUdwI46+jUzsxbwJ6TNzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAz\ns4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzM\nzCzjcDAzs4zDwczMMjXDQdJ8SXslvSLpsKQvp/o9kgYlHUyXVRXz3C1pQNJrkq6vqHen2oCk9RX1\nhZKeT/XvSrpwqh+omZnVr54th2GgJyIWA8uAdZIWp2lfj4gl6bIDIE27Cfg40A18U9I0SdOAh4GV\nwGLg5orlPJiWdQVwBrhjih6fmZk1oGY4RMSJiHgpXf858Cowd5xZVgO9EfFORLwODADXpMtARByN\niF8CvcBqSQKuA55I828Fbmj0AZmZ2eQpIuofLC0AngWuBP4EuB04B7xIeevijKRvAPsi4q/TPI8C\nO9MiuiPiC6l+K3AtcE8af0Wqzwd2RsSVVe5/LbAWoFQqLe3t7a2796GhIWbOnFn3+FZpVV/9g2cn\nPE9pBpx8uwnNTIGi9laaAb966SXtbqOqor4GoLi9FbUvaLy35cuXH4iIzlrjpte7QEkzge8BX4mI\nc5IeAe4DIv3cBHx+wp1OQERsBjYDdHZ2RldXV93z9vX1MZHxrdKqvm5f/8yE5+npGGZTf91PkZYq\nam89HcPcWMDnGRT3NQDF7a2ofUHze6vr1SXpAsrB8O2I+D5ARJysmP4t4Ol0cxCYXzH7vFRjjPqb\nwCxJ0yNieNR4MzNrg3rOVhLwKPBqRHytoj6nYthngEPp+nbgJkkXSVoILAJeAPYDi9KZSRdSPmi9\nPcr7tfYCn03zrwGemtzDMjOzyahny+G3gVuBfkkHU+1PKZ9ttITybqVjwB8BRMRhSY8Dr1A+02ld\nRLwLIOlOYBcwDdgSEYfT8u4CeiXdD/yAchiZmVmb1AyHiHgOUJVJO8aZ5wHggSr1HdXmi4ijlM9m\nMjOzAvAnpM3MLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOz\njMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzTM1wkDRf0l5Jr0g6LOnL\nqX6ppN2SjqSfs1Ndkh6SNCDpZUlXVyxrTRp/RNKaivpSSf1pnockVftaUjMza5F6thyGgZ6IWAws\nA9ZJWgysB/ZExCJgT7oNsBJYlC5rgUegHCbABuBayt8XvWEkUNKYL1bM1z35h2ZmZo2qGQ4RcSIi\nXkrXfw68CswFVgNb07CtwA3p+mpgW5TtA2ZJmgNcD+yOiNMRcQbYDXSnaR+NiH0REcC2imWZmVkb\nTOiYg6QFwFXA80ApIk6kSW8ApXR9LvDTitmOp9p49eNV6mZm1ibT6x0oaSbwPeArEXGu8rBARISk\naEJ/o3tYS3lXFaVSib6+vrrnHRoamtD4VmlVXz0dwxOepzSjsflaoai9lWZQyOcZFPc1AMXtrah9\nQfN7qyscJF1AORi+HRHfT+WTkuZExIm0a+hUqg8C8ytmn5dqg0DXqHpfqs+rMj4TEZuBzQCdnZ3R\n1dVVbVhVfX19TGR8q7Sqr9vXPzPheXo6htnUX/ffDy1V1N56Ooa5sYDPMyjuawCK21tR+4Lm91bP\n2UoCHgVejYivVUzaDoyccbQGeKqifls6a2kZcDbtftoFrJA0Ox2IXgHsStPOSVqW7uu2imWZmVkb\n1POn128DtwL9kg6m2p8CG4HHJd0B/AS4MU3bAawCBoC3gM8BRMRpSfcB+9O4eyPidLr+JeAxYAaw\nM13MzKxNaoZDRDwHjPW5g09VGR/AujGWtQXYUqX+InBlrV7M3g8WNLALb6oc2/jptt23nV/8CWkz\nM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPB\nzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzTM1wkLRF0ilJhypq90galHQw\nXVZVTLtb0oCk1yRdX1HvTrUBSesr6gslPZ/q35V04VQ+QDMzm7h6thweA7qr1L8eEUvSZQeApMXA\nTcDH0zzflDRN0jTgYWAlsBi4OY0FeDAt6wrgDHDHZB6QmZlNXs1wiIhngdN1Lm810BsR70TE68AA\ncE26DETE0Yj4JdALrJYk4DrgiTT/VuCGCT4GMzObYoqI2oOkBcDTEXFlun0PcDtwDngR6ImIM5K+\nAeyLiL9O4x4FdqbFdEfEF1L9VuBa4J40/opUnw/sHLmfKn2sBdYClEqlpb29vXU/0KGhIWbOnFn3\n+FZpVV/9g2cnPE9pBpx8uwnNTIGi9tbuvjrmXjLmtKK+BqC4vRW1L2i8t+XLlx+IiM5a46Y31BU8\nAtwHRPq5Cfh8g8uqW0RsBjYDdHZ2RldXV93z9vX1MZHxrdKqvm5f/8yE5+npGGZTf6NPkeYqam/t\n7uvYLV1jTivqawCK21tR+4Lm99bQszgiTo5cl/Qt4Ol0cxCYXzF0XqoxRv1NYJak6RExPGq8mZm1\nSUOnskqaU3HzM8DImUzbgZskXSRpIbAIeAHYDyxKZyZdSPmg9fYo79PaC3w2zb8GeKqRnszMbOrU\n3HKQ9B2gC7hM0nFgA9AlaQnl3UrHgD8CiIjDkh4HXgGGgXUR8W5azp3ALmAasCUiDqe7uAvolXQ/\n8APg0Sl7dGZm1pCa4RARN1cpj/kGHhEPAA9Uqe8AdlSpH6V8NpOZmRWEPyFtZmYZh4OZmWUcDmZm\nlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZ\nmWUcDmZmlinel/CaWcMWjPNd4T0dww19l3g9jm38dFOWa+3jLQczM8s4HMzMLFMzHCRtkXRK0qGK\n2qWSdks6kn7OTnVJekjSgKSXJV1dMc+aNP6IpDUV9aWS+tM8D0nSVD9IMzObmHq2HB4DukfV1gN7\nImIRsCfdBlgJLEqXtcAjUA4TYANwLeXvi94wEihpzBcr5ht9X2Zm1mI1wyEingVOjyqvBram61uB\nGyrq26JsHzBL0hzgemB3RJyOiDPAbqA7TftoROyLiAC2VSzLzMzapNFjDqWIOJGuvwGU0vW5wE8r\nxh1PtfHqx6vUzcysjSZ9KmtEhKSYimZqkbSW8u4qSqUSfX19dc87NDQ0ofGt0qq+ejqGJzxPaUZj\n87VCUXsral/Q3N4m+xz+oL8+G9Hs3hoNh5OS5kTEibRr6FSqDwLzK8bNS7VBoGtUvS/V51UZX1VE\nbAY2A3R2dkZXV9dYQzN9fX1MZHyrtKqvRs5v7+kYZlN/MT8KU9TeitoXNLe3Y7d0TWr+D/rrsxHN\n7q3R3UrbgZEzjtYAT1XUb0tnLS0DzqbdT7uAFZJmpwPRK4Bdado5ScvSWUq3VSzLzMzapOafEZK+\nQ/mv/sskHad81tFG4HFJdwA/AW5Mw3cAq4AB4C3gcwARcVrSfcD+NO7eiBg5yP0lymdEzQB2pouZ\nmbVRzXCIiJvHmPSpKmMDWDfGcrYAW6rUXwSurNWHmZm1jj8hbWZmGYeDmZllHA5mZpZxOJiZWcbh\nYGZmGYeDmZllHA5mZpZxOJiZWaaY/wTGzN5Xxvvu6no0+v3W/u7q5vGWg5mZZRwOZmaWcTiYmVnG\n4WBmZhmHg5mZZRwOZmaWcTiYmVnGn3NooWrngjd6freZWTN5y8HMzDIOBzMzy0wqHCQdk9Qv6aCk\nF1PtUkm7JR1JP2enuiQ9JGlA0suSrq5Yzpo0/oikNZN7SGZmNllTseWwPCKWRERnur0e2BMRi4A9\n6TbASmBRuqwFHoFymAAbgGuBa4ANI4FiZmbt0YzdSquBren6VuCGivq2KNsHzJI0B7ge2B0RpyPi\nDLAb6G5CX2ZmVidFROMzS68DZ4AA/mtEbJb0s4iYlaYLOBMRsyQ9DWyMiOfStD3AXUAX8OGIuD/V\n/wx4OyL+S5X7W0t5q4NSqbS0t7e37l6HhoaYOXNmw491KvQPns1qpRlw8u02NFMH9zZxRe0L3Fsj\nxuurY+4lrW1mlEbf05YvX36gYk/PmCZ7KusnImJQ0q8CuyX9qHJiRISkxtNnlIjYDGwG6OzsjK6u\nrrrn7evrYyLjm6HaKas9HcNs6i/mGcXubeKK2he4t0aM19exW7pa28wozX5Pm9RupYgYTD9PAU9S\nPmZwMu0uIv08lYYPAvMrZp+XamPVzcysTRoOB0kXS/rIyHVgBXAI2A6MnHG0BngqXd8O3JbOWloG\nnI2IE8AuYIWk2elA9IpUMzOzNpnMdlwJeLJ8WIHpwN9ExP+QtB94XNIdwE+AG9P4HcAqYAB4C/gc\nQESclnQfsD+NuzciTk+iLzMzm6SGwyEijgK/WaX+JvCpKvUA1o2xrC3AlkZ7MTOzqeVPSJuZWaZ4\npweYmb0PVPtHmq1wbOOnW3I/3nIwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgc\nzMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMwsU5gv+5HUDfw5MA34\ny4jY2Kz7ateXdJiZvV8UYstB0jTgYWAlsBi4WdLi9nZlZvbBVYhwAK4BBiLiaET8EugFVre5JzOz\nD6yihMNc4KcVt4+nmpmZtYEiot09IOmzQHdEfCHdvhW4NiLuHDVuLbA23fwN4LUJ3M1lwD9MQbtT\nrah9gXtrRFH7AvfWiKL2BY339msRcXmtQUU5ID0IzK+4PS/V3iMiNgObG7kDSS9GRGdj7TVPUfsC\n99aIovYF7q0RRe0Lmt9bUXYr7QcWSVoo6ULgJmB7m3syM/vAKsSWQ0QMS7oT2EX5VNYtEXG4zW2Z\nmX1gFSIcACJiB7CjiXfR0O6oFihqX+DeGlHUvsC9NaKofUGTeyvEAWkzMyuWohxzMDOzAjnvw0FS\nt6TXJA1IWt/mXuZL2ivpFUmHJX051e+RNCjpYLqsalN/xyT1px5eTLVLJe2WdCT9nN3inn6jYr0c\nlHRO0lfatc4kbZF0StKhilrVdaSyh9Jz72VJV7eht/8s6Ufp/p+UNCvVF0h6u2L9/UWL+xrz9yfp\n7rTOXpN0fbP6Gqe371b0dUzSwVRv5Tob672idc+1iDhvL5QPbv8Y+BhwIfBDYHEb+5kDXJ2ufwT4\ne8r/LuQe4N8WYH0dAy4bVftPwPp0fT3wYJt/n28Av9audQb8DnA1cKjWOgJWATsBAcuA59vQ2wpg\nerr+YEVvCyrHtaGvqr+/9Hr4IXARsDC9fqe1srdR0zcB/6EN62ys94qWPdfO9y2HQv1bjog4EREv\npes/B16l+J8EXw1sTde3Aje0sZdPAT+OiJ+0q4GIeBY4Pao81jpaDWyLsn3ALElzWtlbRPxtRAyn\nm/sof4aopcZYZ2NZDfRGxDsR8TowQPl13PLeJAm4EfhOs+5/LOO8V7TsuXa+h0Nh/y2HpAXAVcDz\nqXRn2hzc0updNxUC+FtJB1T+NDpAKSJOpOtvAKX2tAaUP/9S+UItwjqDsddR0Z5/n6f81+WIhZJ+\nIOl/SfpkG/qp9vsr0jr7JHAyIo5U1Fq+zka9V7TsuXa+h0MhSZoJfA/4SkScAx4B/imwBDhBeVO2\nHT4REVdT/u+46yT9TuXEKG+/tuX0NpU/HPkHwH9PpaKss/do5zoaj6SvAsPAt1PpBPBPIuIq4E+A\nv5H00Ra2VMjf3yg3894/Rlq+zqq8V/yjZj/XzvdwqOvfcrSSpAso/7K/HRHfB4iIkxHxbkT8X+Bb\nNHEzejwRMZh+ngKeTH2cHNk8TT9PtaM3yoH1UkScTD0WYp0lY62jQjz/JN0O/D5wS3pDIe22eTNd\nP0B53/6vt6qncX5/RVln04E/BL47Umv1Oqv2XkELn2vnezgU6t9ypH2YjwKvRsTXKuqV+wY/Axwa\nPW8LertY0kdGrlM+kHmI8vpak4atAZ5qdW/Je/6KK8I6qzDWOtoO3JbOJFkGnK3YJdASKn+J1r8H\n/iAi3qqoX67y96gg6WPAIuBoC/sa6/e3HbhJ0kWSFqa+XmhVXxV+F/hRRBwfKbRynY31XkErn2ut\nOPLezgvlo/h/Tznlv9rmXj5BeTPwZeBguqwC/groT/XtwJw29PYxymeJ/BA4PLKugF8B9gBHgP8J\nXNqG3i4G3gQuqai1ZZ1RDqgTwP+hvF/3jrHWEeUzRx5Oz71+oLMNvQ1Q3hc98nz7izT2X6Tf80Hg\nJeCft7ivMX9/wFfTOnsNWNnqdZbqjwF/PGpsK9fZWO8VLXuu+RPSZmaWOd93K5mZWQMcDmZmlnE4\nmJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmaZ/wfXxzuDTm7MKwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2c618b4ef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Remove very long jokes\n",
    "length_threshold=200\n",
    "jokes = jokes[jokes.apply(len)<length_threshold]\n",
    "jokes.apply(len).hist()\n",
    "print(\"Number of jokes: %d\" %len(jokes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing: data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lowercase\n",
    "jokes = jokes.apply(lambda x: x.lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[serious] holocaust jokes are not funny and never will be.\n",
      "it's a very sore spot for many people. personally, my grandfather died at dachau, he got drunk and fell out of his watchtower.\n",
      "-----\n",
      "i'm told i sleep like a baby.\n",
      "... i wake up crying every few hours\n",
      "-----\n",
      "it's international holocaust remembrance day\n",
      "and fugghedaboütit day in germany.\n",
      "-----\n",
      "a giraffe walks into a bar...\n",
      "and says highballs are on me. \n",
      "-----\n",
      "i don't think i'm ever going to score in asia...\n",
      "...everyone keeps refusing my tip.\n",
      "-----\n",
      "what did the poop say to the pee when he was hitting on her?\n",
      "i may be a two but your an eight..\n",
      "-----\n",
      "what my friend heard in the school hallway.\n",
      "guy one: i can put windows on any mac laptop.\n",
      "guy two: i can put mac on any windows laptop.\n",
      "*third guy shows up*\n",
      "i can put deez nutz on every laptop.\n",
      "-----\n",
      "what do you call a jewish pokemon trainer?\n",
      "ash\n",
      "-----\n",
      "why did the h kill himself?\n",
      "because the g had.\n",
      "-----\n",
      "i went tonthe zoo and all they had was one small dog\n",
      "and an empty gorilla enclosure...\n",
      "\n",
      "it was a shotzu.\n",
      "-----\n"
     ]
    }
   ],
   "source": [
    "# get some samples from the training corpus\n",
    "\n",
    "for i in range(10):\n",
    "    print(random.choice(list(jokes)))\n",
    "    print('-----')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create tokens and encode train and validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create token_dict based on ascii letters\n",
    "all_letters = string.ascii_letters[0:26] + \"\"\" \".,;'-+=?!$%():\\n\"\"\"\n",
    "token_dict = {token:k for token,k in zip(all_letters, range(1,len(all_letters)+1))}\n",
    "\n",
    "end_token = '<end>'\n",
    "pad_token = '<pad>'\n",
    "\n",
    "token_dict[pad_token] = 0\n",
    "token_dict[end_token] = len(token_dict)\n",
    "\n",
    "decoder_dict = {token_dict[k]:k for k in token_dict.keys()}\n",
    "tokens_count = len(token_dict)\n",
    "assert len(decoder_dict)==len(decoder_dict)\n",
    "tokens_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.83 s, sys: 22.1 ms, total: 1.85 s\n",
      "Wall time: 1.85 s\n"
     ]
    }
   ],
   "source": [
    "def text_encoder(text):\n",
    "    return [token_dict[token] for token in text if token in token_dict] + [token_dict['<end>']]\n",
    "\n",
    "%time jokes_encoded =  list( jokes.apply(text_encoder) ) # as a list is much faster to retrieve elements"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, \n",
    "                 tokens_count,\n",
    "                 embedding_dimension, \n",
    "                 rnn_dim,\n",
    "                 #relu_dim,\n",
    "                 num_layers, \n",
    "                 dropout):\n",
    "        \n",
    "        super(RNN, self).__init__()\n",
    "        \n",
    "        self.rnn_dim = rnn_dim\n",
    "        \n",
    "        self.embedding = nn.Embedding(tokens_count, embedding_dimension)\n",
    "        self.lstm = nn.LSTM(input_size = embedding_dimension, \n",
    "                            hidden_size = rnn_dim, \n",
    "                            num_layers = num_layers, \n",
    "                            dropout = dropout,\n",
    "                            batch_first = True)\n",
    "        #self.linear_1 =  nn.Linear(rnn_dim , relu_dim) \n",
    "        #self.relu_1 = nn.ReLU()\n",
    "        #self.linear_2 =  nn.Linear(relu_dim , tokens_count)\n",
    "        #self.relu_2 = nn.ReLU()\n",
    "        \n",
    "        self.linear = nn.Linear(rnn_dim,tokens_count)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.softmax = nn.LogSoftmax(dim=2)\n",
    "\n",
    "    def forward(self, inp, seq_length, hidden_states, train=True):\n",
    "        embedded = self.embedding(inp)\n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, seq_length, batch_first=True)\n",
    "        packed_output, (ht, ct) = self.lstm(packed_embedded, hidden_states)\n",
    "        (lstm_output, length) = nn.utils.rnn.pad_packed_sequence(packed_output, batch_first=True)\n",
    "        \n",
    "#         output_linear_1 = self.linear_1(lstm_output)\n",
    "#         output_relu_1 = self.relu_1(output_linear_1)\n",
    "#         output_linear_2 = self.linear_2(output_relu_1)\n",
    "#         output_relu_2 = self.relu_2(output_linear_2)\n",
    "        \n",
    "        output_linear = self.linear(lstm_output)\n",
    "        if train: output = self.dropout(output_linear)\n",
    "        output = self.softmax(output_linear)\n",
    "        return output, (ht, ct)\n",
    "\n",
    "    def initRNN(self, batch_size):\n",
    "        if use_cuda:\n",
    "            return (Variable(torch.zeros(num_layers, batch_size, self.rnn_dim).cuda()), \n",
    "                Variable(torch.zeros(num_layers, batch_size, self.rnn_dim)).cuda())\n",
    "        else:\n",
    "            return (Variable(torch.zeros(num_layers, batch_size, self.rnn_dim)), \n",
    "                Variable(torch.zeros(num_layers, batch_size, self.rnn_dim)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = True\n",
    "embedding_dimension = 1024\n",
    "num_layers = 3\n",
    "rnn_dim = 2048\n",
    "#relu_dim = 256\n",
    "\n",
    "dropout = 0.1\n",
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if use_cuda:\n",
    "    rnn = RNN(tokens_count, embedding_dimension, rnn_dim, num_layers, dropout).cuda()\n",
    "else:\n",
    "    rnn = RNN(tokens_count, embedding_dimension, rnn_dim, num_layers, dropout)\n",
    "loss_function = nn.NLLLoss(ignore_index=token_dict[pad_token])\n",
    "optimizer = torch.optim.Adam(rnn.parameters())\n",
    "learning_rate = 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters count: 98439\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method Module.named_parameters of RNN(\n",
       "  (embedding): Embedding(45, 1024)\n",
       "  (lstm): LSTM(1024, 2048, num_layers=3, batch_first=True, dropout=0.1)\n",
       "  (linear): Linear(in_features=2048, out_features=45)\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (softmax): LogSoftmax()\n",
       ")>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of parameters\n",
    "parameters_count=0\n",
    "for a in rnn.parameters():\n",
    "    parameters_count+=len(a)\n",
    "print(\"Parameters count: %d\" %parameters_count)\n",
    "rnn.named_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lengths(v):\n",
    "    return np.array([i for i in map(len, v)])\n",
    "\n",
    "def pad(v):\n",
    "    lens = np.array([len(item) for item in v])\n",
    "    mask = lens[:,None] > np.arange(lens.max())\n",
    "    out = np.zeros(mask.shape,dtype=int)\n",
    "    out[mask] = np.concatenate(v)\n",
    "    return out\n",
    "\n",
    "losses = []\n",
    "step_count = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load previously trained model\n",
    "rnn.load_state_dict(torch.load('../models/model.pickle'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bartender said, \"knock knock\"\n",
      "\"who's there?\"\n",
      "\n",
      "\"brat\"\n",
      "\n",
      "\"bark walker walks into a bar.\"<end>\n"
     ]
    }
   ],
   "source": [
    "def generates_sequence(start_string, max_length):\n",
    "    \n",
    "    input_encoded = []\n",
    "    for char in start_string:\n",
    "        input_encoded.append(token_dict[char])\n",
    "\n",
    "    top_index = 0\n",
    "    result = start_string\n",
    "    hidden = rnn.initRNN(1)\n",
    "    \n",
    "    if use_cuda:\n",
    "        input_var =Variable(torch.cuda.LongTensor([input_encoded]))\n",
    "    else:\n",
    "        input_var = Variable(torch.LongTensor([input_encoded]))\n",
    "    \n",
    "    while input_encoded[0] != token_dict['<end>']:\n",
    "    \n",
    "        \n",
    "        output, hidden = rnn(input_var,\n",
    "                              np.array([len(input_encoded)]), \n",
    "                              hidden ,\n",
    "                              train = True)\n",
    "\n",
    "        input_encoded = [int(output.cpu().data.topk(1,dim=2)[1].numpy()[0][-1])]\n",
    "       \n",
    "        result+=decoder_dict[input_encoded[0]]\n",
    "        \n",
    "        if len(result)==max_length: break\n",
    "        \n",
    "        if use_cuda:\n",
    "            input_var = Variable(torch.cuda.LongTensor(input_encoded)).view(1,1)\n",
    "        else:\n",
    "            input_var = Variable(torch.LongTensor(input_encoded)).view(1,1)\n",
    "        \n",
    "    return result\n",
    "\n",
    "print(generates_sequence('bartender said, \"knock knock\"', 1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 1283/9530 [28:09<3:00:59,  1.32s/it]"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    random.shuffle(jokes_encoded)\n",
    "    batches = np.array_split(jokes_encoded, int(len(jokes_encoded)/batch_size))\n",
    "\n",
    "    for batch in tqdm(batches):\n",
    "\n",
    "        # Prepare batch input and target (sort and pad)\n",
    "        batch = list(batch)\n",
    "        batch.sort(key=len, reverse=True)\n",
    "\n",
    "        batch_input = [i[:-1] for i in batch]\n",
    "        batch_target = [i[1:] for i in batch]\n",
    "\n",
    "        batch_input_pad = pad(batch_input)\n",
    "        batch_target_pad = pad(batch_target)\n",
    "\n",
    "        # Forward pass and loss function\n",
    "        if use_cuda:\n",
    "            input_var = Variable(torch.cuda.LongTensor(batch_input_pad))\n",
    "        else:\n",
    "            input_var = Variable(torch.LongTensor(batch_input_pad))\n",
    "        \n",
    "        output, _ = rnn(input_var, get_lengths(batch_input), \n",
    "                                  rnn.initRNN(len(batch_input)) ,\n",
    "                                  train = True)\n",
    "        flat_output = output.contiguous().view(-1,tokens_count)\n",
    "        if use_cuda:\n",
    "            flat_target = Variable(torch.cuda.LongTensor(batch_target_pad.flatten()))\n",
    "        else:\n",
    "            flat_target = Variable(torch.LongTensor(batch_target_pad.flatten()))\n",
    "        loss = loss_function(flat_output , flat_target)  # loss is already averaged\n",
    "        losses.append(float(loss))\n",
    "        \n",
    "        # Optimise\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        step_count+=1\n",
    "        \n",
    "#         if step_count%100 == 0:\n",
    "#             print(generates_sequence(random.choice(string.ascii_letters[0:26]), 500))\n",
    "#             print('-----')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a dog was arrested for salt\n",
      "the couch was coming out of the closet and said \"i don't know what they're saying when i was born in the morning.\"<end>\n",
      "---\n",
      "a dog was walking down the street and saw a man walking down the street...\n",
      "he said, \"i was gonna see your mom a chicken!\"<end>\n",
      "---\n",
      "a dog walks into a bar...\n",
      "and says \"hey, it's driving me nuts\"<end>\n",
      "---\n",
      "a dog was walking down the street and saw a man walking down the street...\n",
      "he said \"hey man, i can see your password and the dog should be a dog.\" the guy says, \"no, i just walked out of here and should i see you there.\"<end>\n",
      "---\n",
      "a dog walks into a bar...\n",
      "and he says \"ouch!\"<end>\n",
      "---\n",
      "a dog walks into a bar...\n",
      "and the bartender says \"why the long face?\"<end>\n",
      "---\n",
      "a dog walks into a bar...\n",
      "and says \"ouch!\"<end>\n",
      "---\n",
      "a dog walks into a bar...\n",
      "and says \"i'm looking for the man who shot my paw\"<end>\n",
      "---\n",
      "a dog was arrested the other day...\n",
      "he was a man of a bit of a million dollars.<end>\n",
      "---\n",
      "a dog walks into a bar...\n",
      "and he says \"i can clearly see your nuts\"<end>\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(generates_sequence('a dog', 500))\n",
    "    print('---')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(rnn.state_dict(), '../models/model.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}