NMT-pairwise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### An LSTM encoder-decoder based implementation of Neural Machine Translation on German-English dataset\n",
    "(https://arxiv.org/pdf/1406.1078.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import numpy as np\n",
    "np.set_printoptions(precision=2)\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import clear_output\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import keras.backend as K\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import RepeatVector\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "from unicodedata import normalize\n",
    "import string\n",
    "import pickle\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Pre-processing ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the file to preprocess\n",
    "def load_file(filename):\n",
    "    '''Function to load file'''\n",
    "    file = open(filename, mode='rt', encoding='utf-8')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text\n",
    "\n",
    "def getPairs(text):\n",
    "    '''Function to get eng-german keyword pairs from input text'''\n",
    "    pairs = [line.strip().split('\\t') for line in text.strip().split('\\n')]\n",
    "    return pairs\n",
    "\n",
    "def cleanPairs(lines):\n",
    "    '''Function to clean the eng-german keyword pair list and removing unprintable chars'''\n",
    "    cleaned = list()\n",
    "    # Regex for char filtering\n",
    "    re_print = re.compile('[^%s]' % re.escape(string.printable))\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    for pair in lines:\n",
    "        clean_pair = list()\n",
    "        for line in pair:\n",
    "#             print(\"line0: {0}\".format(line))\n",
    "            line = normalize('NFD', line).encode('ascii', 'ignore')\n",
    "#             print(\"line1: {0}\".format(line))\n",
    "            line = line.decode('UTF-8')\n",
    "            line = line.split()\n",
    "#             print(\"line2: {0}\".format(line))\n",
    "            line = [word.lower() for word in line]\n",
    "            line = [word.translate(table) for word in line]\n",
    "#             print(\"line22: {0}\".format(line))\n",
    "            line = [re_print.sub('', word) for word in line]\n",
    "            # Remove numeric chrs\n",
    "            line = [w for w in line if w.isalpha()]\n",
    "#             print(\"line3: {0}\".format(line))\n",
    "            clean_pair.append(' '.join(line))\n",
    "        cleaned.append(clean_pair)\n",
    "    return np.array(cleaned)\n",
    "\n",
    "# Save the cleaned data to the given filename\n",
    "def save_data(sentences, filename):\n",
    "    '''Function to save save the file in pickle format'''\n",
    "    pickle.dump(sentences, open(filename, 'wb'))\n",
    "    print('Saved: %s' % filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved: english-german.pickle\n"
     ]
    }
   ],
   "source": [
    "filename = 'deu.txt'\n",
    "file = load_file(filename)\n",
    "pairs = getPairs(file)\n",
    "cleaned_pairs = cleanPairs(pairs)\n",
    "save_data(cleaned_pairs, 'english-german.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "169813"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cleaned_pairs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating train/validation/test datasets ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_pkl_data(filename):\n",
    "    '''Function to load pickle file'''\n",
    "    file = open(filename, 'rb')\n",
    "    return pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved: english-german-both.pickle\n",
      "Saved: english-german-train.pickle\n",
      "Saved: english-german-test.pickle\n"
     ]
    }
   ],
   "source": [
    "raw_data = load_pkl_data('english-german.pickle')\n",
    "\n",
    "dataset = raw_data[:10000, :]\n",
    "random.shuffle(dataset)\n",
    "\n",
    "train_set = dataset[:9000, :]\n",
    "test_set = dataset[9000:, :]\n",
    "\n",
    "save_data(dataset, 'english-german-both.pickle')\n",
    "save_data(train_set, 'english-german-train.pickle')\n",
    "save_data(test_set, 'english-german-test.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding the input/output & model training ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit a tokenizer\n",
    "def create_tokenizer(lines):\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(lines)\n",
    "    return tokenizer\n",
    "\n",
    "# max sentence length\n",
    "def max_length(lines):\n",
    "    return max(len(line.split()) for line in lines)\n",
    "\n",
    "# encode and pad sequences\n",
    "def encode_sequences(tokenizer, length, lines):\n",
    "    # integer encode sequences\n",
    "    X = tokenizer.texts_to_sequences(lines)\n",
    "    # pad sequences with 0 values\n",
    "    X = pad_sequences(X, maxlen=length, padding='post')\n",
    "    return X\n",
    "\n",
    "# one hot encode target sequence\n",
    "def encode_output(sequences, vocab_size):\n",
    "    ylist = list()\n",
    "    for sequence in sequences:\n",
    "        encoded = to_categorical(sequence, num_classes=vocab_size)\n",
    "        ylist.append(encoded)\n",
    "    y = np.array(ylist)\n",
    "    y = y.reshape(sequences.shape[0], sequences.shape[1], vocab_size)\n",
    "    return y\n",
    "\n",
    "# Map an integer to a word\n",
    "def map_int_to_word(integer, tokenizer):\n",
    "    for word, index in tokenizer.word_index.items():\n",
    "        if index == integer:\n",
    "            return word\n",
    "    return None\n",
    "\n",
    "# Predict the target sequence\n",
    "def predict_sequence(model, tokenizer, source):\n",
    "    pred = model.predict(source, verbose=0)[0]\n",
    "    integers = [np.argmax(vector) for vector in pred]\n",
    "    target = list()\n",
    "    for i in integers:\n",
    "        word = map_int_to_word(i, tokenizer)\n",
    "        if word is None:\n",
    "            break\n",
    "        target.append(word)\n",
    "    return ' '.join(target)\n",
    "\n",
    "# Evaluate the model\n",
    "def evaluate_model(model, tokenizer, source, raw_dataset, cutoff_length=2):\n",
    "    predicted, actual = list(), list()\n",
    "    cnt = 0\n",
    "    for i, sentence in enumerate(source):\n",
    "        raw_target, raw_source = raw_dataset[i]\n",
    "        if(len(raw_target.split()) < cutoff_length):\n",
    "            continue\n",
    "        s = sentence.reshape((1, sentence.shape[0]))\n",
    "        translation = predict_sequence(model, tokenizer, s)\n",
    "        if cnt < 10:\n",
    "            print('src=[%s], target=[%s], predicted=[%s]' % (raw_source, raw_target, translation))\n",
    "        actual.append(raw_target.split())\n",
    "        predicted.append(translation.split())\n",
    "        cnt += 1\n",
    "\n",
    "    # Bleu Scores\n",
    "    print('Bleu-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n",
    "    print('Bleu-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))\n",
    "    print('Bleu-3: %f' % corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))\n",
    "    print('Bleu-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))\n",
    "\n",
    "# define NMT model\n",
    "def define_model(src_vocab, tar_vocab, src_timesteps, tar_timesteps, n_units):\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(src_vocab, n_units, input_length=src_timesteps, mask_zero=True))\n",
    "    model.add(LSTM(n_units))\n",
    "    model.add(RepeatVector(tar_timesteps))\n",
    "    model.add(LSTM(n_units, return_sequences=True))\n",
    "    model.add(TimeDistributed(Dense(tar_vocab, activation='softmax')))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PlotLearning(keras.callbacks.Callback):\n",
    "    '''Class to plot live training loss'''\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.i = 0\n",
    "        self.x = []\n",
    "        self.losses = []\n",
    "        self.val_losses = []\n",
    "        self.acc = []\n",
    "        self.val_acc = []\n",
    "        self.fig = plt.figure()\n",
    "        self.logs = []\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        self.logs.append(logs)\n",
    "        self.x.append(self.i)\n",
    "        self.losses.append(logs.get('loss'))\n",
    "        self.val_losses.append(logs.get('val_loss'))\n",
    "        # self.acc.append(logs.get('acc'))\n",
    "        # self.val_acc.append(logs.get('val_acc'))\n",
    "        self.i += 1\n",
    "        f, ax1 = plt.subplots(1, 1, sharex=True, figsize=(10,6))\n",
    "        clear_output(wait=True)\n",
    "\n",
    "        ax1.plot(self.x, self.losses, label=\"loss\")\n",
    "        ax1.plot(self.x, self.val_losses, label=\"val_loss\")\n",
    "        ax1.legend()\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English Vocabulary Size: 1564\n",
      "English Max Length: 5\n",
      "German Vocabulary Size: 2393\n",
      "German Max Length: 8\n",
      "WARNING:tensorflow:From C:\\Users\\Z0042ZRF\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 8, 256)            612608    \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 256)               525312    \n",
      "_________________________________________________________________\n",
      "repeat_vector_1 (RepeatVecto (None, 5, 256)            0         \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 5, 256)            525312    \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 5, 1564)           401948    \n",
      "=================================================================\n",
      "Total params: 2,065,180\n",
      "Trainable params: 2,065,180\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# load datasets\n",
    "dataset = load_pkl_data('english-german-both.pickle')\n",
    "train = load_pkl_data('english-german-train.pickle')\n",
    "test = load_pkl_data('english-german-test.pickle')\n",
    "\n",
    "# prepare english tokenizer\n",
    "eng_tokenizer = create_tokenizer(dataset[:, 0])\n",
    "eng_vocab_size = len(eng_tokenizer.word_index) + 1\n",
    "eng_length = max_length(dataset[:, 0])\n",
    "print('English Vocabulary Size: %d' % eng_vocab_size)\n",
    "print('English Max Length: %d' % (eng_length))\n",
    "\n",
    "# prepare german tokenizer\n",
    "ger_tokenizer = create_tokenizer(dataset[:, 1])\n",
    "ger_vocab_size = len(ger_tokenizer.word_index) + 1\n",
    "ger_length = max_length(dataset[:, 1])\n",
    "print('German Vocabulary Size: %d' % ger_vocab_size)\n",
    "print('German Max Length: %d' % (ger_length))\n",
    "\n",
    "# prepare training data\n",
    "trainX = encode_sequences(ger_tokenizer, ger_length, train[:, 1])\n",
    "trainY = encode_sequences(eng_tokenizer, eng_length, train[:, 0])\n",
    "trainY = encode_output(trainY, eng_vocab_size)\n",
    "\n",
    "# prepare validation data\n",
    "testX = encode_sequences(ger_tokenizer, ger_length, test[:, 1])\n",
    "testY = encode_sequences(eng_tokenizer, eng_length, test[:, 0])\n",
    "testY = encode_output(testY, eng_vocab_size)\n",
    "\n",
    "# define model\n",
    "model = define_model(ger_vocab_size, eng_vocab_size, ger_length, eng_length, 256)\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy')\n",
    "# summarize defined model\n",
    "print(model.summary())\n",
    "# plot_model(model, to_file='model.png', show_shapes=True)\n",
    "# fit model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAFpCAYAAABeYWb6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd8leX9//HXlb1DFhlAEvZKGCZsiVsRFNzirKjgRm21/ba1trX667Bq3VatChUVHBVU3IoMBUmQPWWEDUmAMELIun5/3EEiBghykvvknPfz8bgf55z73DnnE2z1zXVd9+cy1lpERERE5MQFuF2AiIiIiK9QsBIRERHxEAUrEREREQ9RsBIRERHxEAUrEREREQ9RsBIRERHxEAUrEREREQ9RsBIRERHxEAUrEREREQ9RsBIRERHxkCC3vjgxMdFmZma69fUiIiIiDVZQUFBsrU061nWuBavMzEzy8/Pd+noRERGRBjPGFDbkOk0FioiIiHiIgpWIiIiIhyhYiYiIiHiIa2usREREpOlUVlayceNGysvL3S7Fq4WFhdG6dWuCg4N/1s8rWImIiPiBjRs3Eh0dTWZmJsYYt8vxStZaSkpK2LhxI23btv1Zn6GpQBERET9QXl5OQkKCQtVRGGNISEg4oVE9BSsRERE/oVB1bCf6Z6RgJSIiIk0iKirK7RIanYKViIiIiIcoWImIiEiTstZy7733kpWVRXZ2NhMnTgRgy5Yt5OXl0atXL7KyspgxYwbV1dVcd911P1z72GOPuVz90emuQBERET/z5/eWsHTzbo9+Zre0GP54fvcGXfvOO+8wf/58FixYQHFxMX369CEvL4/XXnuNc845h9///vdUV1dTVlbG/Pnz2bRpE4sXLwZg165dHq3b03x2xGrfgSq+WL6Nnfsq3C5FRERE6pg5cyZXXHEFgYGBJCcnc8oppzB37lz69OnDyy+/zJ/+9CcWLVpEdHQ07dq1Y82aNdxxxx189NFHxMTEuF3+UfnsiNXqor1c/0o+T17Rm/N7prldjoiIiNdo6MhSY7HW1ns+Ly+P6dOn88EHH3DNNddw7733cu2117JgwQI+/vhjnn76aSZNmsRLL73UxBU3nM+OWHVNjSE8OJCCwp1ulyIiIiJ15OXlMXHiRKqrqykqKmL69On07duXwsJCWrZsyejRo7nhhhuYN28excXF1NTUcPHFF/OXv/yFefPmuV3+UfnsiFVwYAC92rQgv3CH26WIiIhIHRdeeCHffPMNPXv2xBjDP/7xD1JSUhg3bhwPP/wwwcHBREVFMX78eDZt2sSoUaOoqakB4K9//avL1R+dOdJwXGPLzc21+fn5jfodj3yygmemrWbhH88mMtRnM6SIiMgxLVu2jK5du7pdRrNQ35+VMabAWpt7rJ/12alAgJyMOKprLPM3ePcdBCIiIuIbfDpYnZQRhzGQv07rrERERKTx+XSwigkLpnNytNZZiYiISJPw6WAFznTgd+t3UV3jzloyERER8R8+H6xyM+PYe6CKFVv3uF2KiIiI+DjfD1YZ8QAUaDpQREREGpnPB6vWceG0jA4lX41CRUREpJH5drAq/h5jDLmZcbozUEREpBmJioo64nvr1q0jKyurCatpON8NVhvmwlO58O5tDEgLYtOu/WwtLXe7KhEREfFhvtuOPCUbBv8SZv6LkWEfMyvgKvILe3NeD23ILCIifu7D/4Otizz7mSnZcO7fjvj2b37zGzIyMrj11lsB+NOf/oQxhunTp7Nz504qKyt58MEHGTFixHF9bXl5Obfccgv5+fkEBQXx6KOPctppp7FkyRJGjRpFRUUFNTU1vP3226SlpXHZZZexceNGqqur+cMf/sDll19+Qr/24Xw3WAWHwRn3Q7cLCJp8O8/t/xfLPv8O2j4P0SluVyciIuJXRo4cyV133fVDsJo0aRIfffQRd999NzExMRQXF9O/f3+GDx+OMabBn/v0008DsGjRIpYvX87ZZ5/NypUree6557jzzju56qqrqKiooLq6mqlTp5KWlsYHH3wAQGlpqcd/z2MGK2NMGDAdCK29/i1r7R8Pu+Y64GFgU+2pp6y1L3q21J8ptQdm9Be8+vivubT0v/BUXzjnQeh9DRzHPzgRERGfcZSRpcbSu3dvtm/fzubNmykqKiIuLo7U1FTuvvtupk+fTkBAAJs2bWLbtm2kpDR8AGTmzJnccccdAHTp0oWMjAxWrlzJgAEDeOihh9i4cSMXXXQRHTt2JDs7m3vuuYff/OY3nHfeeQwePNjjv2dD1lgdAE631vYEegFDjDH967luorW2V+3hHaHqoMAgtna/iaEVf6M6OQum3AHjh8OONW5XJiIi4jcuueQS3nrrLSZOnMjIkSOZMGECRUVFFBQUMH/+fJKTkykvP7710NbW3wD8yiuvZMqUKYSHh3POOefwxRdf0KlTJwoKCsjOzua3v/0tDzzwgCd+rR85ZrCyjr21L4Nrj2bXxjwnM47VNSnMGfwKnPcYbJ4PzwyEr5+Emmq3yxMREfF5I0eO5I033uCtt97ikksuobS0lJYtWxIcHMyXX35JYWHhcX9mXl4eEyZMAGDlypWsX7+ezp07s2bNGtq1a8fYsWMZPnw4CxcuZPPmzURERHD11Vdzzz33MG/ePE//ig27K9AYE2iMmQ9sBz611s6p57KLjTELjTFvGWPaeLRKDzgpvXZD5vWlkHs93Dob2p0Kn9wHL54J25a4XaKIiIhP6969O3v27KFVq1akpqZy1VVXkZ+fT25uLhMmTKBLly7H/Zm33nor1dXVZGdnc/nll/PKK68QGhrKxIkTycrKolevXixfvpxrr72WRYsW0bdvX3r16sVDDz3Efffd5/Hf0RxpCK3ei41pAfwPuMNau7jO+QRgr7X2gDHmZuAya+3p9fz8GGAMQHp6es7PSaYn4pzHppMcG8b46/s6J6yFJe/A1F9D+S44+ZeQdw8EhTZpXSIiIo1t2bJldO3a1e0ymoX6/qyMMQXW2txj/exx9bGy1u4CpgFDDjtfYq09UPvyBSDnCD//vLU211qbm5SUdDxf7RE5mXF8V7jz0IbMxkDWxXD7XMi6BKb/A54bDOvrG5ATERERObpjBitjTFLtSBXGmHDgTGD5Ydek1nk5HFjmySI9JTcjjj0Hqli57bANmSPi4aJ/w1VvQ2UZvHSOM4p1YG/9HyQiIiKNbtGiRfTq1etHR79+/dwu66ga0scqFRhnjAnECWKTrLXvG2MeAPKttVOAscaY4UAVsAO4rrEKPhEHN2TOL9xJ19SYn17Q8Uy49Rv4/AH49nlY8SGc/xh0OLOJKxUREZHs7Gzmz5/vdhnH5ZjBylq7EOhdz/n76zz/LfBbz5bmeW3iw0mKDqVg3Q6u6Z9R/0Wh0TD0YWeKcMod8OrF0P4MaDsY2vSDtN4QHN60hYuIiHiAtfa4mm/6o+NZe14f3+28Xg9jDLkZceQXNmBD5vT+cNMMmPkYLH4LPvvcOR8QDKk9oE1/aNPXCVsxqUf/LBEREZeFhYVRUlJCQkKCwtURWGspKSkhLCzsZ3+GXwUrgJyMOD5cvJVtu8tJjjnGH1xwGJz2W+fYVwwb58KGObDhW8j/D8x22ugTm34oZLXpC8lZEOh3f7QiIuLFWrduzcaNGykqKnK7FK8WFhZG69atf/bP+91//XMza9dZrdvJsB7HMdIUmQidz3UOgKoKZwPLDXOco3CWM7IFEBwBrXJqg1Y/aJ3rLJAXERFxSXBwMG3btnW7DJ/nd8Gqe1oMYcEB5BfuOL5gdbigEGid4xwDbnV6YpVuPDSitWGOM41oa7u6J3WFwb+C7Eu0R6GIiIiP8rtgFRwYQM/WLShoyDqr42EMtGjjHNmXOOcq9sGmeU7IWjoZ3rkRvhsPQx+BpE6e/X4RERFx3XE1CPUVuZlxLNm8m7KKqsb9opBI527CvHtgzDQY9ihsWQDPDoTP/gwVZY37/SIiItKk/DNYZcRTXWOZv2FX031pQCD0uQFuL4DsS2Hmo/B0P1g+telqEBERkUbll8HqpPQ4AArWeXg6sCGikuDCZ+G6qc6I1htXwGsjYWfT7psoIiIinueXwSo2IphOyVEN62fVWDIHwc0z4Ky/wNrpzujV9H9C1YFj/6yIiIh4Jb8MVgA5GfHMW7+TmpoT67B6QgKDYdBYuP1b6HgWfPEXeHYQrJnmXk0iIiLys/ltsMrNiGNPeRUrt+859sWNLbY1XP5fuOotqKmC8SPgrRtgz1a3KxMREZHj4L/BKtNZZ5XvxjqrI+l4Ftw6G075P1j2HjyZC7Ofg+pGvntRREREPMJvg1V6fASJUaGe72d1og5uo3PrN872OB/9Bl44FTbMdbsyEREROQa/DVaHNmTe4XYp9UtoD1e/DZeOg30l8J8zYcodUOal9YqIiIj/BitwpgM37NjP9t3lbpdSP2Og+wXO4vYBt8N3E+CpPk4XdxEREfE6fh2scjJq11l523Tg4UKj4ZyH4KbpENsKJl0Lb17njGSJiIiI1/DrYNU9LZbQoADvWsB+NClZcOPncPp9sOx9eLqvRq9ERES8iF8Hq5CgAHq2aUGBt66zqk9gMOTdCzd9VWf0apRGr0RERLyAXwcrcPpZLdm8m/0V1W6XcnySuzujV6fd57Rm0OiViIiI6xSsMuOoauoNmT0lMBhOuRfGTNPolYiIiBfw+2D1w4bMzWk68HAH114dHL16ph8sneJ2VSIiIn7H74NVi4gQOrZ0eUNmT6g7ehWTBpOugbeu1+iViIhIE/L7YAXOdOC8Qpc3ZPaUuqNXS6do9EpERKQJKVjhTAfuLq9i1fa9bpfiGXVHr6JTNXolIiLSRBSsgNzMeADv3d7m50rJgtFfwGm/PzR6tew9t6sSERHxWQpWQGZCBAmRId63IbMnBAbDKb8+NHo18Wpn9GrXBrcrExER8TkKVjgbMudkxPlmsDro8NGrJ3rBu7dB8fduVyYiIuIzFKxq5WbGUVhSRtGeA26X0ngOjl6N/Q5yr4fFb8HTfZzeV1sXu12diIhIs6dgVSsnw1ln1az7WTVUizYw9GG4axEMHAurPoHnBsFrl8OGuW5XJyIi0mwpWNXKahVDSHPakNkTolrCWX+GuxfDqb+DDXPgP2fCuPNhzVdgfaD9hIiISBNSsKoVGhRIz9axzb9R6M8RHgen/gbuWgxnPwhFK2D8cPjPWbDiQwUsERGRBlKwqiMnI54lm0spr2xmGzJ7SmgUDLwD7lwIwx6BPdvg9ZHw3Mmw+G2o8dM/FxERkQZSsKojNyOOymrLgua4IbMnBYdBnxth7Dy44DmornBaNDzdF757Faor3a5QRETEKx0zWBljwowx3xpjFhhjlhhj/lzPNaHGmInGmO+NMXOMMZmNUWxjy8lwNmT2y+nA+gQGQ68r4NbZcOk4CI6AybfBE71hzvNQud/tCkVERLxKQ0asDgCnW2t7Ar2AIcaY/oddcwOw01rbAXgM+Ltny2wacZEhtE+K9O1+Vj9HQCB0vwBumg5XvQUxreDDe+HpfrDhW7erExER8RrHDFbWcXATveDa4/DVzCOAcbXP3wLOMMYYj1XZhHIz4inwlQ2ZPc0Y6HgWXP8RXDsZsPDSEJj2d6iucrs6ERER1zVojZUxJtAYMx/YDnxqrZ1z2CWtgA0A1toqoBRI8GShTSUnM47S/ZWsLvKRDZkbgzHQ7lS4eSZkXwLT/h+8Mgx2FrpdmYiIiKsaFKystdXW2l5Aa6CvMSbrsEvqG536yZCPMWaMMSbfGJNfVFR0/NU2gVyts2q4sFi46Hm46EXYvtS5e3DhJLerEhERcc1x3RVord0FTAOGHPbWRqANgDEmCIgFftLC3Fr7vLU211qbm5SU9LMKbmxtEyNJiAzxr0ahJ6rHpc7oVctu8M5oeHs0lJe6XZWIiEiTa8hdgUnGmBa1z8OBM4Hlh102BfhF7fNLgC+sbZ5dJY0xnJQR5x9b23hSXAZc94GzyfPit53Rq/Wz3a5KRESkSTVkxCoV+NIYsxCYi7PG6n1jzAPGmOG11/wHSDDGfA/8Evi/xim3aeRmxLHO1zdkbgyBQc4mz9d/DCYAXj4Xvvx/WtguIiJ+I+hYF1hrFwK96zl/f53n5cClni3NPbmZzjqrgsKdDMlKcbmaZqhNH7hpBnz4G/jq77D6C7joBYhv63ZlIiIijUqd1+uR1SqWkKAATQeeiLAYuPBZuOQlKFoJzw2GBW9o30EREfFpClb1CA0KpEcrP92Q2dOyLoZbZkFKNvzvJnj7Btjv51sGiYiIz1KwOoKczDgWb/LjDZk9qUUbuO59OP0PsORdZ2F74dduVyUiIuJxClZHkJsRT2W1ZeFGtQ3wiIBAyLsHbvgUAoKchqJfPKgNnUVExKcoWB3BoQ2Ztc7Ko1rnOD2vel0J0x+Gl85xRrEqytyuTERE5IQd865AfxUfGUK7pEgK1CjU80KjYMTT0OFM587BN38BwRHQ6RzofiF0OAtCItyuUkRE5LgpWB1FbkYcnyzdRk2NJSCgWe4p7d26Xwhdh0PhLFjyP1g6xXkMjoTOQ2pD1pkQHO52pSIiIg2iYHUUuRnxTMrfyJrivXRoGe12Ob4pIBDa5jnHuQ8fClnLpjgd3EOioPO5TshqfwYEh7ldsYiIyBEpWB1FTm2j0Px1OxWsmkJgELQ7xTmG/hPWzagNWe/BojchJLpOyDpdIUtERLyOgtVRtEuMJD4yhLnrdjKyb7rb5fiXwCBof5pzDHsE1k6Hpe/WhqxJEBoDnYfWhqzTICjU7YpFREQUrI7GGMPJHRL533cbaRMfzu2ndSAoUDdSNrnAYOhwhnMMexTWflU7kvU+LHwDQmOh99Vw2u+chfEiIiIuMdalLUZyc3Ntfn6+K999PPYeqOL+dxfzzneb6JsZz79G9iKthRZTe4WqCidkLZzkTBW2SIcRTznrtURERDzIGFNgrc091nUafjmGqNAgHr28F49e1pMlm0s59/EZfLR4q9tlCUBQCHQ8Cy5+AUZNdRbCjzsf3v8lHNjjdnUiIuKHFKwa6KKTWvP+2MGkx0dw86sF3PfuIm13400yBsLNs6D/bZD/EjwzENZMc7sqERHxMwpWx6FtYiRv3zKQMXnteHX2eoY/NZMVWzUy4jVCImDI/4PrP3ZGs8aPgPfuhPLdblcmIiJ+QsHqOIUEBfC7oV0Zd31fduyrYPhTM3l1diFurVWTeqT3c7bNGTgW5o2HZwbA95+7XZWIiPgBBauf6ZROSUy9czB928Zz37uLueXVeewqq3C7LDkoOBzO/gtc/4kzkvXqRTD5dijXptoiItJ4FKxOQMvoMMaN6svvhnbhs2XbGPr4DL5dq02bvUqbPnDTDBh0F8yf4IxerfrU7apERMRHKVidoIAAw5i89rx9y0CCgwIY+fw3PP7ZKqprNDXoNYLD4Kw/ww2fQWg0TLgE3r0N9u9yuzIREfExClYe0rNNCz4YO5gRvVrx2GcrueKF2Wzetd/tsqSu1jlw03QY/CtY8Do80x9Wfux2VSIi4kMUrDwoKjSIxw72vNqknldeKSgUzrgfRn8O4XHw2mXwzk1QpilcERE5cQpWjUA9r5qBtN4wZhrk/drp2v5Mf2eLHN3dKSIiJ0Bb2jSiiqoa/vnJCp6fvobOydE8cllPslrFul2WHG7LAmfN1bZF0CIDup4P3UZAq1wI0N89RESk4VvaKFg1ga9WFnHPmwvYsa+Cm/LaMfaMjoQFB7pdltRVVQGLJsHSybD6S6iphOg06HoedB3udHYP0D8zERF/pWDlZUrLKnnwg6W8WbCRdomR/O3iHvRtG+92WVKf8lJnUfvSyfD9Z1BVDhGJ0GUYdBsObU+BwGC3qxQRkSakYOWlZqwq4rfvLGLjzv1c0z+DXw/pTHSY/iPttQ7she8/haVTYNUnULEXwmKh81BnJKv96U47BxER8WkKVl6srKKKf368kpe/XktqTBgPXZjNaV1aul2WHEtlOaz+ApZNgRVTnZGtkCjodI4TsjqeBSGRblcpIiKNQMGqGZi3fie/eWshq7bv5YJeadx/fnfiI0PcLksaoqoC1k13RrKWfwBlxRAUDh3OgOxLnWlDTReKiPgMBatm4kBVNc98uZpnpn1PdFgwfzy/G8N7pmGMcbs0aaiaaij82hnJWvYe7NkC0amQMwpyroPoZLcrFBGRE6Rg1cys2LqHX7+9kAUbdnFGl5Y8eGEWqbHhbpclx6um2tmLcO4LzsL3gGBnwXvfMdCmHygwi4g0SwpWzVB1jeXlWWv55ycrCAoI4P/O7cKVfdMJCNB/jJulktUw90X4bgIcKIXkbOg72pkqDIlwuzoRETkOClbN2PqSMv7vnYV8vbqEvm3j+fvFPWibqEXRzVbFPlg4Cb59AbYvce4q7H0N9LkB4tu5XZ2IiDSAglUzZ63lzfyN/OWDpVRU1XD3WZ248eS2BAWqE3izZS2s/wa+fd5Zi1VT7dxJ2Gc0dDhTXd5FRLyYx4KVMaYNMB5IAWqA5621jx92zanAZGBt7al3rLUPHO1zFawaZvvucv4weTEfL9lGVqsYHr6kJ11TY9wuS07U7i1Q8AoUvAx7t0FcW2cEq9dVEKHGsSIi3saTwSoVSLXWzjPGRAMFwAXW2qV1rjkVuMdae15DC1SwOj4fLtrCHyYvYd+BKp66sjdndNWdZj6hqgKWv+dME67/xmnZ0ONSZxQrtYfb1YmISK2GBqtjzj1Ya7dYa+fVPt8DLANanXiJcjzOzU7lwzsH0zE5itHj83ltznq3SxJPCAqBrIvh+o/g5pnQ4zJY+Cb8ezD8+xSY8zyU7XC7ShERaaDjWtRhjMkEegNz6nl7gDFmgTHmQ2NMdw/UJodJig7l9dH9OaVTEr/73yIe+WQFbq2Rk0aQkg3Dn4BfLYMhfwdbDR/eC490hknXwspPoLrK7SpFROQoGrx43RgTBXwFPGStfeew92KAGmvtXmPMUOBxa23Hej5jDDAGID09PaewsPBE6/dLVdU1/GHyYl7/dgMXndSKv13Ug5AgLXz2SVsWwvzXYNEkKCuBqBToebmzFiups9vViYj4DY/eFWiMCQbeBz621j7agOvXAbnW2uIjXaM1VifGWsuTX3zPo5+uZHDHRJ656iRt5uzLqipg1cdOT6xVnzijWa1yofdV0P0iCG/hdoUiIj7Nk4vXDTAO2GGtvesI16QA26y11hjTF3gLyLBH+XAFK894M38Dv31nER2To3llVB+SY8LcLkka297tsHCiE7KKlkFQGHQ5D3pdCe1OhYBAtysUEfE5ngxWJwMzgEU47RYAfgekA1hrnzPG3A7cAlQB+4FfWmu/PtrnKlh5zlcri7j11QJiw4MZd31fOiZHu12SNAVrYfN3tVOFb0L5LohpBT1HOlOFCe3drlBExGeoQaifWbyplFGvzOVAZTUvXJtLv3YJbpckTamyHFZ+6Ixirf4cbA206Q/Zl0DHsyEuw+0KRUSaNQUrP7RhRxnXvfwtG3bs59HLe3JejzS3SxI37N4CC99wRrKKVzrnEjs7Xd47ngXpA502DyIi0mAKVn5qV1kFo8fnM3fdTu4b1pUbTm6Ls0xO/I61UPI9rPrUWfBeOAuqKyAkylmL1fEs6HAWxKotnYjIsShY+bHyymp+OWk+UxdtZdSgTO4b1o3AAIUrv3dgL6yb4YSsVZ9C6QbnfMvuh0az2vSDQN1dKiJyOAUrP1dTY3nwg2W8NGst52al8NjlvQgL1t1iUstaKFp+aDRr/TdQUwWhMdD+NGddVoczITrF7UpFRLyCgpUA8OKMNTw0dRk56XG8cG0ucZFaWyP1KN8Na786NJq1Z4tzPqWHM5LV/nRo3Vdrs0TEbylYyQ8+WLiFuyfOp3V8OONG9aVNfITbJYk3sxa2La4dzfoUNsxxGpIGR0LmIGh3mjOqldQFtH5PRPyEgpX8yJw1JYwen09IUCCvjOpDVqtYt0uS5qK8FNbOgDVfwuovYcdq53x0qrMI/uChaUMR8WEKVvITq7bt4bqX57KzrIIXrs1lUIdEt0uS5mjXeidgrZnmHPt3OOdbdjs0mpUxEEIi3axSRMSjFKykXtt2l3Ptf75lbck+nrnyJM7slux2SdKc1dTA1oWHRrPWz4bqAxAQ7Nxh2P5UaHc6pPXSVjsi0qwpWMkR7dxXwS9e/palm3fz2OW9OL+nGomKh1SUOXcYrvkSVk+DbYuc82EtnEXwWZc4C+G1CF5EmhkFKzmqPeWVXP/KXAoKd/K3i3twWW4bt0sSX7R3O6ydDqu/gOUfOPsZhsdB1+HOdjsZgzSSJSLNgoKVHFNZRRU3/beAGauK+fPw7vxiYKbbJYkvq6pwAtbit2D5VKjc5yyA734RZF8MaSfpLkMR8VoKVtIgB6qquf217/h06TZ+PaQzt57awe2SxB9U7IOVH8Git5yWDjWVEN8Osi52pgtbdnG7QhGRH1GwkgarrK7hnjcXMHn+Zm4/rQO/OruT9heUprN/Jyx7zwlZ62aArYHkLGeqMOtiaJHudoUiIgpWcnyqayy//98i3pi7gVGDMrn/vG4KV9L09myDJf9zpgs3znXOtennjGJ1vwCiWrpbn4j4LQUrOW7WWh54fykvz1rHyD5teOjCbG3eLO7ZuQ4Wv+2MZG1fCiYAMgc7PbLSejuHgpaINBEFK/lZrLU8+ulKnvzie4b3TOORy3oSHBjgdlni77YtPbTovWg5UPvvrZjWTo+sg0ErrTdExLtaqoj4poYGq6CmKEaaD2MMvzq7MxEhQfz9o+Xsr6zmySt6ExasW+LFRcndIPl+OON+OLDXaUq6aR5s/s45lr9/6Nq4zDpB6yRI7QlhMa6VLiL+RSNWckTjvl7HH6csYXDHRP59TQ4RIcrh4qX274QtCw4FrU3fQen6Q+8ndHSCVquTnMekzk7TUq0jFJEG0lSgeMSk/A3839sLycmI4z/X9SEmLNjtkkQaZl8xbJ5fG7ZqR7f2bDn0fkgUxLZ2jphWENvm0OvYVs65oFD36hcRr6JgJR7z/sLN3PXGfLqlxTBuVF/iIrUdiTRTu7c4AWvHaihdyKpOAAAgAElEQVTdBKUboHSjc5QV//T6qOQjBK/WEJPmdJFX+BLxC1pjJR5zXo80woMDuWXCPEY+P5v/3tiXltFhbpclcvxiUp2jPpX7YffmOmGrTvDavsxpZFq1/6c/FxQOYbHHcbT48WvtmyjiUzRiJQ026/tiRo/PJzkmjFdv7EerFuFulyTSdKx11nIdDFu7Nzt7H5aXHv2oqTr65yZ0dHp0db8QWnbTui8RL6WpQGkUBYU7uO6lucSEBzPhxn5kJka6XZKI97IWKsuOHLr273S6za+b6XScT+joBKzuFyhkiXgZBStpNIs3lXLNf+YQYAzPXZNDn0z1DRI5IXuLYNkUp+t84SwnZCV2ckJWtwugZVeFLBGXKVhJo1pdtJfR4/LZsLOMBy/I4vI+2s9NxCP2bq8NWe/WCVmd60wXdnW7QhG/pGAlja60rJLbX5/HjFXFXD+oLb8b2oUgdWkX8Zy6IWvdTMDWhqyD04UKWSJNRcFKmkRVdQ0PTV3Gy7PWkdcpiSev6E1suHpdiXjcnm1OyFo6+VDISuriTBV2PhfCW0BAMASGQGBQnefBmkYU8QAFK2lSb3y7nj9MXkyb+AhevDaXdklRbpck4rsOhqyD04Uc49/jJvBQyAoIch4DQ376vEU6pA+A9P6Q0sMJaCICKFiJC+asKeGWCfOoqq7hqStPIq9Tktslifi+PVudcFVZDjWVUF17HO/z6gooXgG7arcCCo6E1rmHglbrXAiNdvd3FXGRgpW4YsOOMkaPz2fltj3cN6wbowZlYjQNIdJ8lG6CDbNh/WxY/w1sW+IsoDcBkJJ9KGi16X/kZqsiPkjBSlyz90AVd0+cz6dLtzGyTxseGJFFSJAWtYs0S+W7YePcQ0FrY/6hDvRxmU7QatPPeUzsBAH6/7r4JgUrcVVNjeWRT1fw9Jer6ZsZz7NXn0RClPZUE2n2qithy0InZB0c2dpX5LwXHgepvZw1WwfXfVnrPD/4eNRzdR5DIiEqCSIPHi0hMtF5HtUSwuO1BkyalMeClTGmDTAeSAFqgOettY8fdo0BHgeGAmXAddbaeUf7XAUr/zB5/iZ+/dZCEqNCefEXuXRNjXG7JBHxJGthxxonaNWdOsTUuRvx4POGngMO7IF9xU5oq6ms54sNRMQfClxRLWsDWGLtudpAFlUbykIiGvEPQfyBJ4NVKpBqrZ1njIkGCoALrLVL61wzFLgDJ1j1Ax631vY72ucqWPmPBRt2MXp8PnsPVPGvy3txdvcUt0sSkebCWmdPxr1FTsg6/Ni7vTaA1T4e2F3/54REHRrtqvv4w/OWh86FRqtFhfxEo00FGmMmA09Zaz+tc+7fwDRr7eu1r1cAp1prtxzpcxSs/Mu23eWMGZ/Pgo2l3HtOZ249tb0WtYuI51WW1xO8ttcJZgefb4eyHdTbqiIorDZo1Y52xbd12k+k9nTWkWkK0i81NFgd1/86jDGZQG9gzmFvtQI21Hm9sfbcEYOV+JfkmDAm3jSAX7+1kIc/XsGKrXv4xyU9CAsOdLs0EfElwWHQoo1zHEt1FZQV14av+oLYdijdAGumHVqwHxTmbJCd2uNQ2GrZzbemGqurDm0SXr4L9u/68fODjxHxkNzd+f2TOkOQ1tHCcQQrY0wU8DZwl7X28LHW+oYefvLXAGPMGGAMQHq69pbzN2HBgTw+shedU6J5+OMVrCvZx/PX5JISG+Z2aSLijwKDIDrFOY6mugpKVjmL9rcuhC0LnA2zC15x3jcBzkhWSo9DgSsl2wkex+snoWbnoWCzf2ed9WbHWrfGka+rqXa+o+7nl++C/bXfW7Hn6DUGR0BYLJSVOP3PwGlCm9jRCVnJ3aBld+exRYbfTas2aCrQGBMMvA98bK19tJ73NRUox+WTJVu5a+J8okKDePbqHHIy4twuSUSk4ax1mqluXVgncC2EPZsPXRObfihoJbSHin11wszOw0aCal8faY3YQSbgx3dT/lyBoc5dnOEtIKzFYc9rXx98fvj7QSHOZ1RXQslq2L7EuWlh21Ln+cEmswAh0c6elsndIDnrUPAKP4F/59fUQFW5c1Tudx5DIo8dkE+QJxevG2AcsMNae9cRrhkG3M6hxetPWGv7Hu1zFaxk+dbdjBlfwOZd+7lvWFd+MVDNREWkmdtX7Ixo1Q1cJav5URAKCKoNKnF1QkvcoUBzpPNhsc4WRHXZn9HWwgQ4U6aNpXw3FC2vDVtLYPtS57F816FrotOcgBXfzgloVQec6dbK8jqPh4Wng+8dHCWrq8+NMOyRxvud8GywOhmYASzCabcA8DsgHcBa+1xt+HoKGILTbmGUtfaoqUnBSgBKyyr55aT5fL58O8N7pvHXi7KJDNXCUBHxIQf2wK4Nzt2G4XHO6Iq//SXSWtiz5dCo1sERrl3rnRGwoHAn7AXVHsFhdc6FO+u3gsNr36v7GOq8n9gRWp3UqL+CGoRKs1FTY3n2q9U88skK2idF8ezVOXRoqU2cRUTEezQ0WGnvAXFdQIDhttM6MP76fpTsq2DEUzOZukg3lIqISPOjYCVe4+SOibx/x8l0TI7m1gnzeOiDpVRW1xz7B0VERLyEgpV4lbQW4Uy6aQC/GJDBCzPWctULc9i+u9ztskRERBpEwUq8TkhQAH8ekcW/Lu/Fok2lDHtyJnPWlLhdloiIyDEpWInXuqB3K969bRDRoUFc+eIcXpi+BrduthAREWkIBSvxap1Topl8+yDO6prMQ1OXceuEeewpr2+nexEREfcpWInXiw4L5tmrT+L3Q7vyydJtjHhqFiu3HWPLBRERERcoWEmzYIxhdF47JtzYj93lVYx4ahaT529yuywREZEfUbCSZqV/uwSmjj2ZrFYx3PnGfP44eTEVVWrJICIi3kHBSpqdljFhvDa6Pzee3JZx3xRy+fPfsKV0v9tliYiIKFhJ8xQcGMB953Xj6StPYuXWPZz3xEy+/r7Y7bJERMTPKVhJszasRyqTbz+ZuMgQrv7PHJ77arVaMoiIiGsUrKTZ69Ayism3DeLc7FT+9uFybn61gN1qySAiIi5QsBKfEBkaxFNX9Oa+YV35bNl2Rjw1ixVb1ZJBRESaloKV+AxjDDcObsdrN/Zj74EqLnhaLRlERKRpKViJz+nXLoEP7jjUkuHP7y2hslotGUREpPEpWIlPOtiS4fpBbXl51jqueH4223aXu12WiIj4OAUr8VnBgQHcf343nriiN0s272bYEzOZs6bE7bJERMSHKViJzxveM43Jtw8iJiyIK1+cw4sz1qglg4iINAoFK/ELnZKjmXz7IM7s2pIHP1jG7a99x94DVW6XJSIiPkbBSvxGdFgwz12dw2/P7cKHi7dwwdOz+H77XrfLEhERH6JgJX7FGMNNp7Tn1Rv6sXNfBSOemsnURVvcLktERHyEgpX4pYEdEnl/7Ml0Sonm1gnzeOiDpVSpJYOIiJwgBSvxW6mx4UwcM4BrB2Twwoy1XPXiHLarJYOIiJwABSvxayFBATwwIovHLu/Jgo27OPfxGXy5YrvbZYmISDOlYCUCXNi7Ne/fcTJJ0aGMenkuD32wlIoqTQ2KiMjxUbASqdWhZTTv3jaIa/o7U4OXPPc164r3uV2WiIg0IwpWInWEBQfylwuyeO7qHNYV7+O8J2dqI2cREWkwBSuRegzJSuHDu/LokhLNnW/M5943F1BWoYaiIiJydApWIkfQqkU4b4zpzx2nd+CteRs578mZLNlc6nZZIiLixRSsRI4iKDCAX53dmQk39mPfgSoufOZrxn29TnsNiohIvRSsRBpgYPtEpo4dzKD2CfxxyhLG/LeAXWUVbpclIiJeRsFKpIESokJ56bo+3DesK9NWbOfcx2fw7dodbpclIiJe5JjByhjzkjFmuzFm8RHeP9UYU2qMmV973O/5MkW8gzGGGwe3451bBhEaFMDI57/h8c9WUV2jqUEREWnYiNUrwJBjXDPDWtur9njgxMsS8W7ZrWN5f+xgRvRqxWOfreTKF2azpXS/22WJiIjLjhmsrLXTAc13iBwmKjSIxy7vxSOX9mTRplKGPj6Dz5Zuc7ssERFxkafWWA0wxiwwxnxojOnuoc8UaRYuznG2w0mNDefG8fn8cfJi9ldUu12WiIi4wBPBah6QYa3tCTwJvHukC40xY4wx+caY/KKiIg98tYh3aJcUxf9uG8ioQZmM+6aQoU/MoKBwp9tliYhIEzvhYGWt3W2t3Vv7fCoQbIxJPMK1z1trc621uUlJSSf61SJeJTQokD+e353XRvejoqqGS5/7mr9/tJwDVRq9EhHxFyccrIwxKcYYU/u8b+1nlpzo54o0VwPbJ/LRXYO5LLcNz05bzfAnZ7F4kzq2i4j4g4a0W3gd+AbobIzZaIy5wRhzszHm5tpLLgEWG2MWAE8AI63aUoufiw4L5m8X9+Dl6/qws6yCC56exROfr6Kyusbt0kREpBEZtzJQbm6uzc/Pd+W7RZrSrrIK7p+8hCkLNtOjdSyPXNqTjsnRbpclIiLHwRhTYK3NPdZ16rwu0shaRITwxBW9efrKk9iwo4xhT87khelr1FRURMQHKViJNJFhPVL55O5TyOuYxENTl3HF87MpLNnndlkiIuJBClYiTSgpOpQXrs3hkUt7smzrbs59fAavzi5EyxJFRHyDgpVIEzPGcHFOaz6+K4+cjDjue3cx1770LZt3aUscEZHmTsFKxCVpLcIZf31fHrwgi4LCnZzzr+m8XbBRo1ciIs2YgpWIi4wxXN0/gw/vHEyXlGh+9eYCxvy3gKI9B9wuTUREfgYFKxEvkJEQyRtjBnDfsK58tbKIsx/7ivcWbNbolYhIM6NgJeIlAgMMNw5ux9SxJ5MeH8Edr3/HrRPmafRKRKQZUbAS8TIdWkbz9i0D+c2QLny+bDtnP/YVUzR6JSLSLChYiXihoMAAbjm1PR+MPZn0hEjGvv4dt7yq0SsREW+nYCXixTomR/P2zQP47bld+GLFds567Csmz9+k0SsRES+lYCXi5YICA7jplPZMHXsymQmR3PnGfG76bwHb95S7XZqIiBxGwUqkmTi49up3Q7swbWURZz82XaNXIiJeRsFKpBkJDDCMyWvP1LGDaZfojF6N+W8B23dr9EpExBsoWIk0Qx1aRvHmzQP5/dCuTF9ZxFmPTed/36lru4iI2xSsRJqpwADD6Lx2TL1zMB1aRnH3xAWMHp/PNo1eiYi4RsFKpJlrnxTFpJucru0zVhVz1qNfac9BERGXKFiJ+ICDXds/uiuPTsnOnoM3jstna6lGr0REmpKClYgPaZsYycSbBnD/ed2YtbqYMx6Zxr+/Wk1FVY3bpYmI+AUFKxEfExhguP7ktnx8Vx4D2ifw1w+XM+Rf0/lyxXa3SxMR8XkKViI+KiMhkhd/0YeXR/UBYNTLc7nhlbmsK97ncmUiIr5LwUrEx53WuSUf3ZXH74Z2YfaaEs5+bDp//2g5+w5UuV2aiIjPUbAS8QMhQQGMyWvPl/ecyvk903h22mpOf2Qa736nzu0iIp6kYCXiR1rGhPHIZT1559aBJMeEcdfE+Vz63Dcs3lTqdmkiIj5BwUrED52UHse7tw7i7xdns7Z4H+c/NZPf/W8RO/ZVuF2aiEizpmAl4qcCAgyX90nni3tOZdTAtkycu4FTH/6ScV+vo6pa7RlERH4OBSsRPxcbHsz953fjozsHk906lj9OWcKwJ2by9epit0sTEWl2FKxEBICOydG8ekM/nrs6h30VVVz5whxumzCPTbv2u12aiEizEeR2ASLiPYwxDMlK4dTOSTw/fQ3PTPuez5dvY9Sgttx8Sntiw4PdLlFExKtpxEpEfiIsOJCxZ3Tk81+dyjndU3h22mry/vEl//5qNeWV1W6XJyLitRSsROSIWrUI5/GRvflg7Mn0atOCv364nNP+OY1JczdogbuISD0UrETkmLqnxTLu+r68Pro/LWPC+PXbCxny+Aw+XrJVDUZFROpQsBKRBhvQPoF3bx3Ic1efRI213PTfAi5+9mvmrClxuzQREa9wzGBljHnJGLPdGLP4CO8bY8wTxpjvjTELjTEneb5MEfEWzgL3VD65K4+/XZTN5l3lXP78bEa9/C3Ltux2uzwREVc1ZMTqFWDIUd4/F+hYe4wBnj3xskTE2wUFBjCybzrT7j2V/zu3CwWFOxn6xAzunjifDTvK3C5PRMQVxwxW1trpwI6jXDICGG8ds4EWxphUTxUoIt4tLDiQm09pz4xfn86YvHZMXbSF0x+Zxp+mLKFk7wG3yxMRaVKeWGPVCthQ5/XG2nMi4kdiI4L57bldmXbvqVyS05r/zi4k7x9f8q/PVrL3QJXb5YmINAlPBCtTz7l6bxMyxowxxuQbY/KLioo88NUi4m1SY8P560U9+PiuPPI6JfGvz1Zxyj++5OVZa9UDS0R8nieC1UagTZ3XrYHN9V1orX3eWptrrc1NSkrywFeLiLfq0DKKZ6/O4d3bBtEpOZo/v7eU0/45jQlzCqmoUg8sEfFNnghWU4Bra+8O7A+UWmu3eOBzRcQH9GrTgtdG92PCjf1IjQ3j9/9bzOmPTGNSvpqMiojvMcdq7meMeR04FUgEtgF/BIIBrLXPGWMM8BTOnYNlwChrbf6xvjg3N9fm5x/zMhHxIdZapq0s4tFPVrJoUyltEyO584yOnN8zjcCA+lYViIh4B2NMgbU295jXudU1WcFKxH9Za/l06TYe/XQly7fuoWPLKO4+qxNDuqcQoIAlIl6oocFKnddFpMkZYzi7ewpTxw7m6StPwgK3TpjHsCdn8unSbdomR0SaLQUrEXFNQIBhWI9UPr4rj8cu78n+iipGj8/ngqdnMW3FdgUsEWl2NBUoIl6jqrqGd+Zt4vHPV7Fp135yM+L45dmdGNg+0e3SRMTPaY2ViDRbFVU1TMrfwFNffM/W3eUMaJfAr87uRG5mvNuliYifUrASkWavvLKa1+as55lpqynee4C8TkncnNeOAe0TcG5IFhFpGgpWIuIzyiqq+O83hTw/fQ0l+yrokhLN9Se3ZXjPNMKCA90uT0T8gIKViPic8spqpszfzEuz1rJ86x4So0K4ql8GV/fPICk61O3yRMSHKViJiM+y1vL16hJemrmWz5dvJyQwgOG90rh+UFu6pcW4XZ6I+KCGBqugpihGRMSTjDEM6pDIoA6JrCnayytfr+PN/I28VbCRAe0SuOHktpzepaWajYpIk9OIlYj4hNKySl6fu55xX69jS2k5mQkRjBrUlktyWhMZqr9DisiJ0VSgiPilyuoaPlq8lZdmreW79buIDgviir7p/GJgJq1ahLtdnog0UwpWIuL35q3fyUsz1/Lh4q0ADOmewvUntyUnI87lykSkudEaKxHxeyelx3HSlXFs2rWf8d+s4/U56/lg0RayW8VyZb90zu+ZRpSmCUXEgzRiJSJ+Y9+BKt6Zt5FXZ69nxbY9RIYEMqJ3K67sm05Wq1i3yxMRL6apQBGRI7DWMm/9Ll7/dj3vL9xMeWUNPVrHckVfjWKJSP0UrEREGqB0fyXvfreJ1+ZoFEtEjkzBSkTkOGgUS0SORsFKRORn0iiWiBxOwUpE5ARpFEtEDlKwEhHxoMNHsSJCAjmrWzLn9Ugjr1MioUGBbpcoIo1IwUpEpBFYa/luwy7ezN/Ah4u3squskuiwIM7pnsJ5PVIZ1CGR4MAAt8sUEQ9TsBIRaWSV1TXM+r6Y9xZs4ZOlW9lTXkVcRDBDslI4r0ca/dslEKiNoEV8goKViEgTOlBVzfSVxby/cDOfLt1GWUU1iVGhDM12QlZuRhwBClkizZaClYiIS/ZXVPPliu28v3AzXyzfTnllDSkxYQzNTuX8nqn0atMCYxSyRJoTBSsRES+w70AVny3bxvsLt/DViiIqqmtoHRfOsB6pnN8jje5pMQpZIs2AgpWIiJcp3V/Jp0u38f7CzcxcVUxVjSUjIYJzs1IZmp1CdqtYhSwRL6VgJSLixXbuq+DjJVv5YNEWvl5dQnWNpVWLcIZmp3Budiq9WrfQmiwRL6JgJSLSTOzcV8Gny7bx0eKtzFhVRGW1JTU2jCFZKQzNTiUnXQvfRdymYCUi0gyV7q/ki+XbmLpoK1+tLKKiqoaW0aEMyUrh3KxU+raNVwsHERcoWImINHN7D1TxxfLtfLhoC1+ucO4uTIwK4ezuKQzNSqV/u3iC1IxUpEkoWImI+JCyiiqmrShi6qItfLF8O2UV1cRFBHN2txSGZKcwoF0CYcHaVkeksShYiYj4qPLKar5aWcSHi7bw2bLt7D1QRWhQAH3bxnNKpyTyOiXRsWWU7jAU8SAFKxERP1BeWc03a0qYvrKI6SuLWF20D4CUmDAGd0wkr1MSJ3dIJC4yxOVKRZo3jwYrY8wQ4HEgEHjRWvu3w96/DngY2FR76ilr7YtH+0wFKxERz9u0az8zVhYxfVURM1cVs7u8CmOgR+sWnFIbtHq1aaG1WSLHyWPByhgTCKwEzgI2AnOBK6y1S+tccx2Qa629vaEFKliJiDSuquoaFmwsdUazVhWxYMMuaixEhwUxqL0TsgZ3TKRNfITbpYp4vYYGq6AGfFZf4Htr7ZraD34DGAEsPepPiYiIq4ICA8jJiCMnI467z+pEaVkls1YX/zBt+NGSrQC0S4z8YcqwX7t4osOCXa5cpPlqSLBqBWyo83oj0K+e6y42xuThjG7dba3dUM81IiLiktiIYIZmpzI0OxVrLauL9vLVSidovTF3Pa98vY7AAEN2q1gGdUhgYPtEcjLidLehyHFoyFTgpcA51toba19fA/S11t5R55oEYK+19oAx5mbgMmvt6fV81hhgDEB6enpOYWGh534TERH52corq5m3fiffrC5h1vfFLNhYSnWNJSQogNyMOAa2T2Bgh0R6tIrV+izxS55cYzUA+JO19pza178FsNb+9QjXBwI7rLWxR/tcrbESEfFee8ormbtuB7O+L+Hr1SUs27IbgKjQIPq1jWdA+wQGdUikc3K0ttsRv+DJNVZzgY7GmLY4d/2NBK487MtSrbVbal8OB5YdZ70iIuJFosOCOb1LMqd3SQagZO8BZq/ZwazVxXyzuoTPl28HICEyhP7tExjUPpGB7RPISIhQ/yzxa8cMVtbaKmPM7cDHOO0WXrLWLjHGPADkW2unAGONMcOBKmAHcF0j1iwiIk0sISqUYT1SGdYjFXDaOnyzuoSvvy9m1upiPljo/N06LTaM3Mx4cjOdRfNdUmK0t6H4FTUIFRGRE2KtZU3xPr5eXcLs1SXkF+5g2+4DgDN12Du9BTkZceRmxNMrvQVRoQ2ZLBHxLuq8LiIirrDWsnHnfgoKd5JfuIP8dTtZsW0P1kKAga6pMeRmxJGTGU9uRhxpLcLdLlnkmBSsRETEa+wur+S79bsoWLeD/MKdzN+wi7KKasCZPjwYspzpw2jdeShex5OL10VERE5ITFgwp3RK4pROSYDTFX7Zlj3OiFbhTuau3cF7CzYDEBkSSPdWsXRLjaFrajRdU2PolBytflrSLGjESkREXGetZdMuZ/qwoHAnizeVsnzrnh9GtQIMtEuKomudsNUtNYaW0aG6C1GahEasRESk2TDG0DougtZxEYzo1QqAmhpL4Y4ylm3Z/cMxr3DnDyNbAPGRIU7QSompDV0xdGgZRUiQphLFHQpWIiLilQICDG0TI2mbGMnQ7NQfzpeWVbJs6+46gWsP42cXUlFVA0BwoKF9UhTdUmPolhZTO6UYQ1xkiFu/ivgRBSsREWlWYiOC6d8ugf7tEn44V1Vdw9rifSytDVrLtuxm5vfFvPPdph+uSY0N+0nYSo+PUOd48SgFKxERafaCAgPomBxNx+RoRvQ6dL547wGWbdnN0s3O6NbSLbuZtrKI6hpnfXFkSKCzXist5od1W51TtFBefj4FKxER8VmJUaEM7pjE4I5JP5wrr6xm5bY9PwSupVt28868Tew9UAg4C+Xb1y6U75YWQ6fkKNomRtE6LpxgtYGQY1CwEhERvxIWHEiP1i3o0brFD+dqaiwbdpb9KGwVFO5kSp2F8kEBhjbxET+s+8pMjKRd7fOUmDBNKQqgYCUiIkJAgCEjIZKMhEiGZB1aKL+rrILVRftYW7yPtcV7ax/L+Hp1MeWVNT9cFxYcQGZCZL2hKz4yRC0h/IiClYiIyBG0iAghJyOEnIy4H52vqbFs21NeG7T2sbY2fK3YtodPl26jquZQj8josCDaJUaSnhBJenw4GfGRpCdEkJEQQXK0Rrp8jYKViIjIcQoIMKTGhpMaG87A9ok/eq+quoaNO/eztuRQ4FpXso8FG3YxddGWHxbOA4QEBdAmLpyMhEjS452wlZEQQXq809NLi+ibHwUrERERDwoKDCCzdjrwtM4/fq+quobNu8op3LGPwpIy1u8oY31JGYU7ypizpoR9tZ3mAYyBlJgw0uMjfghdreMiSIkNIy02nOTYUEKDFLy8jYKViIhIEwkKDCA9IYL0hAgGd/zxe9ZaSvZVUFhSxoYdZRSWlFG4Yx/rS8r4amUR2/cc+MnnJUaFkBIbVjt6VvfRea7w1fQUrERERLyAMYbEqFASo0J/sqYLoKyiii2l5WzZVc6W0v3O81Ln+YYdZXy7dgel+yt/8nOJUSGkxobXBrBD4evg6+T/3979xMZxlnEc/z7x7jpeO43j1NDShn8RByQEBVUICQQVAgQcKEhQUQmpnOgBpCIuIC4UJCSEAHErAlGpSECoaIEe6aEIuIS2IaUpERCqUNL8MXHtBNtJ1k4eDjspdmKntnfWE4+/HynamfHs7JtHj9Y/z/uufcN2pxxLZLCSJGkTaLca7B0fYe/4yIrnzF5Y4OTZlcPX/ucnOXt+4arnjQ23uOmG7UsC101XBLB2y8iwGlZJkqSaGB5cffg6WYSuk4sC2PEz5znwwhRTc1ff+do51H9qmYsAAAWBSURBVFwStMZHBtld3GHbPdLixpEWu4cH2TnU3NKfdDRYSZK0hawmfJ2fv/j/4HX2XBHAznN8urt/6MUzTM52yLz6uY1twdhwqwhdLXa/vL00gHW3B2s3DWmwkiRJS2xvDrz8ycaVLFy8xNTcPJOzF5ic6XB6Zunj5OwFTs90ODo5y+RMh7lFn3hc+lrbGB1qMdpuMtpusqt9ebvF6FB3f2dxfFe7yc52k9GhFq3G9fnnhQxWkiRpzRoD2xjfMcj4jsFVnT/XWVg2eJ05N8/UbIfpc/NMz3U4MjHD1Fx3e/EvWr3ScGugG77aTT72ttdw7/v2lvVf64nBSpIk9V271aA91mDPWHtV52cms52LTM0W4Wuuw3QRuKbn5rvh61x3e/A6untlsJIkSdediGBksMHIYIM9VQ9mDa6fiCdJkrTJGawkSZJKYrCSJEkqicFKkiSpJAYrSZKkkhisJEmSSmKwkiRJKonBSpIkqSQGK0mSpJIYrCRJkkpisJIkSSqJwUqSJKkkBitJkqSSRGZW88IR/wH+tQEvdSNwegNeZyuytv1jbfvL+vaPte0v69s/r1Tb12Xm+CtdpLJgtVEi4qnMvL3qcdSRte0fa9tf1rd/rG1/Wd/+Kau2TgVKkiSVxGAlSZJUkq0QrH5Y9QBqzNr2j7XtL+vbP9a2v6xv/5RS29qvsZIkSdooW+GOlSRJ0oaobbCKiA9HxN8i4khEfKXq8dRNRByNiGcj4mBEPFX1eDaziHgwIiYi4tCiY2MR8XhE/KN43FXlGDezFep7f0S8WPTvwYj4aJVj3KwiYk9EPBERhyPiuYi4rzhu//boGrW1d0sQEdsj4k8R8UxR368Xx98QEfuL3v1FRLTWfO06TgVGxADwd+CDwDHgSeDuzPxrpQOrkYg4Ctyemf4+lR5FxHuBGeAnmfmW4ti3gZcy81vFDwa7MvPLVY5zs1qhvvcDM5n5nSrHttlFxM3AzZl5ICJ2AE8DHwc+i/3bk2vU9i7s3Z5FRADDmTkTEU3gj8B9wJeARzNzX0T8AHgmMx9Yy7XresfqncCRzHw+MzvAPuDOisckLSszfw+8dMXhO4GHiu2H6L6hah1WqK9KkJknMvNAsf1f4DBwC/Zvz65RW5Ugu2aK3WbxL4H3A78sjq+rd+sarG4B/r1o/xg2ZNkS+G1EPB0Rn6t6MDX06sw8Ad03WOBVFY+njr4QEX8ppgqdqupRRLweeDuwH/u3VFfUFuzdUkTEQEQcBCaAx4F/AtOZuVCcsq7sUNdgFcscq9+cZ7XenZnvAD4CfL6YbpE2iweAvcBtwAngu9UOZ3OLiBHgEeCLmXm26vHUyTK1tXdLkpkXM/M24Fa6M11vXu60tV63rsHqGLBn0f6twPGKxlJLmXm8eJwAfkW3KVWeU8Uai8trLSYqHk+tZOap4k31EvAj7N91K9anPAL8NDMfLQ7bvyVYrrb2bvkycxr4HfAuYDQiGsWX1pUd6hqsngTeVKzubwGfBh6reEy1ERHDxWJKImIY+BBw6NrP0ho9BtxTbN8D/KbCsdTO5W/6hU9g/65LsQD4x8DhzPzeoi/Zvz1aqbb2bjkiYjwiRovtIeADdNexPQF8sjhtXb1by08FAhQfQf0+MAA8mJnfrHhItRERb6R7lwqgAfzM+q5fRPwcuIPuX1Y/BXwN+DXwMPBa4AXgU5npAux1WKG+d9CdSkngKHDv5TVBWr2IeA/wB+BZ4FJx+Kt01wLZvz24Rm3vxt7tWUS8le7i9AG6N5kezsxvFN/f9gFjwJ+Bz2TmhTVdu67BSpIkaaPVdSpQkiRpwxmsJEmSSmKwkiRJKonBSpIkqSQGK0mSpJIYrCRJkkpisJIkSSqJwUqSJKkk/wMQhY1HEtD8IAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x218d729d0f0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = 'model.h5'\n",
    "checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "model.fit(trainX, trainY, epochs=30, batch_size=64, validation_data=(testX, testY), callbacks=[checkpoint, PlotLearning()], verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating the model ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['hi', 'hallo'],\n",
       "       ['hi', 'gru gott'],\n",
       "       ['run', 'lauf'],\n",
       "       ['hi', 'hallo'],\n",
       "       ['hi', 'hallo'],\n",
       "       ['hi', 'gru gott'],\n",
       "       ['hi', 'gru gott'],\n",
       "       ['hi', 'hallo'],\n",
       "       ['hi', 'hallo'],\n",
       "       ['go on', 'mach weiter']], dtype='<U291')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 99,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [426, 298,   0,   0,   0,   0,   0,   0],\n",
       "       [299,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 99,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 99,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [426, 298,   0,   0,   0,   0,   0,   0],\n",
       "       [426, 298,   0,   0,   0,   0,   0,   0],\n",
       "       [ 99,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 99,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [ 28,  86,   0,   0,   0,   0,   0,   0]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainX[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['hi', 'gru gott'], dtype='<U291')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing on trained examples\n",
      "src=[hallo], target=[hi], predicted=[hi]\n",
      "src=[gru gott], target=[hi], predicted=[hi]\n",
      "src=[lauf], target=[run], predicted=[run]\n",
      "src=[hallo], target=[hi], predicted=[hi]\n",
      "src=[hallo], target=[hi], predicted=[hi]\n",
      "src=[gru gott], target=[hi], predicted=[hi]\n",
      "src=[gru gott], target=[hi], predicted=[hi]\n",
      "src=[hallo], target=[hi], predicted=[hi]\n",
      "src=[hallo], target=[hi], predicted=[hi]\n",
      "src=[mach weiter], target=[go on], predicted=[go on]\n",
      "Bleu-1: 0.067523\n",
      "Bleu-2: 0.000000\n",
      "Bleu-3: 0.000000\n",
      "Bleu-4: 0.000000\n"
     ]
    }
   ],
   "source": [
    "print('Testing on trained examples')\n",
    "evaluate_model(model, eng_tokenizer, trainX, train, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### For cutoff length = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing on test examples\n",
      "src=[tom fiel], target=[tom fell], predicted=[tom fell]\n",
      "src=[wo ist tom], target=[wheres tom], predicted=[wheres tom]\n",
      "src=[ich habe einen lkw], target=[i have a truck], predicted=[i have a truck]\n",
      "src=[raus aus den federn], target=[get out of bed], predicted=[get get the]\n",
      "src=[vielleicht regnet es], target=[it may rain], predicted=[it may rain]\n",
      "src=[mir ist zum weinen zumute], target=[i want to cry], predicted=[hes very bored]\n",
      "src=[tom ist gemein], target=[toms mean], predicted=[tom is mean]\n",
      "src=[das wird ein spa], target=[thisll be fun], predicted=[nobody enjoy]\n",
      "src=[sie ist auf diat], target=[shes dieting], predicted=[theyre hard]\n",
      "src=[fangen wir an], target=[lets begin], predicted=[lets begin]\n",
      "Bleu-1: 0.070420\n",
      "Bleu-2: 0.000000\n",
      "Bleu-3: 0.000000\n",
      "Bleu-4: 0.000000\n"
     ]
    }
   ],
   "source": [
    "print('Testing on test examples')\n",
    "evaluate_model(model, eng_tokenizer, testX, test, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### For cutoff length = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing on test examples\n",
      "src=[ich habe einen lkw], target=[i have a truck], predicted=[i have a truck]\n",
      "src=[raus aus den federn], target=[get out of bed], predicted=[get get the]\n",
      "src=[mir ist zum weinen zumute], target=[i want to cry], predicted=[hes very bored]\n",
      "src=[darf ich dich kussen], target=[can i kiss you], predicted=[can i hear you]\n",
      "src=[machs wie du es fur richtig haltst], target=[do as you want], predicted=[do as you want]\n",
      "src=[sie ist gerade auer haus], target=[she is out now], predicted=[she is a]\n",
      "src=[hol mir mal meine werkzeugkiste], target=[get my tool box], predicted=[get me me]\n",
      "src=[ich bin lehrerin], target=[i am a teacher], predicted=[im healthy]\n",
      "src=[ich bin auf dem dach], target=[im on the roof], predicted=[im not by bus]\n",
      "src=[das ist ein zeichen], target=[this is a sign], predicted=[thats a fake]\n",
      "Bleu-1: 0.162406\n",
      "Bleu-2: 0.000000\n",
      "Bleu-3: 0.000000\n",
      "Bleu-4: 0.000000\n"
     ]
    }
   ],
   "source": [
    "print('Testing on test examples')\n",
    "evaluate_model(model, eng_tokenizer, testX, test, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### For cutoff length = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing on test examples\n",
      "src=[das mussen sie wissen], target=[it is up to you], predicted=[we want to]\n",
      "src=[er hat einen hut auf], target=[he has a hat on], predicted=[he looks in]\n",
      "src=[ich sage tue es], target=[i say go for it], predicted=[i can you it]\n",
      "src=[ich war auf einem fahrrad], target=[i was on a bike], predicted=[i was at home]\n",
      "src=[das hangt von dir ab], target=[it is up to you], predicted=[its on to us]\n",
      "Bleu-1: 0.111111\n",
      "Bleu-2: 0.000000\n",
      "Bleu-3: 0.000000\n",
      "Bleu-4: 0.000000\n"
     ]
    }
   ],
   "source": [
    "print('Testing on test examples')\n",
    "evaluate_model(model, eng_tokenizer, testX, test, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- It can be seen as we increase the size of the input phrase, the BLEU score decreases. Hence, we can look at attention mechanisms for improvement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}