im2latex_test.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxwaaMw5qeYa",
        "outputId": "c4a2da76-f2cc-43db-d2f3-7ecf31edf985"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.8.2\n",
            "2.8.0\n",
            "1.21.6\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "import numpy as np\n",
        "import os\n",
        "from datetime import datetime\n",
        "from im2latex import build_inference_model, build_training_model\n",
        "from data_loader import get_dataset\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "from tqdm import tqdm\n",
        "from IPython.display import display, Math, Latex, Image\n",
        "import cv2\n",
        "\n",
        "print(tf.__version__) # 2.8.0\n",
        "print(tf.keras.__version__) # 2.8.0\n",
        "print(np.__version__) # 1.21.6\n",
        "tf.executing_eagerly() # True\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  from google.colab.patches import cv2_imshow\n",
        "  IN_COLAB = True\n",
        "except:\n",
        "  IN_COLAB = False"
      ],
      "metadata": {
        "id": "ewEuogUJg-VB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load model"
      ],
      "metadata": {
        "id": "q5Bc_dKKPB8k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# saved_model = None\n",
        "saved_model = r'./checkpoints/cp-0020.ckpt'\n",
        "im2latex_encoder, im2latex_decoder = build_inference_model(saved_model)"
      ],
      "metadata": {
        "id": "U8PdbCzOOnYr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ffde3cf8-faed-49d0-a9a1-bc7ad2cf8932"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model loaded from /content/drive/MyDrive/im2markup/model_checkpoints_15/cp-0020.ckpt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Read test data from npy"
      ],
      "metadata": {
        "id": "whSTnE7-tCyx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_images = ['1a6e6aed62.jpg', '1a6a769c1f.jpg', '1a6fe45c04.jpg', '1a0b667b59.jpg', '1a7ff7a2f9.jpg']"
      ],
      "metadata": {
        "id": "60vqwxSj-qxd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "npy_path = r'./datasets/test_buckets.npy'\n",
        "image_path=r'./images'\n",
        "test_data = {}\n",
        "data_dict = np.load(npy_path, allow_pickle=True).tolist()\n",
        "for keys in data_dict.keys():\n",
        "    data_list = data_dict[keys]\n",
        "    for x, y in data_list:\n",
        "        x = x[:-4] + '.jpg'\n",
        "        img = cv2.imread(os.path.join(image_path, x), 0)\n",
        "        img = np.expand_dims(img.astype(np.float32), axis=0)\n",
        "        img = np.expand_dims(img, axis=-1)\n",
        "        seq = np.array(y).astype(np.float32)\n",
        "        test_data[x] = (img, seq[:-1])"
      ],
      "metadata": {
        "id": "ss7INDsFtNCJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "max_seq_len = 150\n",
        "preds = []\n",
        "truth = []\n",
        "imgs = []\n",
        "for bn, img_name in enumerate(test_images):\n",
        "  sample = test_data[img_name]\n",
        "  img = sample[0]\n",
        "  latex_actual = sample[1]\n",
        "  encoder_out = im2latex_encoder.predict(img)\n",
        "  \n",
        "  token = 500\n",
        "  latex_pred = []\n",
        "  decoder_state  = np.zeros([1, 1024])\n",
        "  while token != 501 and len(latex_pred) <= max_seq_len:\n",
        "    latex_pred.append(token)\n",
        "    pred, decoder_state = im2latex_decoder.predict([np.array([latex_pred[-1:]]), \n",
        "                                                    encoder_out,\n",
        "                                                    decoder_state])\n",
        "    token = np.argmax(pred[0, -1], axis=-1)\n",
        "\n",
        "  imgs.append(img)\n",
        "  preds.append(latex_pred)\n",
        "  truth.append(latex_actual.astype(np.int).tolist())"
      ],
      "metadata": {
        "id": "wrzm2fETtH4y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Note:\n",
        "\n",
        "*   501 => END OF SEQUENCE\n",
        "*   500 => START OF SEQUENCE\n",
        "*   499 => UNKNOWN\n",
        "*   0   => null (for padding purpose)\n",
        "\n"
      ],
      "metadata": {
        "id": "5r4PhEnj5LoU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_dict = np.load(r'./datasets/properties.npy', allow_pickle=True)\n",
        "vocab_dict = vocab_dict.tolist()\n",
        "vocab_map = lambda x: vocab_dict['idx_to_char'][x]\n",
        "render_latex = lambda x: display(Math(x))\n",
        "\n",
        "from IPython.display import Image\n",
        "\n",
        "for img_name, img, actual, pred in zip(test_images, imgs, truth, preds):\n",
        "  print('Input image and actual latex seq:')\n",
        "  if IN_COLAB:\n",
        "    cv2_imshow(img[0])\n",
        "  else:\n",
        "    Image(filename=os.path.join(image_path, img_name)) \n",
        "  actual = \" \".join(list(map(vocab_map, actual))).split('#END')[0].split('#START')[1]\n",
        "  print(actual, '\\n')\n",
        "\n",
        "  print('Rendered image and predicted latex seq:')\n",
        "  pred = \" \".join(list(map(vocab_map, pred))).split('#END')[0].split('#START')[1]\n",
        "  render_latex(pred)\n",
        "  print(pred, '\\n\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 917
        },
        "id": "t5oVuPAnu6r0",
        "outputId": "ffd30852-5559-497b-a87c-55da36127560"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input image and actual latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=132x32 at 0x7F14FD65C5D0>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAIQAAAAgCAIAAABcq0cIAAAKEElEQVR4nO1a23bazA7W2Ca0T9SVEJ8gq6/UtZKADSTNXR+owUdIm0f6a+yZ/+KLVWEnNJDsi713dcEytkYjaXS2lTGGBBhjlFL8a4yxLIsvtNb4q5QioqZpbNvGr1yOCyJSSmmticiyLEYAKToK6rp2HIeJ83bvBZLgccT7Gng9HdU0DetXLoPeiWi73Q4Gg/5KRugzobXm42FknMHRuuPtOvu+F7wkzqGAhTBQ1ioRvYaaZVlWVVVAlUdqWdZ2uyUinARuaq2bpsFfHKHWGrbPIhGRbdu4MMYAXymF45HIBwH8kvcF2aOpvUSfej59KCilqqqCsCCilDrMMyAbSPRdtWkapRQbTseI+qZa1zW0Lx8daiYd6MTGZ/d9CzB7fbc+CNgn6HAxn4SxbRvRH3+VUlEULRYLy7LKspQpgcMatYbJZ8mm5DiObdvb7dayrLqugQw/O9riQMQYIzXFDL8dOEqzRRJRXdeH0rFtG7bIlvd6eR0YF9synEBrfXFxYYzJ8/z8/JyEF4PXpmkcx2HDxE0+JPCBTOs4DrIOwt3RdtfxsP9E5oBbIAZgC8dxDiUCzeD6YD5R3iAc86/WOkmSLMu01lVVGQHAKYqCiOq6xl8AnAAXWuvJZBIEQdM0/KhpGuAz5kHAq7TWaZoifh5B5yUYjUYwQWjgOD6LosjzvK5rydsr6RDW8N6s3Pl8zsbS+U2SRCmVZRkwmXVjDKgxzSAI0jRltM7FEdA0TV3XZVkSEVN+FwBXnufBBI+mE0URtfEcd7bb7SvX7niGPBi4apqmfc6CIJjNZlK5HUXDEOq6zvOc7Zedo3NgfSIsBntS0wKrbDqddthmOn90l2fx8ZumKWJ1nwhzKBUi72DJHr39EUhqgSnmeY6Qt1wu5SOtdZZlHNx+/frFKmZPZEOAqwZBEEVRZxfgsCOy5H0vxC5SHavVCkmyY3pSWfvP41l85sR13SiKgNb35r6ZS7Q9etvDDwPxAin2bDZDmGK9M0M3Nzeu60oSLPx2u+ULfrpcLplIxxsk2Y4xyvTQ2WixWPi+b1pt4iY7H4vwEuzBBzUI3vGePp1OKq2qarvdxnH8kt5eAztqMq0WUA6FYci1KfSLUnKxWHSkyrLM87zBYBAEQRzHZVnCXpBpUYlJ7mWwetaQWQAZ3ECQiGazWccji6LwfZ+ILi4uZrPZ/f39SwLvx2+aJkkSViXHWyMMnE2NPYz536O31xyGQ6KdxkWapkEQaK1938+y7OvXr9PpFHWqZVk8k0AJWNf1er2eTCZpmrquO5/P4zj2PA/4SqkwDLXW6/Xa8zy5kIjiOAaabitx007AUKe7rhuGIYm6FqC15irctu0syyaTyWq1Go/HsE0Ue8/CHnwMvsbjMRGVZXl6eopyHL0beEDlCtF4bIGLPM8h7LN6e01l+zR9AlEcyePjo+u6SqnZbOY4TlEUv379AgKs4/T0lNsOx3GCIHBd1/d9CKOUCoIAR8v6TZJEdpS48DxPJk/VArWdihYzFVgoHn3+/JnaDtwYMx6Pfd8Pw9C0XRt6o2dhDz4Uh5tpmg4GAzwdDodgTGt9d3enlIKYw+GQiKqqAuWyLEej0Ut6e81h/A5tMhBxOPZ9n8e3iDP4yx6Kep9TdBiG3FuYNkXbto2U2AmGRkReTiedACX3YvZkjZAkieM48/m8ruu6rsHAnjC9B59jpuM4cRwD//r6Wim1WCxkrUFEcRxz4MIvG9CzensNOJil6LYxLooiDEM23vF4XBRFHMdRFPHsOsuyIAiAkGWZZVlxHMNO8zyfTqfc1XMUAmc8Hsb0BjQhCc9zgIahi+d5vu/z0M20/v74+Pjp06fBYGBZ1nq91lpfX1/jUZZlcRzvGQrtwWfX5BGOMWY4HOIM4BmmfePAyNAjJyGQ6uvtVVNbFhX/8zyHojlV4oRYHcYYyCNXgVf0YmEYovHh6G+MQTDlJaYNSnxCHWdlyijbWR24mSTJYDDAHbCEvdbrNRGdn5/P53PQRDZmcdgG+/hglRWNqMtDUmaV4zPTBE6e50g2uA+9oc3CjmVZKqXG4/FkMoF+iIjbkSdqsqa8v79HSIEDopbvdDHo+Nj7cE5GlNgcymFfSZLA87CFDCCyDu70FqYXphg/DEMwAGRYgDEmTVM+S/yuViullOd5MsK8hM9FEeYLpg04UBw3uWAGkZn/FkWhlELg4nhLRCcnJ6jToJyyLBFmi6Jomma1Wg0GAwRA7G5xgWSMeXh4uLq6IqKHhwcEk9vb26urq9lsVpblZrOpqsr3fRjUYDCo69p13cVigaiK8DUcDr9//07tNLcsy/Pzc9OmXxae2vpEDuNgm2BOTiQhNl6x+L6/2WxYg2dnZ2BguVwixiJaaq0nk8l0Ov3x44cc2L2Ezy7+8+dPz/M4UoG929tbJO2PHz+ysePpdrstigKp5eHhAbssFosoir58+bLZbNI0zbLM9/2zs7Plcuk4zmg0siwrCIKrq6vT09Pf1Vq/3ufinTMPLths4f5s6bKIlu2oaeftMCIJ7BPcJ5rdVpEdRYv5CmPCuMxuH2DaplX2dOwlEv6IH8cx41xeXtq2jaTIwnKPLdmTA4XOTCGKIighiiLP88bjMc8geOGTZ1RVJV+IWtZTIhkMBqDIqRuCzWazm5sbtk12LGqrQ5hY0zS3t7dnZ2eoTEhM9hF5HceBueERbJArExJZqjN1n06n19fX1I7umfOTkxN2djydz+fcEvdr6z7+dDo9Pz+PoggTFyL68OGDrLzRWhERzAh0Oh8CgKBp8xwnVLwigpj4hR74BUTXaowoIjsForzD41g2DVmVAuD7crjbsfGqqjrFrpyCyIG85AFLLi4ukiRBeWp2a2K5ynVdLSYFfL+Pv91uObIbUWrHcUxE8/lcTrSku8uExDR5ZMA31+s1wgnGB3p3JPHklDLO9PsAEOpXylpr7uxkeGlaMMZ4nodg0jQNS9Kf7Hfy87Ojf2ZSMiZHZP1VjZhddlT20i5BEKA5bcTcMI5jhCkjLMayLBRgpp2H8hYdcZiapCn5kconsxv3ZQKQfyWhzhh1zyppHZ2CqmOY5jlrfVahZte9OltLNI7y8s5L+PKRZAPzmPl8znkORSq3ip3EZnaPh5/KeNPf6CkSmt0a//8czO4HGHEcc6OqtUYLxXMUIkIbwF/lvMP2f0GCjIccpXFzNpt1RjudMu+N8NczfoNum3DVfhOk2gGzaT+0uLy8/Pbt2z///GPER3tcEL7xG4mDv374XwWepKn2Exv5zY4x5u7u7v7+frPZjEYj05bvaAm4PjZvC1Z/PeM3dDyjqir5PkMqmq9xhEzhjYfx/h+t/veCES8IiOjk5ERqlq8xQESZ5DgOt35ajBqPg7+e8QSdosiIT+V0++ZRjhtknuCMIvvwI+DvYewAK9eId6XyYFjvEv+9dv8XFbmDqprWqZkAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " \\check { \\Lambda } ( a ) = { ( a , a ) _ { \\c H } } \\Lambda _ { a } . \n",
            "\n",
            "Rendered image and predicted latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Math object>"
            ],
            "text/latex": "$$ \\tilde { \\Lambda } ( a ) = ( a , a ) _ { { H } } \\Lambda _ { a } .$$"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " \\tilde { \\Lambda } ( a ) = ( a , a ) _ { { H } } \\Lambda _ { a } . \n",
            "\n",
            "\n",
            "Input image and actual latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=120x25 at 0x7F14FD65C450>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAHgAAAAZCAIAAAB1rn16AAAIYElEQVR4nO1Z3XLaOheVLRPSR+o0GJs0075SpzE2Bpo8UqcFbEM60z5RG8uWvosVVjem+CT0nLtvXzCyLe0/rf0j4TnnmqYJgsA5p5TyPE89g5xznCnHz6fOqrZttdZ4L9U4j/lzRMNq9Qxb8BK/1lrf9+UbqXlnLSbj11dK0cv8LB87hK8QgMe/cYS1FgOpK2wAQVCPPmdQv5dpPuRSQ3pW7dGAT9Ccfjj2nu/7bdv6SqmmacAIvHzf7/EdpllrtdbYK+h0hrVt22J50zTcNnCDCKihtf7XQQ0vW2thO+Ria2G+c05rjUcswRxsEnBKF2OaMUYpheV0CDfDI3AYFHTEKS2NMYPBQB3G1EtNZcSRuG0Sbvz6b/kauwi/wAoQraBiUkMkWM4/ThccyHRB5k3TeNwoa+1gMMBGGWP6DWvbFvPVX6RR6qSUonKe5yFWAJz/IkdTtEzW3ICiKLbbred50+lUCZjLherQocAvIoDIA1vO95kKsMw59/j42G+bc+7du3fL5dI5l+f5bDY7w1RjzHA4hG3gA0WBHQqCxmfwP0WMdAhdr9coUVrr7XYbRdF8PnfO1XXted719TU1NMbsdjuttQTZYrHQWi+XSwDc9/0sy5DZgyAAVhSNARljwBGmuhOET0qpsiydc6PRaD6fo268lIqiUErVdQ2Gs9kMXjDGAAvOufM49xM4O+c8z8vzHCKgzHg8dnsbrbWj0SiOY3jWOZdlWRRF1tqiKOj9yWSSJIlzLkmSi4sLcG7blkyePGatpWC1L7X95lVVhR1C1FdVRQ4voul0GscxBkpUeWgi1ejZ+DOI2lpraa/v+9LLxpimaWDpYrEwxrRtG8dxmqbOuTRNoygCE9/3N5sNSneWZZK5dMtT6mBkAep8A1eyxcHisiwx4eHhQSlVFMV8PscEhjnruDoq7hyUZRlFUZ7neZ7vdjs2jlSG5QVZRYn2RvLnez66wwar88iix7SwWCyUUkAr5gRB4HneaDSKoihJEqZvrN1ut2EYDgaD1WoFj8VxjJmU2E2/8DeiQ+KoaZooiiAeYSXThed5EA8ZEiZEH/lUVRUEge/7q9UqjmOlVBiGgIDWOk1TxiaxwPb5GInoBZ0IcClOQhKwkHzaPcVxHARBGIZ4iZAqy3K323GyMaaqqjRNPc+rqgrwiqIoDENYnWUZEBZF0Waz4UIkQ8qFDgFrvdvXVrjv/fv3SZIURaG1fvv2rXMuCILVavXmzZtv375BvDEGoAZHtr0SR9vtNs/zx8fHLMtubm7SNB2Px8vlEq5kz4S6gQouG1g+Yuz7PrtXDGazGUQDm3iPMDXGaK2jKLq6upIGZlnmed6HDx8+ffoE81HNiqJAYUdfACYobuv1Gl4ejUbX19dhGE6nU0Dk9vYW9v769evy8hJ1ErLQeIBPwJYDA7Vv9R4fHyeTCUCRJIm19uvXr3EcIyN//PgRre5oNKJ3pLPgEWvtw8MDsD8YDNAzISSxE2DSti1PEHQrJshjEeaow+YMKkGBuq7h5aZp5KEZPQAtbdt2vV4jEJX6XaW01jc3N69fv4bQIAi+fPkCzlrr9XqNugcPYoOZuDzPu7y8VPvcy/zjeKMgo0+pg0cmTTBFOKxWq06bIQP5j7UUliA8WYU7xOwhiyFZsQMxxhzzxxsEbCcFyXzSaWO01re3t3hMkgSIpjgIotpFUaRpWlUVE9dx/WenAVu4f1ji0SrsAKpKXdcXFxdt297d3SVJAoR2jkNOXKlgjLMWAcjNxHvP89I0RV1mBYaKnYKM/MP65pxbLBYsCZ0rsDzPm6Z59eoV6q1zrq7ry8tLWAvIj8fjTlnKsizPc7dH9G63i+P49vY2yzL2uNDq4uIC6Z4aSpOpBuOMXuIj0pov8ykff/z4gRPE58+fkUOttTc3N0o0EvLGgwfL3W7Hwx4UnU6n379/32w2FI+srUTdV4dERBhj7u7uEFLWWjRbaLHV/pYHcn/+/MkLk+FwCGQNh0M2PDQTWeXh4QFlGbkuDMMwDO/v73nHAuZFURhjUOjUvgEFCORM6UOOy7KEf56OjhL88FHTNFmWKaWqqoISEInW3R32FTKOkK+hFgMH9QQNDD6hjZFFuTOQqWMymYRhiIw/Ho9l58slXIh4Z/B2cpSc5vv+bDYLwxB9MSgMQyRuJIHtdquUiuNYSgFbMu/0PPzdbDbAB5PJQSpU+xMq5o1Go7IsJ5OJUgoKHScmUtu2ZVnGcYxjEvcDwJnP5zx6oUc8RTJHo2ePoigIgvF4vNvt6Dtu5EsJW1UURRAEV1dX0nxrLRAGneM47le134TZbLbZbLDlT46m9uyQ3BHQmOn7KU1TiRGUr45HWJd6FGVsxXGMHcrzHLElNTzD1x1zWEWdOC7DNX/D3x1WbGvtUzoHloMggFTmZbW/t5Zt1imq63o4HIJ1p7jxKo5l5BQTJD55YauUYuvGVOjEjdpLSXaHap+j6AG3vxHt17OH2BTg8emSj4dj5MEgCDDJ7U8B9G/H+GPt7+/vEd0sx2rfz+FkKLuRU4SOnidDpN2madiH4CvvKl/qhbqueU+pREH2fR9eBs54VD6DAEq1r7QA2e+Lf2IEuJYti+zY+jeZhdiKvxGwBPvsDv+vO6UlT0/q8AJeNir/GGE9SsqFbN2OFTtDhHSR5HYAvU6zzAObE23yc7THWIr54390p0i2omofeurw0PVMVn/Uk6zkvZISx3p7+F/ES0XgFNLR8/dfc/LtKRueY9uxxyFYorufD1NEB7ncrRdtW7+SHQ5S+X5gnSLpQFkGPCcOIJ0F/uFfiP3b+wfWJwb9JJPV8SGzJ+c8k6QaUjd1CIJ+wP0jf/6S7e/U8X/6T+l/qQOyIpzXFGkAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " \\lbrack x ^ { \\mu } , x ^ { \\nu } ] _ { * } = i \\Theta ^ { \\mu \\nu } . \n",
            "\n",
            "Rendered image and predicted latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Math object>"
            ],
            "text/latex": "$$ [ x ^ { \\mu } , x ^ { \\nu } ] _ { * } = i \\Theta ^ { \\mu \\nu } .$$"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " [ x ^ { \\mu } , x ^ { \\nu } ] _ { * } = i \\Theta ^ { \\mu \\nu } . \n",
            "\n",
            "\n",
            "Input image and actual latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=123x50 at 0x7F14FD65C610>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAHsAAAAyCAIAAADz8jOMAAANa0lEQVR4nN1bW2xUVRdee5+5tOg4RUINPhhRMBrvUaMhPhgTFQwhIcGQGOOD6QMWbaUgaKHnzFSFGiiIA9QmotF4aWIkGlHjiw9eE42ERCVNRAF9MZQUZ2ht51z2+h8WZ7F6znQY/KetdT20p+fsy1rfXnvd9q5CRPgXECIiotY6CALLshBRKUU/6Ss9/AdIz+DcQRAAACIGQaCU0loDgGVZxhilFL3krzPIZ31pJhG3LAsAEJEfjDH0iV4S1pZl+b4/g3zWl9RMWRWp18YYeuBPpOlaa2bvP6PmM6bjlmVprY0xxpjLLrtMa51IJNavX+95XqFQaGpqsixLKTU+Pv5fMuIAocuafgqCwPO8IAiuueaa9vZ2ROzt7QWA9evXA0CxWPQ8DxEbGxt93zfGzBSfdacZQxwRfd9HxEWLFjGgO3fuJNC5zdy5c0+fPh0EwcywOAU0Y1bFGMMO8/jx46TRWuuOjo5du3Zt2LABEQHg77//bmpqmikmp4QQkVSM9IgMa90XtuLg9Oz7/vz5823bvuSSS5566qliscjmJZPJPP3009SdGhtjpL5HWKVP8ic3ky0n+ySfqzDM7+NA1QId0NaOMFHfXUxTMDF8PIvruoVCoVgsMgOI6DjO6Ogo9aU3tA94EG4cF34yLHjqyCA8Mq+r7Ou6brxLxe61kDr7SykAcF03lUrVdw/JyI9SG5lGUiAIAOVyOZ1O+76fSCR48wEAZ54oIpZINMl/0mg0CAeXFUfgXp7nJZPJyDgEOg8ljQFxy/zLBr7vU3xVHZCz8TiLR8vLkXK9SAoWBIHWOsIuSSuXhBfDdd1kMjlZ0i/VxbIsRkSSXCFSTIkjST2ZvAQIFx5gYroQkagWKLTnefREHWgZ6ws3ACSTSQwtACkCiR0EAe10qqhIHFndUqmUUop2LgD4vs+sQqgrvu+nUinChd7LNJU1iVeRSM5IJQcyYiDcm9aaupCx9TyPZtFa0xS0HqRANEh1Oqvjcg+SSBWV5Z+RLE4BgOu6r7766tDQEOsOUyqVotmNMalUitKfdDo9Pj7e2Ni4efNmEpI1FEL9YlXliTZt2rR3796xsbGuri7btlmHeO8z+tKYsBa3tLR89dVXR48ezefznZ2djClNHTEgxHyNanouy69o7P5/kgOykH19fU888QRJm8/n6b1SigwI/UlQAkA+n6cdUC6XE4kE4Ssl5GF5rqGhoauuuqpUKp05c2bu3LlXXHHFsWPHyLJFzIg0Dgzo559/3tLS8ssvv5w4cWLRokUPPvjgBx98QJ9ooyQSCSmX9BznBUTR9uHpAUAplc/nJ8PdcZx/gLj0E/Rw4403/vzzz5Zl/fTTT9deey1MUpIlNB3HeeGFF0ql0pw5cyLlXMdxyEN0dXXxjKdOnWpubmbzunDhwl9//VUpNTw8XCgUfN9fsGDB448/DqEtlV4aEb/88st77rmHkEkkEkuXLv3kk0+Ic611Pp8/B1+o+NRYfqqGCFtY8iq+75NaAUB3d3cul7Nt23Gc22+/HUJPIuMnGTzFiaPjeMempibyk6dOnYp/jYRoPT09IGoSMjwHAJaZ42h6OHHihNa6UCh4ntfW1jZv3rzdu3e3tbVdd911zJgMNOmB6wqffvopALzzzjuMjDGmu7ubsM4JWr58OdRWMgGeQ07peV4ikdi1axevhDHmwIEDK1euRBECX1DYHsE0CAIyyqxlPFrkgZ0e2R9+z1Hw2NgYWSROqej9yMiIZVk9PT3U3rKsl156CREB4Oabb0ZE27ZzuVw+n7dtm7Tqueee279/Pw1y9OhRAHj77bcx1BgWAQAefvhhful53qFDh+68885aAnMwxjCL5XKZRl+3bh0A9PX1RcR+7bXXaFApW/VKE3MpEWeVLxQKtJmomBVRN5nj8YCsHMxYuVwGgGw2KxkrFosAsH379jNnzuzYseP333+XAc++ffs49ZMJlOd59Hzs2DEAePPNNwcHByXoiPjuu+9qrR955BGZfAVB8Nhjj50XbmQdj6RS7e3tZENpxP7+fuLv8OHDk0leheSiStBp/MWLF5M//O2333g/RZJAZkxmyNSSntkKUwA3Ojqqtb744otzudy6desWLlx48uRJsjytra0cIDFXkcLA4OCg1vrKK690HOfRRx994IEHpCoMDAxQaIiIruu2tLTQ9jp48GAtm35CzomInuelUinSOwrU7rjjjjVr1rS0tPi+r7WmPC2RSJAb5ISluucE4ZY5c2OP3dTUVCwWlVLFYvGiiy7igwiZLqKI0NlrsWstlUrZbHbr1q3PPvssAIyMjGzfvp0gdl33+eefN8Y4jqOUOn369FtvvTU8PBxng33j8ePHX3/9deqOiBwsUGJMC0brvXr16mw2u3///oiw1YjXWW5hFonCgO+++66ihfI877333pPRcZwok/Q8L1Jd4e1vjHn55ZdJ5ieffJLyiIqqF6mK8CCe51FYmclkEJGieKnFxpi2trY9e/YgomVZN9xwA060lnJSHpm2JgnOD+So5ansgQMH2K9WVe6zBIwd92lvb7csa/fu3TQlZSXUjLYPo0/bvPpM0o67rhs3/WQZ9uzZQwJ0dHRICMi1oPAEEbCIecuyNm3aBGESH4Hbdd2Ojg6q+pL7lX5Ihij8U8YwLKDv+++//z4ArF69WmanEWHPg3gkbguCYMOGDSCCB9u2WZW4m6z/VTflbJonC0X4IZPJKKXo4C0CmZw30sv3fdu2GxoaSqWSZVnbtm2TwkdKjDjRY8k3ka88kdyLiDgwMEDJDv2Zy+Wk/6wJ8Ug7KrOQ6WRFi8cM/H7VqlXVrRYZ7kjEgkKVWKRsNsv5pNRB+cYIovdbtmwhg+t5nlKKIpb6EsMtk9tIvEgtv//+e8dxcrncxx9/LLGSCJ/TcbZTWuve3t44ynGNkL0mYzeu4BVX8eqrrwaAv/76K95dtqfpOPR+5pln2D3Sn+TYLwTPmojBBYBVq1bRS14JmnHZsmW33XbbwYMHly1bRsGu7/tffPHF3XffLYcCFKq0ZcuWJUuWAMDy5csdx/nmm28i5xURmOKnGVVIAkFrQJEcIpIRL5VKUnkjM6JYcnro6upKJBLsk40xxWLRsqwXX3yxFn5qJJ7Otu2HHnpIKXXTTTfZtj0wMBBhz7Ksjz76CBEpBKCX999/PwBs3LiRm1U4A4roYCRvNmFwLbWvuo7LQeKWrlAoAMDw8HDkqCUIAnrDChFRXmMM+xg2gLZtsymvF7HDl7LzvMRkf38/WRuKmqgOQ1zt3LlzAuJSABTOKq5cWAn92vmW0LMRJLjb2towdvQVmXFsbCybzY6NjZGEcVdc41a7UIrLGNnx9OcPP/xATmjFihUQVp9IzPvuu++zzz7j7iBDUc5xJS6R+eKpZkXNjXAcibS4+6WXXtrR0SFVJiIPq9W2bdsqBn84SRBVd4q7MenYfd93HMdxnKVLly5ZsoRN/LfffnvXXXfJcQBjZiQek5kwwmOppIQ18sqNGaB58+a1trayzkoZeMnpTS6X01qPjIzELV5kqeoOt9x50k/yROTGV6xY8eGHHyKiUmrt2rXc4Ouvv47kK4qOMzhJBXH8xrk+F+MrnvDi+VJb2RHDA+XLL798aGho7dq1zc3NVD8wxtD5IafRvb29o6OjchwiWQaAWG59Xn4ulFAUKipWGhBx5cqVf/zxx6FDh7q7uzdv3kzFDyOuTjJL03fTkycyxtx6660//vijPKmItInIwyUUPi2rO6zTRlOOeCAOczG8gIATj76C8JgtUlqKX16gggZr+pRyPkU0rTpOZ1QQHg1jpSNQIj66xfDgm7+aidcW6ngCPj00HYjHldeE1yXidxlk+ZfOgtkaBkEg77cAwGxU8ynnGMN/gSCMAnE7RRaaISyBcjUcAKjBjh07lFINDQ25XC4Izx9qv5Hzr6N/HDbVSNVjSgoN48UWWgYqBWutqQDQ2NjY2dmJIlCrsST9r6LpsCpsTMrlMt0YoSt9W7duJUuSy+VAXCSCiREqWX/LsubMmTM+Pk7VcJi1/wA3HXaQcUkmk3SpbO/evQT9/Pnzu7u7KQhJJpN0AY98plKKKtEQmhd+honXYGYZTfUmYgvAP8vl8i233LJmzRpEXLx4seQhYnnYalPSm8lkIEz05Zizi6odUdaFzuVaYcKZSqUOHz78xhtvIKJlWa2trQBg2zYiptNpXiHaBBs3buT4PRD/HkfDzkarMuWIExFkruum02lEXLBggda6v79/cHCQLh1wKkTt6WBBa51OpxOJxOjoaCaT8X2/s7PThBcIggu5Q/wvomnbTWwBPM/bt2+fbdt0S+/kyZMYRh3x/4IYHx/v6elpaGgAgM7OzkgNcjbSlDsfztQp4Esmk319fUeOHCkUCtdff/2RI0cQkS9LyisoUifoIVJHIwFmXRI05ezKeINuv/i+/8orrzQ3N997771//vkniPvgGGY3IIr6cZPtui6ISyOzi6Zcx1EUrVgrKb6mZ7LLMLFSiLECKUy8po2zMxiHadDxOJrGGLIhBKWEz4Tn5awHtCQAwEUVHmqqdWWKaHYmEbOZZp8dnO30PwTa6n3gyAS6AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " F = \\sum _ { g = 0 } ^ { \\infty } \\lambda ^ { 2 g - 2 } F _ { g } , \n",
            "\n",
            "Rendered image and predicted latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Math object>"
            ],
            "text/latex": "$$ F = \\sum _ { g = 0 } ^ { \\infty } \\lambda ^ { 2 g - 2 } F _ { g } ,$$"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " F = \\sum _ { g = 0 } ^ { \\infty } \\lambda ^ { 2 g - 2 } F _ { g } , \n",
            "\n",
            "\n",
            "Input image and actual latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=148x25 at 0x7F14FD65CCD0>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJQAAAAZCAIAAABYcJCFAAALFklEQVR4nO1a2Y7bOhItbe4G8kFB4Fib2438UgJblOR2P+SHsrUlb+l8UtqWSN6HE1XKkpOZuckdZICpB0OmyGLtVSzRsdYSkTHGdV1rreM4RIQH/iUijP+Z0CO+aZogCPCKBzGHBGvWWoz0lhOR1trzPLkFj2ByDxv/pYHoWIDUyRCoelJlPI7j8MLhdj34Tj1wGWN4pUTHe/wOaf9mYOLxNwiCtm211tQRzAyCHUyGULTWknesYs1prY0xGNFa4y3vCPwwAkwDfqk5qQ/MB3KtteM4QNi2LahyXVeKmql1XRczsQuDI/cbmicmtW3LtvynAUunbVvf9+WrnrtIHvkVBuXanqPILTD5ItohVT9xTSAfLpQxg+F4PI5Gox6SbxSyhoFOa+37/sVQeZHKPwEkYcNQAY6koFkHkAUm0KXYZYxhL+zFW7mR9Nee29HATXmVFDKIkUuMMcYYEAbbGsrfZa6ACJOkz7FQ/gm5/zog/oBOa+3hcPB9H9JkmULcmMli9TwPCsB8GaPwEMfxq1eviKhtW7wFBlaGVDB2Yblhaw7OGKzrejQa4S9CJSiRJjKbzeI4BsHMCGuurwUsttY2TaO1xoPv+9gYbCilYML2jwTQr7WuqgoysoLaxWJRVVXTNLaTLNSGCVjLcDqdsBYPURRtNpvhjowcCGWs8zxvPp8zZmyktf706ZPjOOv1Wm7KeDByOp201re3t3Vda61BM3PXI9VaSxfJAulElOe5HD8ej/++TP87wJZnrQ3DEAS3bYu3TdM4jpPnOWwUrrNYLLCQ5SIR4i9+1+s1HBoIWeWYgEEimkwmvEQp5bqu1B8gTVOQAWyMkEcYP0xQ2lZPxwwk/8NDP336ZLtUoZTChDAMXdctiuIXZf3PwXq9RvxnQWCcOUf0I6KyLO25zowxcpUUWZIkCDxDhBgkoqIopKyJaLlcYqY0gou2MgStdRzH2LRpmov7AlxOmPg/Ho9vb29JnBPgf4fDYbFYZFlGl/If19BD4Mkc6/mZmacue/MI6OmhAgPUZX65ioj2+/3Lly+RtOwgt1trXdcty9LzvNevX1NXR2A7WeMwIE1GUbRarZCiegUIEd3d3bmuO5lMmLCqqlzXBZtc+Gy32zAMkZu5wujxTkSn0wn4oyhaLpc/IozXukwQZ2nbFS985kPARUTiAseKCNM70vZEADzgGbl3v9+HYZim6e3tbZqmTIDv+8vl0hhzdXWFkaIosiwDD2maXl1d4UGenPBcFMXNzY3tyrm2beM4dl2XbZFE8QnhKqUgI8g0DMPr62vHcbbbLTzV87woilDsYc5Q3MaYJEmCIDDGHA6H2WxGRIjMLPrVagU2uR7kWjdJEjC12WxGoxFwxnEMofGZhI3GivOfLw89TdNw4YTXGPd9H0medQxluK5bVdXhcPhRCwCA2EvdwctxnDiO8zxHZHBdF2ax2Wym02kURY7jzOfzx8fHjx8/gvTRaKSUyvN8MpkQ0Ww2u7m5ef/+PSpp3p1Pvp7nvX37VmtdluVisWDmi6IoyxIcrdfrz58/v3v3DhU5DOLNmzcPDw+73W48HkMfk8nE9/26rpMkseIYgDPZ/f2953l5nmPfu7u7+XyOLdinOdfCCFijxpg4jj3Pe3p62u/3SZLASUajEcxuv9+D394hB8uPx6Mr1RsEwfF4JCLEaIDsBZCIkND/dDqlnzbPYKQwEZCulAqCIMsy2GCe5+A2iqLJZJKmaVmWdV1/+PCBLRSF0ng8hsXM5/O6rlerlRUH3iAIwjCk7mzw9evX/X7/8PAgy2b2FSLabreLxSIIAlhP0zTb7VYptdvtHMdBm8Z1XfRWUMHyaYGIoFpr7Xw+XywWWEhEaZpCB2zNsNfJZMKuA1+/v79/fHys69pxnDAMETy4/2KMqaqKVSUPslDB1dXV91JK5tvZbAZ7hNm6rpumqUyVnEXRiLqUd78Dl7xIv0icfKiCH3BuhwKsqB2MMUopp+shsPSzLJNkEJ3xwnEP5UnbtlmWwYTZFWx3QIqiiNduNpssy1DdMBJOGbYrfLTWeZ5TdzLhypOLF57/9etXRF0e5MnYCBaAXfBXax0EAZesQNhjzVrrk0jdEOhutwvD8Obmpm3bL1++JEkSx/GHDx84+LANEhFsBxxe9DxjDGpUzt4IwqfTqdfMxWR4ZxiGSDysZsdxTqfTaDTCqiAI2BgRPDzP22w2cRwjbCKIaa3n8zniFfosHPyBx/O87XYr40ee53meywYYnddT3Fbe7Xau6yI/Ia3IJWhrNU1zfX3dtu3j4+OLFy8Q50Eq0gcWBkGgtUYegZCbpnn27BlTa7ouJnUtIWPMtzYP508iGo/H7AFt2yql1us1fB91FOcVz/MQW36S8GRk4waS1vrz589N01AXruGXt7e3k8mkqqrHx0cEDeryq7UW+RylU9M0KE/Y5rTW2+0WogFVh8MBNd719TVWZVm22+2Wy2Xbti9evIjj2HGcuq7ZLpMk8X1/PB7D2DmVpmnKURQSQL5PkoQFslqtHMfBhDzPUYbw+f3h4QFax151XUPxkLxSKooi6rISjPj58+e97vl+v4f1f6PNnp+E2NnruvZ9H/GkaRr5Cs88It35IsgACEiSBJhPp9N0Og3DcLPZoB4DMXDTzWYD5AibSZIYY+q6JqLpdCr3bZpmNpsppXgQMi3LMooiuD4RlWWZpikOskhRiCsooBzHAVUMbBDDwIVcJQ++qIqzLKvrGkU7UlfbttPptCgKI9o6cRwj71proyiaTqdSEUDeky0Yl+c/kvLlBKaUQnWH36qq5LTfAkVRwCTLskQihGeAjKIoiqJQSqE7xXKB7WdZJqWJ5zzPucLm7EXd4Udrvdls4FvMy3q9VkpxAYz6HimNOVVKYYnMPRALE8lv0zRNkoSIqqoyXWMMtMFHmX3YDbyt1/oA+9hUtnuQ+5FiMfi9PSbFIZUkW02/qLCLIHsT5rzTwVUD8jnesnk+PT3ZrmPHh3ooW3YCJSoYLAcSay0cjqeBR16OOohrEIz3ApXtuon2vCHCwG1VyP3p6QnJEn1O4MEc7j7CUiVyJkyqiZimYQHJU+0lpf4iyB3RJ2TjgID47el0KssS5ySef7Ehmed5FEVGdKGkwfXCO1QYRRHExELgpmXTNEopOC7zzmjZ2np8SeGy8VlrlVKoAUEn5/6hZBaLBToDUlaSF0ZLsnUtBSqX8fi/TG//KUiL4RE2ZGut1nq5XHJxKIXLy/lTgLU2jmMUIIyZv5awQUgegyBAtOzV99ba9XqN47kcl3bT88KeJljELPQwDB8eHqy1sAnQA+I5KlRV5fs+8hQORdLEJbamaS543vF47Nm+PQ9EvwV4RynTnnHIysieK9gOYgij4qJDGgFP6NHAXXgrLBV4YAfy1ZD4IWE/ClFAe3NzM/Q2aQ3YtDfOnPY4+nby5U9cvc/wVnxN1j/tYf4NMN2NBD47yj4CdQcMeU3IdDU9X1yw53d1QCTOnbLvY8WR0YpWEw1uS/AS1Kv8zZbXkuh3aNFZRFNG3kNg6enuK64kj8ngK0a9aUy5ZER339yJyJGIeKoRXVEJdtCt/9vAFiPlZX58Wcqc32PAoCNudvDD09MTDnaMiqVvzju8zuBDAXTAn6Pl1nTeIDaiZcEWRoPLDT0e0WeQ8mRGrGi/uYMLanRuWxafKu351T9JhxTcRV3+Ogwp65mRNJehauVy6r5g9Nq4PF/eMpKmPVS/Fd9VWOvDTXvPZtCO4b945vtFwzBw0YjpXGFDHX234v/D/xz8BTJKWC/Swo5oAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " \\Psi = \\operatorname { e x p } \\left( i l \\varphi \\right) R \\left( \\rho \\right) . \n",
            "\n",
            "Rendered image and predicted latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Math object>"
            ],
            "text/latex": "$$ \\Psi = \\operatorname { e x p } \\left( i l \\varphi \\right) R \\left( \\rho \\right) .$$"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " \\Psi = \\operatorname { e x p } \\left( i l \\varphi \\right) R \\left( \\rho \\right) . \n",
            "\n",
            "\n",
            "Input image and actual latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=106x47 at 0x7F14FD65C490>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGoAAAAvCAIAAACE1sgfAAAKbElEQVR4nOVb23LayhKdi4grn+SyBZLAD/kk3yQBtvNLqdhIXJxK/ijbSDN9HlbUbgQGTPapCjv94JLEaGa6py+ru2VNROp4CLvVWvMtERljcOu9x7Ucxg+Z5BNcExHP+da6Wmsi8t5ba/m52fLOH0haa/DpnKvrWmvdEoT3nkdimDHGOUdEzjn8ysOk6LcTn4e1tq7r1/0ci/ZJNWFlgXTkEzyUCrJcLj98+MC3rEE7NU5SXddBEKwr8tGITzUW5L1nq2QxQb+CIJCyk9d4t6qqTqfDZ8Azb5EjiwzrtkYek/Gy6YGNu7u7LMvyPK/r2loL2bFQiAiyc87xDJ1ORymltXbORVH08PAADyDHtOj5+VlrvVgspN69jqcjIem/vPfdbnc4HCZJYozJ83w4HCqlgiAIgkA1npGIpMqkaVrXtfeeiHq9Xr/fxzX+bll3NpuFYYgJW3Q04gOTECI1lui9Z66qqoJrZ3Hgp9lsZoyZzWa+oaIolFIs341yaRFOa3380YiPiOq6hvi892EYXlxctGQKa2Xx4SKO46IomGfvfRRFo9GIiHAAcoZ18t5XVTWbzVjikg4UH6/HNnXYPIct7b0vy1IpNZ1Oeenlcjkej40xzKRzbjQadbvd1g6VUmVZEpH3HrOR0G55iwF4S2ud53mL03eLT54A5tpH+X+fYDVYsSzLOI6NMYAyvIfb21sEVpaCVEZMkqYpwm5VVfw8y7Lb29vlcol3kyRRSllrpZSTJMnzvMXvuyMvDARTwCUDQ713nvcS/JpSajqdxnF8dXU1mUzAOSMYa21VVVmWITpHUTSfz7FJyAWT0GrqUhTF8/PzaDTqdDpVVSVJ4r0fj8enp6fz+RxjjDHn5+e4lfy+W3x4MwgCCVN/TzL7EkDyaDRKkmQwGPR6PaUUdFA1WoYnRFQUBXjm7XFU6fV6jJyJaLFYXF1dVVWllPr8+bNSqizLy8vL2Wwm0bUxpigKzmpAwcGcKHEO+8P33yFjzHQ6nc1mj4+PEARvAAgZCAYMDwaD6XSKt9RqsnFxccHbds7BJLHEZDJBlDDGzOfzl5cXjAGDUBqZxrxb+7TWgKkPDw9pmk6n01Ya+P8jrfV8PvfeA+5hXVglNAsX1tperzccDk9PT8E8EWE8JPjlyxd2QYwKiWg6nUZRBCertR6Px0mS4JAwDGx++PCBdfCQrMNaOxgMnHODwWAwGICZf09Kmwk7Xi6XnCQ8PT1Za4fDobUWCoJtpGlqrb25uQGEDoLg1VUZ45z7/v07YksQBMaYMAy73a5SCqpgjAmCIEmS5XLZ6/VY+4io2+1C1q/WdlgQ1FrDZ+d5ftgkB5BzbjqdBkFQFIVzrt/vd7tdiWPZpRRFQQ344EwDagiLpiY613VdlqW1NoqiXq8Xx3GWZVrrOI5bUOb8/Pzu7o5WI69iiWAoaD03ttYCf2J8lmVlWQKC4mRafPLWSaAnxgqcAMi3WnB3/RrAoigK7C3LstYAbKzX620/BiiaxHRlWUIPjDFRFGVZJnNEMKKUAvsr4msBRSatdRiGzENRFIPBAJrMI8uyRD4E+M7Sl9xuFFALddd1zZhLHuT6ruScLSlL6W+B8c65+/t7YGkQDjVNU6WU3IDc3t3dHcJOK01Q8ob1HOUtmCf/+vXrV+TnWGMymWity7J8a69S1+RfaWvyXamw6zJqTcJLSG7lMbwlPgzTWj89PbFxeO8HgwHUmTeJmcGF1hqZX+tQlVyJr6HJwKVS3p1OJ01TIkLOBGVWDeZqiUZOu26n8mA4gLbebc0mDYU5pzUN3ZlBwpgAX1j6SqnRaMRSk/P0ej0Y4voB//J9Ly8vcotZlrF/lZwgrjEQRWiLoogHsB9EnQdBKo5jTNvygLTq1zEDfm1VTVqbXk8c+fWd4pPGBHXDi9AVEioMKooCMFuu/poIe1G8RRMAkTsMQ6gYwBTMWWs9HA6vr69bVVy8FYbhaDRCzFosFlEUwdLH4zE16Mw1FWCcELCY1hrFTswDhBVFEeISr8Jqzqv71eo537rVen2LwIta6zQhq8GRk+gK8GywepmKBFzChuwgC+dcFEV4jtFBENzc3CAfwhpVVQGL4on3/uTkBAeF+IUqI9fgpOyUUkmSzOfzqqpOTk5+/vzJYoKgORVTTZqIvcnGEMuRxcHZ2xbZAe7Be3DqySsyL7L9Zq1dLpedTkdCSMWbY5XGnvI8R36Hnzikog7BD3HBtjYej29vb6kxW9gFpgWU5xelc9mIUTYimF/G0iAqFiUDrPWLjcSSlWk7VB4/8RLylq+NMWAT9EuHpdICE8VxrBor0FpPJpPFYgGLk1vEsUsNV0ohmcO22AOyYWLF0Wi0kT3oFCofQHCykeZXM/btxMkcuIOS8lQbZ5YqT3u0MV+ngwIT0Xw+j6IIi0Eis9lsMBicnZ0hIm8kdA/CMIRVYutlWfb7/clk4kUTlns9G+cxxlRVxV0Llvv+UgPB86jVk2bTY3uXfl81PmH7DleIBBZ1ziFccF2/KArUF8fj8c6yaLfbNcYAf+Z5HsextXY6nfISMp76N4gRXCtjOYDYOVRVBSeDv8wvw0YSCdK7Fv3V7sRfhEioCRFB85VSaZrCH/ORvnUSo9GoKIpv374ppS4vL9kA34qV67RxwAEKyOqfJAmi1mKxOD8/p8baZPyV0YZEF3i3Aq43nPhwXl5e3nXUvw5E6+Fw2FLVLSh3I63j8PcSdBkllvl8nmXZ4+MjNYq/jsxZE5nrfZZWtAruZUeKRETeh2ciQiYHtgEvWyJrJRjrxD/JRSXS3p+Adcqy5NclO845PG/pjWR8J618XMQ1Ra5wye8TtmsxESml0A1AgOcqI8NJ1FmVCHDrxD/hXWp6AzuMaNN+tNY3NzcXFxcwXtXkUTw5ggkCpve+0+mAcebX7exDtI69pSwtJ7pFB2Xxh4Tv5xd55p2K3Dr5g0NHVVVoFcH/8rpI2/lJnudpmkqc27rYQsfUJn+LWr6M2e52u3EcR1GECiuGcUXee19VVRzHsl6Ph7Sf7Aiw+aiJ0TgRVVWFRgTC148fP/755x9YJaRmjInjGGZhjPn06RMXzfr9PhHBopVSzjmJsd+iY/rCaiMhtiqltNbcAHPO5Xl+fX1NRGgewaMtFovr62tOb4FzIUqkpOwi9/S2Ry8+atqV1IAPY8z9/b219vb2liEehPj4+Pjx40dU3rMsS9OU5QiHyNUUtH130tGLjwOl1rrT6bC8oJUw6k6ng15aEATcpXx4eAjDUCllrU2SBB0i1XhA/opyx/K/7bj/CJJwsq5rJI64Rm2CETh6XtQUwYqiiKIINUqOG7Q31P0viE/yzGgpiqIwDO/v71XzPRUIpSDVfN1hreW2kYRWe8bfY/q2+S2i1ZowX6DMgfKHaupyGIwPtM7Pz1FD4h5D658UaJf9/hfEdwAZY8qyhN5xefmQef71nf35hCiBVASfUx78kclfJz7vfb/fPzs7Y43b0hjZSX+d8Up35lcL9AfQX6d96IJyoPjd2f427ZPkxb8Q7tXZWKO/TvuUUkB2arWvdthU/wMnlv58Uwb5DgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " D ^ { 2 } \\varphi = \\frac { \\delta V ( \\varphi ) } { \\delta \\varphi } \n",
            "\n",
            "Rendered image and predicted latex seq:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Math object>"
            ],
            "text/latex": "$$ D ^ { 2 } \\varphi = \\frac { \\delta V ( \\varphi ) } { \\delta \\varphi }$$"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " D ^ { 2 } \\varphi = \\frac { \\delta V ( \\varphi ) } { \\delta \\varphi } \n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# BLEU score calculation"
      ],
      "metadata": {
        "id": "d6MQEwiE_MJe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 20\n",
        "# validation\n",
        "# dataset = get_dataset(npy_path=r'./datasets/valid_buckets.npy',\n",
        "#                         image_path=r'./images',\n",
        "#                         batch_size=batch_size)\n",
        "# test\n",
        "dataset = get_dataset(npy_path=r'./datasets/test_buckets.npy',\n",
        "                        image_path=r'./images',\n",
        "                        batch_size=batch_size)"
      ],
      "metadata": {
        "id": "iROEvYFV1VPx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_preds = []\n",
        "all_refs = []\n",
        "max_seq_len = 200\n",
        "\n",
        "# for bn, batch in tqdm(enumerate(test_dataset)):\n",
        "for bn, batch in tqdm(enumerate(dataset)):\n",
        "  img = batch[0][0]\n",
        "  latex_actual = batch[0][1]\n",
        "  encoder_out = im2latex_encoder.predict(img)\n",
        "  latex_pred = np.ones(shape=[batch_size, 1]) * 500\n",
        "  decoder_state  = np.zeros([batch_size, 1024])\n",
        "  eos = False # end of sequence\n",
        "  while (not eos) and latex_pred.shape[-1] <= max_seq_len:\n",
        "    pred, decoder_state = im2latex_decoder.predict([latex_pred[:, -1:], \n",
        "                                                    encoder_out, \n",
        "                                                    decoder_state])\n",
        "    tokens = np.argmax(pred[:, -1:], axis=-1)\n",
        "    latex_pred = np.hstack((latex_pred, tokens))\n",
        "    \n",
        "    # check if eos(501) is in each row of latex_pred\n",
        "    eos = (np.full([batch_size, 1], 501) == latex_pred).any(1).all()\n",
        "  all_preds.append(latex_pred)\n",
        "  all_refs.append(latex_actual)"
      ],
      "metadata": {
        "id": "2jpIGPWb98AS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17ee7a9a-c085-43a4-d043-cb1aa9845172"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "419it [48:50,  6.99s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preds_list = []\n",
        "refs_list = []\n",
        "for pred, ref in zip(all_preds, all_refs):\n",
        "  for i in range(pred.shape[0]):\n",
        "    end = np.where(pred[i] == 501)[0]\n",
        "    end = end[0] if (end.size != 0) else None\n",
        "    preds_list.append([pred[i][:end].astype(np.int32).tolist()])\n",
        "\n",
        "    end = np.where(ref[i] == 0)[0]\n",
        "    end = end[0] if (end.size != 0) else None\n",
        "    refs_list.append(ref[i][:end].numpy().astype(np.int32).tolist())"
      ],
      "metadata": {
        "id": "P_Na9mHUR-Io"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# validation dataset - 10340 images\n",
        "print('BLEU score {:.2f}%'.format(corpus_bleu(preds_list, refs_list) * 100))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uAOcrA2f2nAi",
        "outputId": "12fca1af-dc8b-41aa-db7a-b9aab01c62ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU score 84.44%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# test dataset - 9340 images\n",
        "print('BLEU score {:.2f}%'.format(corpus_bleu(preds_list, refs_list) * 100))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DbwDFplLZGs9",
        "outputId": "9dda7752-90cf-4abf-893b-a79662b21639"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU score 84.30%\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "im2latex_test_v4_20220620.ipynb",
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}