nlp/Hello_VizSeq.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Hello VizSeq.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iP3vy43SlxsM",
        "colab_type": "text"
      },
      "source": [
        "# VizSeq"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJ3CVtpWl2cF",
        "colab_type": "text"
      },
      "source": [
        "## Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bdl_pozJYtjp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "eb947933-d2a8-4f53-e853-7085ef5214e5"
      },
      "source": [
        "!pip install -q git+https://github.com/facebookresearch/vizseq"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  Building wheel for vizseq (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p7z1WwtQbmNP",
        "colab_type": "text"
      },
      "source": [
        "Make sure to have NLTK version 3.4.5 installed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UmALjH2RaSh2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 358
        },
        "outputId": "c30aeb74-040c-4077-c240-0333f9c668c8"
      },
      "source": [
        "!pip install -q 'nltk==3.4.5'"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting nltk==3.4.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f6/1d/d925cfb4f324ede997f6d47bea4d9babba51b49e87a767c170b77005889d/nltk-3.4.5.zip (1.5MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5MB 2.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk==3.4.5) (1.12.0)\n",
            "Building wheels for collected packages: nltk\n",
            "  Building wheel for nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nltk: filename=nltk-3.4.5-cp36-none-any.whl size=1449907 sha256=b1a12c5d709cad33efc7bc62c83da89fdaf1f5581cb2aa241e7d18f647fa1837\n",
            "  Stored in directory: /root/.cache/pip/wheels/96/86/f6/68ab24c23f207c0077381a5e3904b2815136b879538a24b483\n",
            "Successfully built nltk\n",
            "Installing collected packages: nltk\n",
            "  Found existing installation: nltk 3.2.5\n",
            "    Uninstalling nltk-3.2.5:\n",
            "      Successfully uninstalled nltk-3.2.5\n",
            "Successfully installed nltk-3.4.5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "nltk"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FLAGnpVEZsYN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "from typing import List, Dict\n",
        "import os.path as op\n",
        "from glob import glob\n",
        "import vizseq"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OqdBCar_ikq-",
        "colab_type": "text"
      },
      "source": [
        "## Data\n",
        "Downalod the data for the NLP task and prepare inputs for VizSeq"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lwWXXluHZE3y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "f9fd21be-1372-4548-ad51-c7fdd85353b5"
      },
      "source": [
        "!curl -O http://www.manythings.org/anki/fra-eng.zip\n",
        "!unzip -q fra-eng.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 5800k  100 5800k    0     0  12.6M      0 --:--:-- --:--:-- --:--:-- 12.6M\n",
            "replace _about.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: A\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jZ_rep1TZQhN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('fra.txt', sep='\\t', names=['en', 'fr', 'license'])[['en', 'fr']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nIJUjKeJfIKD",
        "colab_type": "text"
      },
      "source": [
        "VizSeq expects input to be organized as follows (see [documentation](https://facebookresearch.github.io/vizseq/docs/features/data/)):\n",
        "* Sources: Source-side inputs for text generation tasks.\n",
        "* References: Target-side references for text generation tasks.\n",
        "* Hypothesis: Model predictions. One sentence per example per model.\n",
        "* Tags: Per-example tags for example grouping."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TjoVa3dlfTKN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sources = {'fra.txt': df['fr'].values.tolist()}\n",
        "references = {'fra.txt': df['en'].values.tolist()}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5L08Q04Bigpc",
        "colab_type": "text"
      },
      "source": [
        "## Dataset statistics\n",
        "VizSeq let you quickly view simple statistics on your dataset using `vizseq.view_stats`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z98ZM7i6cKId",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 553
        },
        "outputId": "6eb292bb-76b8-42b0-8277-ddc5f234709f"
      },
      "source": [
        "%matplotlib inline\n",
        "vizseq.view_stats(sources, references)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<!DOCTYPE html>\n",
              "<html lang=\"en\">\n",
              "<head>\n",
              "    <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css\"\n",
              "          integrity=\"sha384-GJzZqFGwb1QTTN6wy59ffF1BuGJpLSa9DkKMp0DgiMDm4iYMj70gZWKYbI706tWS\" crossorigin=\"anonymous\">\n",
              "\n",
              "    <style>\n",
              "        body {\n",
              "            padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */\n",
              "        }\n",
              "    </style>\n",
              "</head>\n",
              "\n",
              "<body>\n",
              "\n",
              "<div class=\"container-fluid\">\n",
              "    <div class=\"row\">\n",
              "        <table class=\"table table-hover table-bordered\">\n",
              "            <thead class=\"thead-light\">\n",
              "            <tr>\n",
              "                <th scope=\"col\"></th>\n",
              "                <th scope=\"col\">Type</th>\n",
              "                <th scope=\"col\">Examples</th>\n",
              "                <th scope=\"col\">Tokens / Frames</th>\n",
              "                <th scope=\"col\">Chars / Duration (ms)</th>\n",
              "            </tr>\n",
              "            </thead>\n",
              "            <tbody id=\"statsTable\">\n",
              "            \n",
              "            <tr>\n",
              "                <th scope=\"row\">Source fra.txt</th>\n",
              "                <td>Text</td>\n",
              "                <td>174,481</td>\n",
              "                <td>1,170,035</td>\n",
              "                <td>6,317,810</td>\n",
              "            </tr>\n",
              "            \n",
              "            \n",
              "            <tr>\n",
              "                <th scope=\"row\">Reference fra.txt</th>\n",
              "                <td> Text </td>\n",
              "                <td>174,481</td>\n",
              "                <td>1,075,042</td>\n",
              "                <td>5,340,266</td>\n",
              "            </tr>\n",
              "            \n",
              "            </tbody>\n",
              "        </table>\n",
              "    </div>\n",
              "</div>\n",
              "\n",
              "<script src=\"https://code.jquery.com/jquery-3.3.1.slim.min.js\"\n",
              "        integrity=\"sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js\"\n",
              "        integrity=\"sha384-wHAiFfRlMFy6i5SRaxvfOCifBUQy1xHdJ/yoi7FRNXMRBu5WHdZYu1hA6ZOblgut\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js\"\n",
              "        integrity=\"sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "\n",
              "</body>\n",
              "</html>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAAE/CAYAAACKFmYTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de7hddX3n8ffHhIuKWpSMo4QQrFjF\nS2GMoa1WaQcwlhbs84hCtYWOTqotvaltaW2BRu2gtV5aGYVqvI4iYsdGiYO0QFtrsQmKYEBqiJEk\nKiARlYpg4Dt/rBXcbPbJ2Sec21p5v55nP9nrer57nWT98tm/31orVYUkSZIk9cGD5roASZIkSZou\nBhxJkiRJvWHAkSRJktQbBhxJkiRJvWHAkSRJktQbBhxJkiRJvWHAkQYkeXSSf07yvSR/Ndf1zGdJ\nLk/ysrmuQ5KmW5JXJLkpye1JHjXX9eyOJD+R5Kq2Pfudua5nPkuyOcnRc12Hpo8BRzMmybOSfDbJ\nd5JsT/KvSZ4x13VNYiXwLeDhVfWqB7qzJEcl2TqF9U9N8plJ1pn1YJHkrCQfnM2fKUm7q/0P6x1t\nQPlmkvcm2W/MbfcC3gwcW1X7VdWtM1vtjPlD4LKqelhV/fUD3dk47dPQ+pO2G3MRLNq/C6+bzZ+p\n2WfA0YxI8nDgk8DfAI8EDgT+HLhzBn7Wwmnc3cHAtTXBE3Cn+WdJkmbOL1XVfsDhwBHAH4+53aOB\nfYENu/NDkyzYne1mwMHs4jPMozqlaWfA0Ux5AkBVfbiq7q6qO6rq01V1NUCSByX50yRfS3Jzkvcn\neUS77H69HoPf8rTfCl2Y5INJvgucmmRBkj9JckPbHX9lkoPa9Z+Y5JK2F+n6JC8cVXCS9wKnAH/Y\nfut39AQ/a3mSf0tyW5JvJHl7kr1H7O+hwKeAx7b7uz3JY5OsHRz+luT8JKuTPAl4J/DT7bq3TfWg\nJ/mpttfstiRfTHLUwLLLk7y27Un7XpJPJzlgYPmvtb+PW5P82c5jnmQF8CfAi9q6vjjwIw+eaH+S\nNB9U1TeBi2mCDgBJ9knypiQ3tkPR3pnkwUmeAFzfrnZbkkvb9SdsR9oegXe05/b/BH5uov236x+V\nZGuSV7Xt3zeS/PrA/h6c5K/a8/F3knxmYNsJz/GD2rp/Dnh7e95+wgR1HpfkC0m+m2RLkrMm2N/9\n2qcke6cZAvfb7ToL2vbgjEnajbEk+cV2/7e1n/lpA8s2J3l1kqvbY/SRJPsOLP/D9rh+PcnLklSS\nxydZCbyYH7Xznxj4kYdPtD91UFX58jXtL+DhwK3A+4DnAfsPLf8fwEbgccB+wN8BH2iXHQVsHVp/\nM3B0+/4s4IfA82lC+oOBPwCuAX4CCPCTwKOAhwJbgF8HFtJ8i/ct4LAJ6n4v8LqB6VE/6+nAT7X7\nWwpcB/zeBPsb9Vn+K3Az8PM0J9pNwMPaZacCn5nk2F4OvGzE/APbY/4Lba3HtNOLBra7gSZ8Prid\nPrtddhhwO/AsYG/gTe3nHjzmHxxRx8j9+fLly9dcvobajMVt+/C2geVvAdbQjDB4GPAJ4H+1y5YC\nBSxsp3fZjrTtxneAZ7bn3n0n2f9RwA5gFbBXe87+Pm07CZzTnk8PBBYAPwPsM9k5fsQxuE9bMUGd\nRwFPbaefBtwEPH+C/Z3KUPsEPAX4NvAk4DXAFcCCdtn92o1d/Z6G5h9B004e2R6DU9p19xnY7t+B\nx7bH+Drg5e2yFcA3gScDDwE+2P4+Hz9wHF43oo6R+/PVzZc9OJoRVfVdmv8sF/C3wC1J1iR5dLvK\ni4E3V9WmqrqdZujASRl/CNi/VdXHq+qeqroDeBnwp1V1fTW+WM246V8ENlfVe6pqR1V9AfgYcOIU\nPs59flZVXVlVV7T72wycCzxn3J1V823iK2jC39uAX6uq702hnom8BFhbVWvbWi8B1tM0hju9p6r+\noz1mF/CjbzRfAHyiqj5TVXcBZ9D87iYz0f4kaa59PMn3aMLJzcCZAElCc73l71fV9vb8+xfASRPs\nZ5x25O+r6l+r6h6aodiT7f+HwKqq+mFVraX5guknkjyI5gvA362qbdWMgPhsVd3JeOf4ydxbZ1X9\noKour6pr2umrgQ8ztfbsS8DrgI8DrwZ+tarunkI9E1kJnFtVn2uPwftojutPDazz11X19araThMg\nd7Y/L6RpmzZU1fdpgtY4JtqfOsiAoxlTVddV1alVtZjmW57HAm9tFz8W+NrA6l+j+Wbs0Yxny9D0\nQTS9CcMOBo5su7hva4d9vZimF2Vc9/lZbVf/J9NcuPpdmoZrqkOzPkHzrdT1VTX2RZuTOBg4ceiz\nPgt4zMA63xx4/32a3jNofh/3fs62URjnwtqJ9idJc+35VfUwml6KJ/Kj8/Qimm/2rxw4V/6/dv4o\n47Qjg+3EOPu/tap2DEzvPH8eQNOzMlF7Ntk5fjLD7dmRSS5LckuS7wAvZ+rt2fva2tZW1VemuO1E\nDgZeNfRZD6Jpq3Yaqz3j/v9fmIjtWY8YcDQrqurLNN3CT2lnfZ3mBLbTEpou+5uA/6RpHIB7L4Qc\nbniGexe2AD8+4kdvAf6pqn5s4LVfVb1iKuUPTb8D+DJwaFU9nGacccbcdqfX03SBPybJyWOsP44t\nNMP8Bj/rQ6vq7DG2/QbNMA6gGQNOM8RvOuqSpDlTVf9E0/68qZ31LeAO4MkD58pHVHNDglHGaUcG\nz5FT3f+gbwE/YOL2bHfP8aPqBPgQzVC6g6rqETTX2Uy1PfvfNDcVem6SZ42x/ji2AK8f+qwPqaoP\nj7HtfdozmmA0yPZsD2DA0YxoL8h8VZLF7fRBwMk043Oh6Qb//SSHpLl1518AH2m/0foPYN/24se9\ngD+lGX+8K+8CXpvk0DSelubZBZ8EnpDkV5Ps1b6e0V4wubseBnwXuD3JE2mGm03kJuBRaW+gAJDk\n2TRjuX+NZlzx3yQ5cGD9xRlx04IhC5PsO/Dai2ac8S8leW57see+aS5mXTzJvgAubLf9mfZnn8V9\nG7mbgKXt8AlJ6pq3Asck+cl2GNnfAm9J8l8AkhyY5LkTbDuldmQ39j+87WrgzWluSrMgyU8n2YcH\ndo6fyMOA7VX1gyTLgV/Zxbr3a5+S/CrNdamnAr8DvC8/uh33uO3GXkPt2UKa4/fytocpSR7a/p/g\nYWN8pguAX0/ypCQPAf5sxOd43Bj7UYf5nxXNlO/RXBz4uTR3a7kC+BKw89kyq4EPAP8MfJXmG6vf\nBqiq7wC/SRNattH06Ez2LJk305zUPk0TPt4NPLgd+3wszdjnr9N0Qb+ByQPTrryaphH4Hs1J+COD\nC5NsSPLi9rN8mSbMbWq72ZcA7wdOa8dX/0tb63vaceGX0tzW85tJvrWLGt5B8w3hztd7qmoLcAJN\nj9ItNN+A/QFj/Duvqg00x/98mm+/bqcZs77ztt4fbf+8NcnnJ9ufJM0nVXULzbn3jHbWH9Hc6OaK\ndqjxP9DcpGbUtrvTjoy9/xFeTXNThHXA9vZnPeiBnON34TeBVe21SmfQtKP3SnOnsZ9tJ+/TPrXt\n2VtpriO9vao+RHNN0Fva9cdtN9Zy3/bsrKpaD/xP4O00NzHYSBOiJlVVnwL+Gris3W7nF6s727N3\nA4e1bfLHx9mnuidV9tRJuq/2G7jbaIbhfXWu65EkaXe0PW1forkD247J1lc/2IMjCYAkv5TkIWme\n3/Mmmm8QN89tVZIkTU2SX07zLKL9aXrAPmG42bMYcCTtdALN8IuvA4cCJ5VdvJKk7vkNmmHWNwB3\ns+trZdVDDlGTJEmS1Bv24EiSJEnqDQOOJEmSpN5YONcFDDvggANq6dKlc13G/HPllT96//Snz10d\nkvYYV1555beqaqKnu+/RbKtGsJ2SNMsmaqfmXcBZunQp69evn+sy5p8MPHPR4yNpFiT52lzXMF/Z\nVo1gOyVplk3UTjlETZIkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZ2XZEWS\n65NsTHL6iOWvTHJtkquT/GOSgweW3Z3kqva1ZnYrlyRNt3l3m2hJkqYiyQLgHOAYYCuwLsmaqrp2\nYLUvAMuq6vtJXgG8EXhRu+yOqjp8VouWJM0Ye3AkSV23HNhYVZuq6i7gfOCEwRWq6rKq+n47eQWw\neJZrlCTNEgOOJKnrDgS2DExvbedN5KXApwam902yPskVSZ4/0UZJVrbrrb/lllseWMWSpBnjEDVJ\n0h4jyUuAZcBzBmYfXFXbkjwOuDTJNVV1w/C2VXUecB7AsmXLalYKliRNmT04kqSu2wYcNDC9uJ13\nH0mOBl4DHF9Vd+6cX1Xb2j83AZcDR8xksZKkmWUPzm5aevpFU95m89nHzUAlkrTHWwccmuQQmmBz\nEvArgyskOQI4F1hRVTcPzN8f+H5V3ZnkAOCZNDcg0Jh2toebR8ybiO2hpJlkwJEkdVpV7UhyGnAx\nsABYXVUbkqwC1lfVGuAvgf2AjyYBuLGqjgeeBJyb5B6aUQ1nD919TZLUMQYcSVLnVdVaYO3QvDMG\n3h89wXafBZ46s9VJkmaT1+BIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0DjiRJ\nkqTeMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0D\njiRJkqTeGCvgJFmR5PokG5OcPmL5K5Ncm+TqJP+Y5OCBZack+Ur7OmU6i5ckSZKkQZMGnCQLgHOA\n5wGHAScnOWxotS8Ay6rqacCFwBvbbR8JnAkcCSwHzkyy//SVL0mSJEk/Mk4PznJgY1Vtqqq7gPOB\nEwZXqKrLqur77eQVwOL2/XOBS6pqe1V9G7gEWDE9pUuSJEnSfY0TcA4EtgxMb23nTeSlwKd2c1tJ\nkiRJ2m0Lp3NnSV4CLAOeM8XtVgIrAZYsWTKdJUmSJEnag4zTg7MNOGhgenE77z6SHA28Bji+qu6c\nyrZVdV5VLauqZYsWLRq3dkmSJEm6j3ECzjrg0CSHJNkbOAlYM7hCkiOAc2nCzc0Diy4Gjk2yf3tz\ngWPbeZIkSZI07SYdolZVO5KcRhNMFgCrq2pDklXA+qpaA/wlsB/w0SQAN1bV8VW1PclraUISwKqq\n2j4jn0SSJEnSHm+sa3Cqai2wdmjeGQPvj97FtquB1btboCRJkiSNa6wHfUqSJElSFxhwJEmSJPWG\nAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmS\nJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSbxhw\nJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeS1HlJViS5PsnGJKePWP7KJNcmuTrJPyY5eGDZKUm+0r5O\nmd3KJUnTzYAjSeq0JAuAc4DnAYcBJyc5bGi1LwDLquppwIXAG9ttHwmcCRwJLAfOTLL/bNUuSZp+\nBhxJUtctBzZW1aaqugs4HzhhcIWquqyqvt9OXgEsbt8/F7ikqrZX1beBS4AVs1S3JGkGGHAkSV13\nILBlYHprO28iLwU+tZvbSpLmuYVzXYAkSbMlyUuAZcBzdmPblcBKgCVLlkxzZZKk6WIPjiSp67YB\nBw1ML27n3UeSo4HXAMdX1Z1T2Ragqs6rqmVVtWzRokXTUrgkafoZcCRJXbcOODTJIUn2Bk4C1gyu\nkOQI4FyacHPzwKKLgWOT7N/eXODYdp4kqaMcoiZJ6rSq2pHkNJpgsgBYXVUbkqwC1lfVGuAvgf2A\njyYBuLGqjq+q7UleSxOSAFZV1fY5+BiSpGliwJEkdV5VrQXWDs07Y+D90bvYdjWweuaqkyTNJoeo\nSZIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk\n3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44k\nSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeqNsQJOkhVJrk+yMcnpI5Y/\nO8nnk+xI8oKhZXcnuap9rZmuwiVJkiRp2MLJVkiyADgHOAbYCqxLsqaqrh1Y7UbgVODVI3ZxR1Ud\nPg21SpIkSdIuTRpwgOXAxqraBJDkfOAE4N6AU1Wb22X3zECNkiRJkjSWcYaoHQhsGZje2s4b175J\n1ie5Isnzp1SdJEmSJE3BOD04D9TBVbUtyeOAS5NcU1U3DK6QZCWwEmDJkiWzUJIkSZKkPhqnB2cb\ncNDA9OJ23liqalv75ybgcuCIEeucV1XLqmrZokWLxt21JEmSJN3HOAFnHXBokkOS7A2cBIx1N7Qk\n+yfZp31/APBMBq7dkSRJkqTpNGnAqaodwGnAxcB1wAVVtSHJqiTHAyR5RpKtwInAuUk2tJs/CVif\n5IvAZcDZQ3dfkyRJkqRpM9Y1OFW1Flg7NO+MgffraIauDW/3WeCpD7BGSZIkSRrLWA/6lCRJkqQu\nMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6g0DjiRJkqTeMOBIkiRJ6o2xHvQp\nSZL6b+npF811CZL0gNmDI0mSJKk3DDiSJEmSesOAI0mSJKk3DDiSJEmSesObDPTQ7lwkuvns42ag\nEkmaHUlWAG8DFgDvqqqzh5Y/G3gr8DTgpKq6cGDZ3cA17eSNVXX87FQtSZoJBhxJUqclWQCcAxwD\nbAXWJVlTVdcOrHYjcCrw6hG7uKOqDp/xQiVJs8KAI0nquuXAxqraBJDkfOAE4N6AU1Wb22X3zEWB\nkqTZ4zU4kqSuOxDYMjC9tZ03rn2TrE9yRZLnT29pkqTZZg+OJGlPd3BVbUvyOODSJNdU1Q3DKyVZ\nCawEWLJkyWzXKEkakz04kqSu2wYcNDC9uJ03lqra1v65CbgcOGKC9c6rqmVVtWzRokW7X60kaUYZ\ncCRJXbcOODTJIUn2Bk4C1oyzYZL9k+zTvj8AeCYD1+5IkrrHgCNJ6rSq2gGcBlwMXAdcUFUbkqxK\ncjxAkmck2QqcCJybZEO7+ZOA9Um+CFwGnD109zVJUsd4DY4kqfOqai2wdmjeGQPv19EMXRve7rPA\nU2e8QEnSrLEHR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9YYBR5Ik\nSVJvGHAkSZIk9YYP+pxFS0+/aMrbbD77uBmoRJIkSeone3AkSZIk9YYBR5IkSVJvGHAkSZIk9YYB\nR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk\n9YYBR5IkSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9cbCuS5AkiTNjKWnXzTXJUjSrLMHR5IkSVJv\nGHAkSZIk9YYBR5IkSVJvGHAkSZIk9cZYASfJiiTXJ9mY5PQRy5+d5PNJdiR5wdCyU5J8pX2dMl2F\nS5IkSdKwSQNOkgXAOcDzgMOAk5McNrTajcCpwIeGtn0kcCZwJLAcODPJ/g+8bEmSJEm6v3F6cJYD\nG6tqU1XdBZwPnDC4QlVtrqqrgXuGtn0ucElVba+qbwOXACumoW5JkiRJup9xAs6BwJaB6a3tvHE8\nkG0lSZIkaUrmxU0GkqxMsj7J+ltuuWWuy5EkSZLUUQvHWGcbcNDA9OJ23ji2AUcNbXv58EpVdR5w\nHsCyZctqzH3vEXY+hXrziHmSJEmS7mucHpx1wKFJDkmyN3ASsGbM/V8MHJtk//bmAse28yRJkiRp\n2k0acKpqB3AaTTC5DrigqjYkWZXkeIAkz0iyFTgRODfJhnbb7cBraULSOmBVO0+SJEmSpt04Q9So\nqrXA2qF5Zwy8X0cz/GzUtquB1Q+gRkmSJEkay7y4yYAkSZIkTQcDjiRJkqTeMOBIkiRJ6g0DjiRJ\nkqTeMOBIkiRJ6g0DjiSp85KsSHJ9ko1JTh+x/NlJPp9kR5IXDC07JclX2tcps1e1JGkmGHAkSZ2W\nZAFwDvA84DDg5CSHDa12I3Aq8KGhbR8JnAkcCSwHzmwfTC1J6igDjiSp65YDG6tqU1XdBZwPnDC4\nQlVtrqqrgXuGtn0ucElVba+qbwOXACtmo2hJ0sww4EiSuu5AYMvA9NZ23kxvK0mahww4kiSNIcnK\nJOuTrL/lllvmuhxJ0gQMOJKkrtsGHDQwvbidN63bVtV5VbWsqpYtWrRotwqVJM08A44kqevWAYcm\nOSTJ3sBJwJoxt70YODbJ/u3NBY5t50mSOsqAI0nqtKraAZxGE0yuAy6oqg1JViU5HiDJM5JsBU4E\nzk2yod12O/BampC0DljVzpMkddTCuS5AkqQHqqrWAmuH5p0x8H4dzfCzUduuBlbPaIGSpFljD44k\nSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoN\nA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIk\nSeoNA44kSZKk3jDgSJIkSeqNhXNdgCRJ2rMsPf2iKW+z+ezjZqASSX1kD44kSZKk3jDgSJIkSeoN\nA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIk\nSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeoNA44kSZKk3jDgSJIkSeqNhXNdgCRJmtzS\n0y+a6xIkqRPG6sFJsiLJ9Uk2Jjl9xPJ9knykXf65JEvb+UuT3JHkqvb1zuktX5IkSZJ+ZNIenCQL\ngHOAY4CtwLoka6rq2oHVXgp8u6oen+Qk4A3Ai9plN1TV4dNctyRJkiTdzzhD1JYDG6tqE0CS84ET\ngMGAcwJwVvv+QuDtSTKNdc44u/4lSZKk7htniNqBwJaB6a3tvJHrVNUO4DvAo9plhyT5QpJ/SvKz\nD7BeSZIkSZrQTN9k4BvAkqq6NcnTgY8neXJVfXdwpSQrgZUAS5YsmeGSJEmSJPXVOD0424CDBqYX\nt/NGrpNkIfAI4NaqurOqbgWoqiuBG4AnDP+AqjqvqpZV1bJFixZN/VNIkiRJEuMFnHXAoUkOSbI3\ncBKwZmidNcAp7fsXAJdWVSVZ1N6kgCSPAw4FNk1P6ZIkSZJ0X5MOUauqHUlOAy4GFgCrq2pDklXA\n+qpaA7wb+ECSjcB2mhAE8GxgVZIfAvcAL6+q7TPxQSRJkiRprGtwqmotsHZo3hkD738AnDhiu48B\nH3uANUqStEtJVgBvo/ki7l1VdfbQ8n2A9wNPB24FXlRVm9vntl0HXN+uekVVvXy26pYkTb+ZvsmA\nJEkzyue1SZIGjXMNjiRJ89m9z2urqruAnc9rG3QC8L72/YXAf+/a89okSeMx4EiSus7ntUmS7uUQ\nNUnSnmys57WBz2yTpK6wB0eS1HUz/ry2drnPbJOkDjDgSJK6zue1SZLu5RA1SVKn+bw2SdIgA44k\nqfN8XpskaSeHqEmSJEnqDQOOJEmSpN4w4EiSJEnqDa/BEQBLT79oyttsPvu4GahEkqT7s52SNC57\ncCRJkiT1hgFHkiRJUm8YcCRJkiT1hgFHkiRJUm8YcCRJkiT1hgFHkiRJUm8YcCRJkiT1hgFHkiRJ\nUm8YcCRJkiT1xsK5LkCSJGm+WHr6RVPeZvPZx81AJZJ2lz04kiRJknrDgCNJkiSpNww4kiRJknrD\na3AkSVIv7c71NJK6z4CjWeXFm5IkSZpJDlGTJEmS1BsGHEmSJEm94RA17TbHNkuSJGm+sQdHkiRJ\nUm8YcCRJkiT1hgFHkiRJUm8YcCRJkiT1hjcZkCRplnmTFkmaOfbgSJIkSeoNA44kSZKk3jDgSJIk\nSeoNA44kSZKk3jDgSJIkSeqNXt5FzbvTSJIkSXsme3AkSZIk9YYBR5IkSVJvGHAkSZIk9YYBR5Ik\nSVJvGHAkSZIk9YYBR5IkSVJvGHAkSZIk9UYvn4Ojftmd5xptPvu4GahEkiRJ850BR5Ik6QHwizhp\nfjHgqJdsbCRJkvZMYwWcJCuAtwELgHdV1dlDy/cB3g88HbgVeFFVbW6X/THwUuBu4Heq6uJpq16a\nRoYiqbtsp6TRbNu0J5o04CRZAJwDHANsBdYlWVNV1w6s9lLg21X1+CQnAW8AXpTkMOAk4MnAY4F/\nSPKEqrp7uj+INBem2nDYaEjTz3ZKXWTwkGbOOD04y4GNVbUJIMn5wAnAYMNxAnBW+/5C4O1J0s4/\nv6ruBL6aZGO7v3+bnvKlbrFBk2aE7ZT2CLvThkh7onECzoHAloHprcCRE61TVTuSfAd4VDv/iqFt\nD9ztaiWNZbYaQcOX5gnbKWkazcboBNspzaR5cZOBJCuBle3k7Umun2DVA4BvzU5VM2K368/gxBt+\ncVqKmaI99tjPtbxh/taeN0y6yrytfUxdrn86aj94OgrpizHaqi7/fRllSp9nHrRTk9mjfz8zaYy2\nYFzT/pmmsbbdMW9+R9NkPn6eke3UOAFnG3DQwPTidt6odbYmWQg8guYiznG2parOA86brJAk66tq\n2Rg1z0tdrr/LtUO367f2udPl+rtc+26Y8XYKJm+r+nbM/TzzW98+D/TvM/l55s6DxlhnHXBokkOS\n7E1zMeaaoXXWAKe0718AXFpV1c4/Kck+SQ4BDgX+fXpKlyQJsJ2SJA2YtAenHat8GnAxze03V1fV\nhiSrgPVVtQZ4N/CB9uLM7TSNC+16F9Bc6LkD+C3vTCNJmk62U5KkQWNdg1NVa4G1Q/POGHj/A+DE\nCbZ9PfD6B1DjoEmHsc1zXa6/y7VDt+u39rnT5fq7XPuUzZN2qm/H3M8zv/Xt80D/PpOfZ46k6aGX\nJEmSpO4b5xocSZIkSeqEzgScJCuSXJ9kY5LT57qeqUiyOck1Sa5Ksn6u65lMktVJbk7ypYF5j0xy\nSZKvtH/uP5c1TmSC2s9KspaxOtEAAASVSURBVK09/lcl+YW5rHEiSQ5KclmSa5NsSPK77fyuHPuJ\n6p/3xz/Jvkn+PckX29r/vJ1/SJLPteedj7QXsM87u6j/vUm+OnDsD5/rWvuqy23UKF1rt4Z1uR0b\npctt2yhdb++Gdbn9G6XrbSJ0ZIhakgXAfwDH0DyEbR1wclVdu8sN54kkm4FlVTXf7h0+UpJnA7cD\n76+qp7Tz3ghsr6qz28Z7/6r6o7msc5QJaj8LuL2q3jSXtU0myWOAx1TV55M8DLgSeD5wKt049hPV\n/0Lm+fFPEuChVXV7kr2AzwC/C7wS+LuqOj/JO4EvVtU75rLWUXZR/8uBT1bVhXNaYM91vY0apWvt\n1rAut2OjdLltG6Xr7d2wLrd/o3S9TYTu9OAsBzZW1aaqugs4Hzhhjmvqrar6Z5q7DA06AXhf+/59\nNP9w550Jau+EqvpGVX2+ff894DqaJ6p35dhPVP+8V43b28m92lcBPw/sDAfz+dhPVL9mh23UPNPl\ndmyULrdto3S9vRvW5fZvlK63idCdgHMgsGVgeivd+otTwKeTXJnmSdhd9Oiq+kb7/pvAo+eymN1w\nWpKr227+ed/lnWQpcATwOTp47Ifqhw4c/yQLklwF3AxcAtwA3FZVO9pV5vV5Z7j+qtp57F/fHvu3\nJNlnDkvss663UaP0od0a1rlz6Rjm/bl1Ml1v74Z1sf0bpettYlcCTtc9q6r+G/A84LfarubOah+O\n16Vvh98B/DhwOPAN4K/mtpxdS7If8DHg96rqu4PLunDsR9TfieNfVXdX1eE0T7JfDjxxjkuakuH6\nkzwF+GOaz/EM4JHAvB/qoXmjV+3WsC6cS8fQiXPrrnS9vRvW1fZvlK63iV0JONuAgwamF7fzOqGq\ntrV/3gz8X5q/KF1zUzvGdOdY05vnuJ6xVdVN7T/Ue4C/ZR4f/3as68eA/1NVf9fO7syxH1V/l44/\nQFXdBlwG/DTwY0l2Pi+sE+edgfpXtMMmqqruBN7DPD/2HdbpNmqUnrRbwzpzLh1H186tw7re3g3r\nQ/s3SlfbxK4EnHXAoe3dG/ameQL1mjmuaSxJHtpecEaShwLHAl/a9Vbz0hrglPb9KcDfz2EtU7Lz\nZNn6Zebp8W8v6ns3cF1VvXlgUSeO/UT1d+H4J1mU5Mfa9w+muVj8OpqT+gva1ebzsR9V/5cH/qMQ\nmrHS8+7Y90Rn26hRetRuDevEuXRcXTi3TqTr7d2wLrd/o3S9TYSO3EUNIM2t9d4KLABWV/Pk6Xkv\nyeNovv0CWAh8aL7XnuTDwFHAAcBNwJnAx4ELgCXA14AXVtW8u+BxgtqPoukeLmAz8BsDY3znjSTP\nAv4FuAa4p539JzTjeLtw7Ceq/2Tm+fFP8jSaCyYX0Hzxc0FVrWr//Z5PM7zrC8BL2t6QeWUX9V8K\nLAICXAW8fODCUU2jrrZRo3Sx3RrW5XZslC63baN0vb0b1uX2b5Sut4nQoYAjSZIkSZPpyhA1SZIk\nSZqUAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSbxhwJEmSJPWGAUeSJElSb/x/\n7Fv0A0vUBVUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1008x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cgAb-IzEiP8u",
        "colab_type": "text"
      },
      "source": [
        "## n-grams\n",
        "VizSeq let you view statistics on n-grams in your dataset using `vizseq.view_n_grams`\n",
        "\n",
        "### Source-side n-grams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KxP_FLTlfjnj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c8dabe33-8ef8-41e1-9210-9784fa636d38"
      },
      "source": [
        "vizseq.view_n_grams(sources)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<!DOCTYPE html>\n",
              "<html lang=\"en\">\n",
              "<head>\n",
              "    <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css\"\n",
              "          integrity=\"sha384-GJzZqFGwb1QTTN6wy59ffF1BuGJpLSa9DkKMp0DgiMDm4iYMj70gZWKYbI706tWS\" crossorigin=\"anonymous\">\n",
              "    <script src=\"https://kit.fontawesome.com/3c563f9647.js\"></script>\n",
              "\n",
              "    <style>\n",
              "        body {\n",
              "            padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */\n",
              "        }\n",
              "    </style>\n",
              "</head>\n",
              "\n",
              "<body>\n",
              "\n",
              "\n",
              "<div class=\"container\">\n",
              "    <div class=\"row\">\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">1-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>je</td><td>42270</td></tr>\n",
              "                \n",
              "                <tr><td>de</td><td>36494</td></tr>\n",
              "                \n",
              "                <tr><td>pas</td><td>26556</td></tr>\n",
              "                \n",
              "                <tr><td>que</td><td>25699</td></tr>\n",
              "                \n",
              "                <tr><td>à</td><td>21778</td></tr>\n",
              "                \n",
              "                <tr><td>ne</td><td>21423</td></tr>\n",
              "                \n",
              "                <tr><td>le</td><td>20281</td></tr>\n",
              "                \n",
              "                <tr><td>la</td><td>19971</td></tr>\n",
              "                \n",
              "                <tr><td>vous</td><td>18058</td></tr>\n",
              "                \n",
              "                <tr><td>?</td><td>17974</td></tr>\n",
              "                \n",
              "                <tr><td>il</td><td>15813</td></tr>\n",
              "                \n",
              "                <tr><td>tom</td><td>13249</td></tr>\n",
              "                \n",
              "                <tr><td>est</td><td>12978</td></tr>\n",
              "                \n",
              "                <tr><td>un</td><td>12535</td></tr>\n",
              "                \n",
              "                <tr><td>ce</td><td>12206</td></tr>\n",
              "                \n",
              "                <tr><td>tu</td><td>11865</td></tr>\n",
              "                \n",
              "                <tr><td>a</td><td>11582</td></tr>\n",
              "                \n",
              "                <tr><td>nous</td><td>10947</td></tr>\n",
              "                \n",
              "                <tr><td>en</td><td>9319</td></tr>\n",
              "                \n",
              "                <tr><td>une</td><td>9152</td></tr>\n",
              "                \n",
              "                <tr><td>les</td><td>9144</td></tr>\n",
              "                \n",
              "                <tr><td>j&#39;ai</td><td>8424</td></tr>\n",
              "                \n",
              "                <tr><td>me</td><td>7708</td></tr>\n",
              "                \n",
              "                <tr><td>suis</td><td>7564</td></tr>\n",
              "                \n",
              "                <tr><td>pour</td><td>7530</td></tr>\n",
              "                \n",
              "                <tr><td>c&#39;est</td><td>6254</td></tr>\n",
              "                \n",
              "                <tr><td>elle</td><td>5922</td></tr>\n",
              "                \n",
              "                <tr><td>dans</td><td>5151</td></tr>\n",
              "                \n",
              "                <tr><td>faire</td><td>5098</td></tr>\n",
              "                \n",
              "                <tr><td>!</td><td>4860</td></tr>\n",
              "                \n",
              "                <tr><td>des</td><td>4827</td></tr>\n",
              "                \n",
              "                <tr><td>plus</td><td>4687</td></tr>\n",
              "                \n",
              "                <tr><td>qui</td><td>4678</td></tr>\n",
              "                \n",
              "                <tr><td>te</td><td>4424</td></tr>\n",
              "                \n",
              "                <tr><td>tout</td><td>4163</td></tr>\n",
              "                \n",
              "                <tr><td>avec</td><td>4063</td></tr>\n",
              "                \n",
              "                <tr><td>mon</td><td>4011</td></tr>\n",
              "                \n",
              "                <tr><td>du</td><td>4009</td></tr>\n",
              "                \n",
              "                <tr><td>se</td><td>3900</td></tr>\n",
              "                \n",
              "                <tr><td>au</td><td>3832</td></tr>\n",
              "                \n",
              "                <tr><td>si</td><td>3819</td></tr>\n",
              "                \n",
              "                <tr><td>veux</td><td>3740</td></tr>\n",
              "                \n",
              "                <tr><td>ça</td><td>3725</td></tr>\n",
              "                \n",
              "                <tr><td>fait</td><td>3508</td></tr>\n",
              "                \n",
              "                <tr><td>et</td><td>3404</td></tr>\n",
              "                \n",
              "                <tr><td>qu&#39;il</td><td>3325</td></tr>\n",
              "                \n",
              "                <tr><td>cette</td><td>3221</td></tr>\n",
              "                \n",
              "                <tr><td>y</td><td>3211</td></tr>\n",
              "                \n",
              "                <tr><td>sont</td><td>3107</td></tr>\n",
              "                \n",
              "                <tr><td>son</td><td>3084</td></tr>\n",
              "                \n",
              "                <tr><td>très</td><td>2966</td></tr>\n",
              "                \n",
              "                <tr><td>votre</td><td>2855</td></tr>\n",
              "                \n",
              "                <tr><td>être</td><td>2769</td></tr>\n",
              "                \n",
              "                <tr><td>pourquoi</td><td>2699</td></tr>\n",
              "                \n",
              "                <tr><td>sur</td><td>2685</td></tr>\n",
              "                \n",
              "                <tr><td>ma</td><td>2665</td></tr>\n",
              "                \n",
              "                <tr><td>pense</td><td>2598</td></tr>\n",
              "                \n",
              "                <tr><td>été</td><td>2546</td></tr>\n",
              "                \n",
              "                <tr><td>était</td><td>2511</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;est</td><td>2470</td></tr>\n",
              "                \n",
              "                <tr><td>peux</td><td>2442</td></tr>\n",
              "                \n",
              "                <tr><td>ils</td><td>2389</td></tr>\n",
              "                \n",
              "                <tr><td>dit</td><td>2362</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;ai</td><td>2338</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">2-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>je ne</td><td>8098</td></tr>\n",
              "                \n",
              "                <tr><td>je suis</td><td>5152</td></tr>\n",
              "                \n",
              "                <tr><td>que tu</td><td>4502</td></tr>\n",
              "                \n",
              "                <tr><td>que je</td><td>3934</td></tr>\n",
              "                \n",
              "                <tr><td>que vous</td><td>3892</td></tr>\n",
              "                \n",
              "                <tr><td>ce que</td><td>3116</td></tr>\n",
              "                \n",
              "                <tr><td>à la</td><td>2672</td></tr>\n",
              "                \n",
              "                <tr><td>de la</td><td>2525</td></tr>\n",
              "                \n",
              "                <tr><td>je veux</td><td>2476</td></tr>\n",
              "                \n",
              "                <tr><td>tom a</td><td>2342</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;ai</td><td>2336</td></tr>\n",
              "                \n",
              "                <tr><td>je me</td><td>2269</td></tr>\n",
              "                \n",
              "                <tr><td>il est</td><td>2130</td></tr>\n",
              "                \n",
              "                <tr><td>il a</td><td>2095</td></tr>\n",
              "                \n",
              "                <tr><td>je pense</td><td>1948</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;est pas</td><td>1942</td></tr>\n",
              "                \n",
              "                <tr><td>vous êtes</td><td>1848</td></tr>\n",
              "                \n",
              "                <tr><td>tom est</td><td>1809</td></tr>\n",
              "                \n",
              "                <tr><td>que tom</td><td>1595</td></tr>\n",
              "                \n",
              "                <tr><td>nous avons</td><td>1499</td></tr>\n",
              "                \n",
              "                <tr><td>tu es</td><td>1498</td></tr>\n",
              "                \n",
              "                <tr><td>pense que</td><td>1486</td></tr>\n",
              "                \n",
              "                <tr><td>pas à</td><td>1429</td></tr>\n",
              "                \n",
              "                <tr><td>pas de</td><td>1420</td></tr>\n",
              "                \n",
              "                <tr><td>je sais</td><td>1391</td></tr>\n",
              "                \n",
              "                <tr><td>je vous</td><td>1386</td></tr>\n",
              "                \n",
              "                <tr><td>ne peux</td><td>1382</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;ai pas</td><td>1380</td></tr>\n",
              "                \n",
              "                <tr><td>quelque chose</td><td>1375</td></tr>\n",
              "                \n",
              "                <tr><td>que nous</td><td>1351</td></tr>\n",
              "                \n",
              "                <tr><td>tu as</td><td>1347</td></tr>\n",
              "                \n",
              "                <tr><td>il y</td><td>1329</td></tr>\n",
              "                \n",
              "                <tr><td>nous sommes</td><td>1283</td></tr>\n",
              "                \n",
              "                <tr><td>y a</td><td>1283</td></tr>\n",
              "                \n",
              "                <tr><td>est-ce que</td><td>1273</td></tr>\n",
              "                \n",
              "                <tr><td>tout le</td><td>1259</td></tr>\n",
              "                \n",
              "                <tr><td>ne suis</td><td>1224</td></tr>\n",
              "                \n",
              "                <tr><td>que ce</td><td>1212</td></tr>\n",
              "                \n",
              "                <tr><td>tu ne</td><td>1194</td></tr>\n",
              "                \n",
              "                <tr><td>suis pas</td><td>1191</td></tr>\n",
              "                \n",
              "                <tr><td>vous avez</td><td>1114</td></tr>\n",
              "                \n",
              "                <tr><td>en train</td><td>1070</td></tr>\n",
              "                \n",
              "                <tr><td>pas que</td><td>1066</td></tr>\n",
              "                \n",
              "                <tr><td>peux pas</td><td>1053</td></tr>\n",
              "                \n",
              "                <tr><td>ce n&#39;est</td><td>1032</td></tr>\n",
              "                \n",
              "                <tr><td>dans la</td><td>1026</td></tr>\n",
              "                \n",
              "                <tr><td>je vais</td><td>1007</td></tr>\n",
              "                \n",
              "                <tr><td>ne pas</td><td>1002</td></tr>\n",
              "                \n",
              "                <tr><td>vous ne</td><td>1001</td></tr>\n",
              "                \n",
              "                <tr><td>ne me</td><td>998</td></tr>\n",
              "                \n",
              "                <tr><td>sais que</td><td>998</td></tr>\n",
              "                \n",
              "                <tr><td>elle a</td><td>987</td></tr>\n",
              "                \n",
              "                <tr><td>tom ne</td><td>981</td></tr>\n",
              "                \n",
              "                <tr><td>un peu</td><td>980</td></tr>\n",
              "                \n",
              "                <tr><td>dans le</td><td>958</td></tr>\n",
              "                \n",
              "                <tr><td>le monde</td><td>953</td></tr>\n",
              "                \n",
              "                <tr><td>de faire</td><td>938</td></tr>\n",
              "                \n",
              "                <tr><td>besoin de</td><td>936</td></tr>\n",
              "                \n",
              "                <tr><td>train de</td><td>931</td></tr>\n",
              "                \n",
              "                <tr><td>ce qui</td><td>906</td></tr>\n",
              "                \n",
              "                <tr><td>a été</td><td>896</td></tr>\n",
              "                \n",
              "                <tr><td>ne veux</td><td>883</td></tr>\n",
              "                \n",
              "                <tr><td>tous les</td><td>882</td></tr>\n",
              "                \n",
              "                <tr><td>je te</td><td>858</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">3-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>je pense que</td><td>1413</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;ai pas</td><td>1379</td></tr>\n",
              "                \n",
              "                <tr><td>je ne suis</td><td>1218</td></tr>\n",
              "                \n",
              "                <tr><td>il y a</td><td>1127</td></tr>\n",
              "                \n",
              "                <tr><td>je ne peux</td><td>1088</td></tr>\n",
              "                \n",
              "                <tr><td>ne suis pas</td><td>1068</td></tr>\n",
              "                \n",
              "                <tr><td>ne peux pas</td><td>1052</td></tr>\n",
              "                \n",
              "                <tr><td>je sais que</td><td>946</td></tr>\n",
              "                \n",
              "                <tr><td>en train de</td><td>915</td></tr>\n",
              "                \n",
              "                <tr><td>ce n&#39;est pas</td><td>883</td></tr>\n",
              "                \n",
              "                <tr><td>tout le monde</td><td>869</td></tr>\n",
              "                \n",
              "                <tr><td>je ne veux</td><td>790</td></tr>\n",
              "                \n",
              "                <tr><td>ce que je</td><td>768</td></tr>\n",
              "                \n",
              "                <tr><td>je me suis</td><td>762</td></tr>\n",
              "                \n",
              "                <tr><td>de ne pas</td><td>678</td></tr>\n",
              "                \n",
              "                <tr><td>ce que tu</td><td>678</td></tr>\n",
              "                \n",
              "                <tr><td>ne veux pas</td><td>661</td></tr>\n",
              "                \n",
              "                <tr><td>je ne sais</td><td>655</td></tr>\n",
              "                \n",
              "                <tr><td>je pensais que</td><td>627</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;arrive pas à</td><td>573</td></tr>\n",
              "                \n",
              "                <tr><td>que ce soit</td><td>572</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;arrive pas</td><td>565</td></tr>\n",
              "                \n",
              "                <tr><td>je veux que</td><td>565</td></tr>\n",
              "                \n",
              "                <tr><td>ce que vous</td><td>561</td></tr>\n",
              "                \n",
              "                <tr><td>ne sais pas</td><td>557</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;est-ce pas ?</td><td>511</td></tr>\n",
              "                \n",
              "                <tr><td>que tu as</td><td>508</td></tr>\n",
              "                \n",
              "                <tr><td>tout ce que</td><td>507</td></tr>\n",
              "                \n",
              "                <tr><td>il n&#39;y a</td><td>489</td></tr>\n",
              "                \n",
              "                <tr><td>vous n&#39;êtes pas</td><td>488</td></tr>\n",
              "                \n",
              "                <tr><td>ne pense pas</td><td>475</td></tr>\n",
              "                \n",
              "                <tr><td>je ne pense</td><td>469</td></tr>\n",
              "                \n",
              "                <tr><td>est-ce que tu</td><td>455</td></tr>\n",
              "                \n",
              "                <tr><td>quoi que ce</td><td>394</td></tr>\n",
              "                \n",
              "                <tr><td>que vous avez</td><td>389</td></tr>\n",
              "                \n",
              "                <tr><td>que je ne</td><td>376</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;aime pas</td><td>349</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;ai jamais</td><td>348</td></tr>\n",
              "                \n",
              "                <tr><td>que tom est</td><td>343</td></tr>\n",
              "                \n",
              "                <tr><td>sais que tom</td><td>343</td></tr>\n",
              "                \n",
              "                <tr><td>combien de temps</td><td>339</td></tr>\n",
              "                \n",
              "                <tr><td>tu n&#39;es pas</td><td>338</td></tr>\n",
              "                \n",
              "                <tr><td>pas à croire</td><td>337</td></tr>\n",
              "                \n",
              "                <tr><td>je vous ai</td><td>327</td></tr>\n",
              "                \n",
              "                <tr><td>je me sens</td><td>324</td></tr>\n",
              "                \n",
              "                <tr><td>que je suis</td><td>324</td></tr>\n",
              "                \n",
              "                <tr><td>je ne me</td><td>318</td></tr>\n",
              "                \n",
              "                <tr><td>à croire que</td><td>302</td></tr>\n",
              "                \n",
              "                <tr><td>pense pas que</td><td>299</td></tr>\n",
              "                \n",
              "                <tr><td>il me faut</td><td>298</td></tr>\n",
              "                \n",
              "                <tr><td>tom a dit</td><td>290</td></tr>\n",
              "                \n",
              "                <tr><td>je pense qu&#39;il</td><td>290</td></tr>\n",
              "                \n",
              "                <tr><td>tu ne peux</td><td>288</td></tr>\n",
              "                \n",
              "                <tr><td>je me demande</td><td>286</td></tr>\n",
              "                \n",
              "                <tr><td>à la maison.</td><td>284</td></tr>\n",
              "                \n",
              "                <tr><td>que vous êtes</td><td>283</td></tr>\n",
              "                \n",
              "                <tr><td>que tu ne</td><td>280</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;y a pas</td><td>277</td></tr>\n",
              "                \n",
              "                <tr><td>que tu es</td><td>262</td></tr>\n",
              "                \n",
              "                <tr><td>je ne vous</td><td>262</td></tr>\n",
              "                \n",
              "                <tr><td>pas ce que</td><td>255</td></tr>\n",
              "                \n",
              "                <tr><td>vous ne pouvez</td><td>249</td></tr>\n",
              "                \n",
              "                <tr><td>à ce que</td><td>249</td></tr>\n",
              "                \n",
              "                <tr><td>veux que tu</td><td>248</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">4-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>je ne suis pas</td><td>1062</td></tr>\n",
              "                \n",
              "                <tr><td>je ne peux pas</td><td>803</td></tr>\n",
              "                \n",
              "                <tr><td>je ne veux pas</td><td>584</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;arrive pas à</td><td>555</td></tr>\n",
              "                \n",
              "                <tr><td>je ne sais pas</td><td>530</td></tr>\n",
              "                \n",
              "                <tr><td>je ne pense pas</td><td>455</td></tr>\n",
              "                \n",
              "                <tr><td>je sais que tom</td><td>341</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;arrive pas à croire</td><td>312</td></tr>\n",
              "                \n",
              "                <tr><td>pas à croire que</td><td>300</td></tr>\n",
              "                \n",
              "                <tr><td>ne pense pas que</td><td>298</td></tr>\n",
              "                \n",
              "                <tr><td>il n&#39;y a pas</td><td>267</td></tr>\n",
              "                \n",
              "                <tr><td>je veux que tu</td><td>247</td></tr>\n",
              "                \n",
              "                <tr><td>quoi que ce soit</td><td>246</td></tr>\n",
              "                \n",
              "                <tr><td>tu ne peux pas</td><td>244</td></tr>\n",
              "                \n",
              "                <tr><td>je veux que vous</td><td>230</td></tr>\n",
              "                \n",
              "                <tr><td>je pense que nous</td><td>230</td></tr>\n",
              "                \n",
              "                <tr><td>je pense que vous</td><td>220</td></tr>\n",
              "                \n",
              "                <tr><td>je pense que tu</td><td>215</td></tr>\n",
              "                \n",
              "                <tr><td>vous ne pouvez pas</td><td>210</td></tr>\n",
              "                \n",
              "                <tr><td>je pense que tom</td><td>210</td></tr>\n",
              "                \n",
              "                <tr><td>je pensais que vous</td><td>205</td></tr>\n",
              "                \n",
              "                <tr><td>je sais que tu</td><td>196</td></tr>\n",
              "                \n",
              "                <tr><td>sais que tom est</td><td>191</td></tr>\n",
              "                \n",
              "                <tr><td>je pensais que tu</td><td>185</td></tr>\n",
              "                \n",
              "                <tr><td>ce que tu as</td><td>176</td></tr>\n",
              "                \n",
              "                <tr><td>nous ne sommes pas</td><td>175</td></tr>\n",
              "                \n",
              "                <tr><td>est en train de</td><td>174</td></tr>\n",
              "                \n",
              "                <tr><td>il y a un</td><td>171</td></tr>\n",
              "                \n",
              "                <tr><td>je ne savais pas</td><td>167</td></tr>\n",
              "                \n",
              "                <tr><td>je pense qu&#39;il est</td><td>165</td></tr>\n",
              "                \n",
              "                <tr><td>je ne vais pas</td><td>160</td></tr>\n",
              "                \n",
              "                <tr><td>je sais que vous</td><td>158</td></tr>\n",
              "                \n",
              "                <tr><td>je pense que c&#39;est</td><td>151</td></tr>\n",
              "                \n",
              "                <tr><td>ce que vous avez</td><td>151</td></tr>\n",
              "                \n",
              "                <tr><td>tout ce que je</td><td>149</td></tr>\n",
              "                \n",
              "                <tr><td>je ne voulais pas</td><td>147</td></tr>\n",
              "                \n",
              "                <tr><td>qui que ce soit</td><td>146</td></tr>\n",
              "                \n",
              "                <tr><td>il y a une</td><td>145</td></tr>\n",
              "                \n",
              "                <tr><td>tom a dit qu&#39;il</td><td>145</td></tr>\n",
              "                \n",
              "                <tr><td>n&#39;y a pas de</td><td>138</td></tr>\n",
              "                \n",
              "                <tr><td>ne veux pas que</td><td>136</td></tr>\n",
              "                \n",
              "                <tr><td>nous ne pouvons pas</td><td>129</td></tr>\n",
              "                \n",
              "                <tr><td>tout ce que tu</td><td>126</td></tr>\n",
              "                \n",
              "                <tr><td>je ne vous ai</td><td>125</td></tr>\n",
              "                \n",
              "                <tr><td>je n&#39;ai pas de</td><td>122</td></tr>\n",
              "                \n",
              "                <tr><td>quoi que ce soit.</td><td>122</td></tr>\n",
              "                \n",
              "                <tr><td>je pense que je</td><td>121</td></tr>\n",
              "                \n",
              "                <tr><td>ne savais pas que</td><td>117</td></tr>\n",
              "                \n",
              "                <tr><td>je suis en train</td><td>115</td></tr>\n",
              "                \n",
              "                <tr><td>je ne comprends pas</td><td>114</td></tr>\n",
              "                \n",
              "                <tr><td>pense qu&#39;il est temps</td><td>113</td></tr>\n",
              "                \n",
              "                <tr><td>je suis sûr que</td><td>110</td></tr>\n",
              "                \n",
              "                <tr><td>tout ce que vous</td><td>109</td></tr>\n",
              "                \n",
              "                <tr><td>je me demande si</td><td>102</td></tr>\n",
              "                \n",
              "                <tr><td>tout le monde est</td><td>99</td></tr>\n",
              "                \n",
              "                <tr><td>que ce soit ?</td><td>99</td></tr>\n",
              "                \n",
              "                <tr><td>ne sais pas ce</td><td>99</td></tr>\n",
              "                \n",
              "                <tr><td>tout le monde a</td><td>97</td></tr>\n",
              "                \n",
              "                <tr><td>ne peux pas te</td><td>97</td></tr>\n",
              "                \n",
              "                <tr><td>un inconvénient à ce</td><td>97</td></tr>\n",
              "                \n",
              "                <tr><td>inconvénient à ce que</td><td>97</td></tr>\n",
              "                \n",
              "                <tr><td>je ne parviens pas</td><td>96</td></tr>\n",
              "                \n",
              "                <tr><td>ne peux pas vous</td><td>96</td></tr>\n",
              "                \n",
              "                <tr><td>sur le point de</td><td>94</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "    </div>\n",
              "</div>\n",
              "\n",
              "\n",
              "<script src=\"https://code.jquery.com/jquery-3.3.1.slim.min.js\"\n",
              "        integrity=\"sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js\"\n",
              "        integrity=\"sha384-wHAiFfRlMFy6i5SRaxvfOCifBUQy1xHdJ/yoi7FRNXMRBu5WHdZYu1hA6ZOblgut\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js\"\n",
              "        integrity=\"sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "</body>\n",
              "</html>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2L5wilGWiZac",
        "colab_type": "text"
      },
      "source": [
        "### Refrence-side n-grams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "76VXJWH6cbkM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "94809cb7-c948-412c-aef7-5c3d4800f771"
      },
      "source": [
        "vizseq.view_n_grams(references)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<!DOCTYPE html>\n",
              "<html lang=\"en\">\n",
              "<head>\n",
              "    <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css\"\n",
              "          integrity=\"sha384-GJzZqFGwb1QTTN6wy59ffF1BuGJpLSa9DkKMp0DgiMDm4iYMj70gZWKYbI706tWS\" crossorigin=\"anonymous\">\n",
              "    <script src=\"https://kit.fontawesome.com/3c563f9647.js\"></script>\n",
              "\n",
              "    <style>\n",
              "        body {\n",
              "            padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */\n",
              "        }\n",
              "    </style>\n",
              "</head>\n",
              "\n",
              "<body>\n",
              "\n",
              "\n",
              "<div class=\"container\">\n",
              "    <div class=\"row\">\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">1-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>i</td><td>48047</td></tr>\n",
              "                \n",
              "                <tr><td>you</td><td>38499</td></tr>\n",
              "                \n",
              "                <tr><td>to</td><td>38412</td></tr>\n",
              "                \n",
              "                <tr><td>the</td><td>34629</td></tr>\n",
              "                \n",
              "                <tr><td>a</td><td>25113</td></tr>\n",
              "                \n",
              "                <tr><td>is</td><td>16917</td></tr>\n",
              "                \n",
              "                <tr><td>tom</td><td>12967</td></tr>\n",
              "                \n",
              "                <tr><td>he</td><td>12131</td></tr>\n",
              "                \n",
              "                <tr><td>of</td><td>11668</td></tr>\n",
              "                \n",
              "                <tr><td>do</td><td>10968</td></tr>\n",
              "                \n",
              "                <tr><td>in</td><td>10287</td></tr>\n",
              "                \n",
              "                <tr><td>that</td><td>10171</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t</td><td>10159</td></tr>\n",
              "                \n",
              "                <tr><td>have</td><td>10040</td></tr>\n",
              "                \n",
              "                <tr><td>was</td><td>9001</td></tr>\n",
              "                \n",
              "                <tr><td>this</td><td>9000</td></tr>\n",
              "                \n",
              "                <tr><td>my</td><td>8464</td></tr>\n",
              "                \n",
              "                <tr><td>it</td><td>7830</td></tr>\n",
              "                \n",
              "                <tr><td>for</td><td>7825</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;m</td><td>7810</td></tr>\n",
              "                \n",
              "                <tr><td>your</td><td>7766</td></tr>\n",
              "                \n",
              "                <tr><td>are</td><td>7720</td></tr>\n",
              "                \n",
              "                <tr><td>what</td><td>7705</td></tr>\n",
              "                \n",
              "                <tr><td>we</td><td>7358</td></tr>\n",
              "                \n",
              "                <tr><td>be</td><td>7055</td></tr>\n",
              "                \n",
              "                <tr><td>me</td><td>6570</td></tr>\n",
              "                \n",
              "                <tr><td>want</td><td>6430</td></tr>\n",
              "                \n",
              "                <tr><td>not</td><td>6335</td></tr>\n",
              "                \n",
              "                <tr><td>she</td><td>6331</td></tr>\n",
              "                \n",
              "                <tr><td>like</td><td>5448</td></tr>\n",
              "                \n",
              "                <tr><td>know</td><td>5341</td></tr>\n",
              "                \n",
              "                <tr><td>with</td><td>5127</td></tr>\n",
              "                \n",
              "                <tr><td>on</td><td>4999</td></tr>\n",
              "                \n",
              "                <tr><td>you.</td><td>4988</td></tr>\n",
              "                \n",
              "                <tr><td>can</td><td>4616</td></tr>\n",
              "                \n",
              "                <tr><td>you&#39;re</td><td>4540</td></tr>\n",
              "                \n",
              "                <tr><td>his</td><td>4517</td></tr>\n",
              "                \n",
              "                <tr><td>at</td><td>4445</td></tr>\n",
              "                \n",
              "                <tr><td>did</td><td>4358</td></tr>\n",
              "                \n",
              "                <tr><td>how</td><td>4276</td></tr>\n",
              "                \n",
              "                <tr><td>they</td><td>4088</td></tr>\n",
              "                \n",
              "                <tr><td>all</td><td>4088</td></tr>\n",
              "                \n",
              "                <tr><td>think</td><td>4017</td></tr>\n",
              "                \n",
              "                <tr><td>and</td><td>3570</td></tr>\n",
              "                \n",
              "                <tr><td>it&#39;s</td><td>3511</td></tr>\n",
              "                \n",
              "                <tr><td>can&#39;t</td><td>3391</td></tr>\n",
              "                \n",
              "                <tr><td>go</td><td>3335</td></tr>\n",
              "                \n",
              "                <tr><td>very</td><td>3283</td></tr>\n",
              "                \n",
              "                <tr><td>didn&#39;t</td><td>3242</td></tr>\n",
              "                \n",
              "                <tr><td>about</td><td>3216</td></tr>\n",
              "                \n",
              "                <tr><td>get</td><td>3112</td></tr>\n",
              "                \n",
              "                <tr><td>were</td><td>3015</td></tr>\n",
              "                \n",
              "                <tr><td>as</td><td>2981</td></tr>\n",
              "                \n",
              "                <tr><td>will</td><td>2895</td></tr>\n",
              "                \n",
              "                <tr><td>no</td><td>2894</td></tr>\n",
              "                \n",
              "                <tr><td>me.</td><td>2877</td></tr>\n",
              "                \n",
              "                <tr><td>if</td><td>2857</td></tr>\n",
              "                \n",
              "                <tr><td>had</td><td>2843</td></tr>\n",
              "                \n",
              "                <tr><td>just</td><td>2768</td></tr>\n",
              "                \n",
              "                <tr><td>why</td><td>2748</td></tr>\n",
              "                \n",
              "                <tr><td>that.</td><td>2668</td></tr>\n",
              "                \n",
              "                <tr><td>has</td><td>2664</td></tr>\n",
              "                \n",
              "                <tr><td>it.</td><td>2579</td></tr>\n",
              "                \n",
              "                <tr><td>going</td><td>2505</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">2-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>do you</td><td>5063</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t</td><td>4463</td></tr>\n",
              "                \n",
              "                <tr><td>want to</td><td>4020</td></tr>\n",
              "                \n",
              "                <tr><td>are you</td><td>3224</td></tr>\n",
              "                \n",
              "                <tr><td>in the</td><td>2778</td></tr>\n",
              "                \n",
              "                <tr><td>i have</td><td>2385</td></tr>\n",
              "                \n",
              "                <tr><td>to be</td><td>2371</td></tr>\n",
              "                \n",
              "                <tr><td>i want</td><td>2333</td></tr>\n",
              "                \n",
              "                <tr><td>did you</td><td>2318</td></tr>\n",
              "                \n",
              "                <tr><td>to do</td><td>2226</td></tr>\n",
              "                \n",
              "                <tr><td>i was</td><td>2160</td></tr>\n",
              "                \n",
              "                <tr><td>i think</td><td>1995</td></tr>\n",
              "                \n",
              "                <tr><td>tom is</td><td>1964</td></tr>\n",
              "                \n",
              "                <tr><td>going to</td><td>1949</td></tr>\n",
              "                \n",
              "                <tr><td>i can&#39;t</td><td>1879</td></tr>\n",
              "                \n",
              "                <tr><td>to the</td><td>1864</td></tr>\n",
              "                \n",
              "                <tr><td>you have</td><td>1841</td></tr>\n",
              "                \n",
              "                <tr><td>have to</td><td>1835</td></tr>\n",
              "                \n",
              "                <tr><td>have a</td><td>1821</td></tr>\n",
              "                \n",
              "                <tr><td>of the</td><td>1714</td></tr>\n",
              "                \n",
              "                <tr><td>is a</td><td>1673</td></tr>\n",
              "                \n",
              "                <tr><td>i know</td><td>1624</td></tr>\n",
              "                \n",
              "                <tr><td>this is</td><td>1597</td></tr>\n",
              "                \n",
              "                <tr><td>on the</td><td>1564</td></tr>\n",
              "                \n",
              "                <tr><td>you to</td><td>1505</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;m not</td><td>1473</td></tr>\n",
              "                \n",
              "                <tr><td>i didn&#39;t</td><td>1470</td></tr>\n",
              "                \n",
              "                <tr><td>you want</td><td>1423</td></tr>\n",
              "                \n",
              "                <tr><td>like to</td><td>1389</td></tr>\n",
              "                \n",
              "                <tr><td>he is</td><td>1355</td></tr>\n",
              "                \n",
              "                <tr><td>to go</td><td>1316</td></tr>\n",
              "                \n",
              "                <tr><td>you don&#39;t</td><td>1294</td></tr>\n",
              "                \n",
              "                <tr><td>it was</td><td>1167</td></tr>\n",
              "                \n",
              "                <tr><td>he was</td><td>1157</td></tr>\n",
              "                \n",
              "                <tr><td>i had</td><td>1102</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t know</td><td>1102</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t want</td><td>1094</td></tr>\n",
              "                \n",
              "                <tr><td>need to</td><td>1065</td></tr>\n",
              "                \n",
              "                <tr><td>have you</td><td>1057</td></tr>\n",
              "                \n",
              "                <tr><td>i just</td><td>1054</td></tr>\n",
              "                \n",
              "                <tr><td>if you</td><td>1036</td></tr>\n",
              "                \n",
              "                <tr><td>can you</td><td>1031</td></tr>\n",
              "                \n",
              "                <tr><td>at the</td><td>1017</td></tr>\n",
              "                \n",
              "                <tr><td>i thought</td><td>1011</td></tr>\n",
              "                \n",
              "                <tr><td>a lot</td><td>1010</td></tr>\n",
              "                \n",
              "                <tr><td>go to</td><td>974</td></tr>\n",
              "                \n",
              "                <tr><td>a good</td><td>967</td></tr>\n",
              "                \n",
              "                <tr><td>is the</td><td>949</td></tr>\n",
              "                \n",
              "                <tr><td>for a</td><td>947</td></tr>\n",
              "                \n",
              "                <tr><td>i am</td><td>934</td></tr>\n",
              "                \n",
              "                <tr><td>you know</td><td>933</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;d like</td><td>923</td></tr>\n",
              "                \n",
              "                <tr><td>to get</td><td>919</td></tr>\n",
              "                \n",
              "                <tr><td>me to</td><td>912</td></tr>\n",
              "                \n",
              "                <tr><td>would you</td><td>908</td></tr>\n",
              "                \n",
              "                <tr><td>i can</td><td>904</td></tr>\n",
              "                \n",
              "                <tr><td>to see</td><td>893</td></tr>\n",
              "                \n",
              "                <tr><td>out of</td><td>891</td></tr>\n",
              "                \n",
              "                <tr><td>you were</td><td>879</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t have</td><td>875</td></tr>\n",
              "                \n",
              "                <tr><td>you should</td><td>873</td></tr>\n",
              "                \n",
              "                <tr><td>do that.</td><td>845</td></tr>\n",
              "                \n",
              "                <tr><td>you think</td><td>842</td></tr>\n",
              "                \n",
              "                <tr><td>know what</td><td>826</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">3-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>i want to</td><td>1389</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t know</td><td>926</td></tr>\n",
              "                \n",
              "                <tr><td>you want to</td><td>866</td></tr>\n",
              "                \n",
              "                <tr><td>do you want</td><td>776</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t want to</td><td>759</td></tr>\n",
              "                \n",
              "                <tr><td>a lot of</td><td>751</td></tr>\n",
              "                \n",
              "                <tr><td>do you have</td><td>738</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;d like to</td><td>722</td></tr>\n",
              "                \n",
              "                <tr><td>want you to</td><td>660</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t want</td><td>650</td></tr>\n",
              "                \n",
              "                <tr><td>do you know</td><td>568</td></tr>\n",
              "                \n",
              "                <tr><td>i have a</td><td>566</td></tr>\n",
              "                \n",
              "                <tr><td>do you think</td><td>566</td></tr>\n",
              "                \n",
              "                <tr><td>i have to</td><td>538</td></tr>\n",
              "                \n",
              "                <tr><td>what do you</td><td>512</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t think</td><td>495</td></tr>\n",
              "                \n",
              "                <tr><td>to do that.</td><td>481</td></tr>\n",
              "                \n",
              "                <tr><td>i want you</td><td>479</td></tr>\n",
              "                \n",
              "                <tr><td>you have a</td><td>450</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;m going to</td><td>434</td></tr>\n",
              "                \n",
              "                <tr><td>why are you</td><td>433</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t like</td><td>416</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t have</td><td>409</td></tr>\n",
              "                \n",
              "                <tr><td>to go to</td><td>407</td></tr>\n",
              "                \n",
              "                <tr><td>why don&#39;t you</td><td>390</td></tr>\n",
              "                \n",
              "                <tr><td>would you like</td><td>379</td></tr>\n",
              "                \n",
              "                <tr><td>want to be</td><td>373</td></tr>\n",
              "                \n",
              "                <tr><td>are you sure</td><td>365</td></tr>\n",
              "                \n",
              "                <tr><td>you have to</td><td>362</td></tr>\n",
              "                \n",
              "                <tr><td>i can&#39;t believe</td><td>349</td></tr>\n",
              "                \n",
              "                <tr><td>tom is a</td><td>348</td></tr>\n",
              "                \n",
              "                <tr><td>you don&#39;t have</td><td>343</td></tr>\n",
              "                \n",
              "                <tr><td>tom and mary</td><td>332</td></tr>\n",
              "                \n",
              "                <tr><td>how did you</td><td>321</td></tr>\n",
              "                \n",
              "                <tr><td>to talk to</td><td>320</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t have to</td><td>314</td></tr>\n",
              "                \n",
              "                <tr><td>go to the</td><td>310</td></tr>\n",
              "                \n",
              "                <tr><td>i need to</td><td>308</td></tr>\n",
              "                \n",
              "                <tr><td>i thought you</td><td>304</td></tr>\n",
              "                \n",
              "                <tr><td>you need to</td><td>299</td></tr>\n",
              "                \n",
              "                <tr><td>i just want</td><td>295</td></tr>\n",
              "                \n",
              "                <tr><td>i know that</td><td>294</td></tr>\n",
              "                \n",
              "                <tr><td>i didn&#39;t know</td><td>293</td></tr>\n",
              "                \n",
              "                <tr><td>do you like</td><td>292</td></tr>\n",
              "                \n",
              "                <tr><td>i think you</td><td>288</td></tr>\n",
              "                \n",
              "                <tr><td>how do you</td><td>287</td></tr>\n",
              "                \n",
              "                <tr><td>have you ever</td><td>287</td></tr>\n",
              "                \n",
              "                <tr><td>you like to</td><td>284</td></tr>\n",
              "                \n",
              "                <tr><td>this is a</td><td>282</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t know what</td><td>281</td></tr>\n",
              "                \n",
              "                <tr><td>want to go</td><td>279</td></tr>\n",
              "                \n",
              "                <tr><td>i had a</td><td>278</td></tr>\n",
              "                \n",
              "                <tr><td>want to do</td><td>278</td></tr>\n",
              "                \n",
              "                <tr><td>i know tom</td><td>271</td></tr>\n",
              "                \n",
              "                <tr><td>to be a</td><td>268</td></tr>\n",
              "                \n",
              "                <tr><td>what are you</td><td>265</td></tr>\n",
              "                \n",
              "                <tr><td>did you get</td><td>257</td></tr>\n",
              "                \n",
              "                <tr><td>want me to</td><td>255</td></tr>\n",
              "                \n",
              "                <tr><td>have to do</td><td>252</td></tr>\n",
              "                \n",
              "                <tr><td>you don&#39;t want</td><td>246</td></tr>\n",
              "                \n",
              "                <tr><td>are you going</td><td>241</td></tr>\n",
              "                \n",
              "                <tr><td>we have to</td><td>240</td></tr>\n",
              "                \n",
              "                <tr><td>i had to</td><td>234</td></tr>\n",
              "                \n",
              "                <tr><td>to do that</td><td>234</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered text-center\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr><th scope=\"col\">4-gram</th><th scope=\"col\">Count</th></tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                \n",
              "                <tr><td>i don&#39;t want to</td><td>469</td></tr>\n",
              "                \n",
              "                <tr><td>do you want to</td><td>434</td></tr>\n",
              "                \n",
              "                <tr><td>i want you to</td><td>428</td></tr>\n",
              "                \n",
              "                <tr><td>you don&#39;t have to</td><td>263</td></tr>\n",
              "                \n",
              "                <tr><td>would you like to</td><td>240</td></tr>\n",
              "                \n",
              "                <tr><td>do you have a</td><td>232</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t know what</td><td>231</td></tr>\n",
              "                \n",
              "                <tr><td>i just want to</td><td>204</td></tr>\n",
              "                \n",
              "                <tr><td>are you going to</td><td>195</td></tr>\n",
              "                \n",
              "                <tr><td>you want me to</td><td>193</td></tr>\n",
              "                \n",
              "                <tr><td>i know tom is</td><td>166</td></tr>\n",
              "                \n",
              "                <tr><td>do you have any</td><td>166</td></tr>\n",
              "                \n",
              "                <tr><td>you don&#39;t want to</td><td>160</td></tr>\n",
              "                \n",
              "                <tr><td>are you sure you</td><td>158</td></tr>\n",
              "                \n",
              "                <tr><td>i want to know</td><td>154</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t know how</td><td>150</td></tr>\n",
              "                \n",
              "                <tr><td>what do you think</td><td>135</td></tr>\n",
              "                \n",
              "                <tr><td>to go to the</td><td>132</td></tr>\n",
              "                \n",
              "                <tr><td>do you want me</td><td>129</td></tr>\n",
              "                \n",
              "                <tr><td>to talk to you</td><td>125</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t think i</td><td>123</td></tr>\n",
              "                \n",
              "                <tr><td>i thought you were</td><td>122</td></tr>\n",
              "                \n",
              "                <tr><td>i think it&#39;s time</td><td>119</td></tr>\n",
              "                \n",
              "                <tr><td>have a lot of</td><td>118</td></tr>\n",
              "                \n",
              "                <tr><td>she advised him to</td><td>118</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;m not going to</td><td>115</td></tr>\n",
              "                \n",
              "                <tr><td>i just don&#39;t want</td><td>115</td></tr>\n",
              "                \n",
              "                <tr><td>why are you so</td><td>114</td></tr>\n",
              "                \n",
              "                <tr><td>you mind if i</td><td>112</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t want you to</td><td>110</td></tr>\n",
              "                \n",
              "                <tr><td>think it&#39;s time for</td><td>110</td></tr>\n",
              "                \n",
              "                <tr><td>i can&#39;t tell you</td><td>108</td></tr>\n",
              "                \n",
              "                <tr><td>i want to be</td><td>107</td></tr>\n",
              "                \n",
              "                <tr><td>time for me to</td><td>107</td></tr>\n",
              "                \n",
              "                <tr><td>it&#39;s time for me</td><td>106</td></tr>\n",
              "                \n",
              "                <tr><td>how did you get</td><td>104</td></tr>\n",
              "                \n",
              "                <tr><td>want to talk to</td><td>102</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t want to be</td><td>101</td></tr>\n",
              "                \n",
              "                <tr><td>i didn&#39;t know you</td><td>99</td></tr>\n",
              "                \n",
              "                <tr><td>you sure you don&#39;t</td><td>96</td></tr>\n",
              "                \n",
              "                <tr><td>i&#39;d like you to</td><td>94</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t know if</td><td>94</td></tr>\n",
              "                \n",
              "                <tr><td>sure you don&#39;t want</td><td>94</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t have a</td><td>93</td></tr>\n",
              "                \n",
              "                <tr><td>do you know how</td><td>92</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t want you</td><td>92</td></tr>\n",
              "                \n",
              "                <tr><td>what do you want</td><td>90</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t know how to</td><td>89</td></tr>\n",
              "                \n",
              "                <tr><td>do you mind if</td><td>88</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t feel like</td><td>88</td></tr>\n",
              "                \n",
              "                <tr><td>have you ever been</td><td>87</td></tr>\n",
              "                \n",
              "                <tr><td>i didn&#39;t want to</td><td>86</td></tr>\n",
              "                \n",
              "                <tr><td>i just wanted to</td><td>85</td></tr>\n",
              "                \n",
              "                <tr><td>how do you know</td><td>84</td></tr>\n",
              "                \n",
              "                <tr><td>you want to go</td><td>83</td></tr>\n",
              "                \n",
              "                <tr><td>don&#39;t think i can</td><td>82</td></tr>\n",
              "                \n",
              "                <tr><td>want to go to</td><td>81</td></tr>\n",
              "                \n",
              "                <tr><td>you have to do</td><td>81</td></tr>\n",
              "                \n",
              "                <tr><td>i would like to</td><td>79</td></tr>\n",
              "                \n",
              "                <tr><td>do i have to</td><td>78</td></tr>\n",
              "                \n",
              "                <tr><td>do you know what</td><td>78</td></tr>\n",
              "                \n",
              "                <tr><td>talk to you about</td><td>77</td></tr>\n",
              "                \n",
              "                <tr><td>i know that tom</td><td>76</td></tr>\n",
              "                \n",
              "                <tr><td>i don&#39;t know why</td><td>76</td></tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "        \n",
              "    </div>\n",
              "</div>\n",
              "\n",
              "\n",
              "<script src=\"https://code.jquery.com/jquery-3.3.1.slim.min.js\"\n",
              "        integrity=\"sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js\"\n",
              "        integrity=\"sha384-wHAiFfRlMFy6i5SRaxvfOCifBUQy1xHdJ/yoi7FRNXMRBu5WHdZYu1hA6ZOblgut\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js\"\n",
              "        integrity=\"sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "</body>\n",
              "</html>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "994oOFpqjCW2",
        "colab_type": "text"
      },
      "source": [
        "## Inference evaluation\n",
        "\n",
        "### Metrics inspection\n",
        "VizSeq let you analyze the performance of your Machine Translation model by comparing the References (true translation) against Hypothesis (inference output) using popular NLP metrics, e.g. BLUE."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J__XKiFdZxMJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "3dc9f23b-d45f-4e1b-d0bb-512ce7607368"
      },
      "source": [
        "vizseq.available_scorers()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Available scorers: bert_score, bleu, bp, chrf, cider, gleu, laser, meteor, nist, ribes, rouge_1, rouge_2, rouge_l, ter, wer, wer_del, wer_ins, wer_sub\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WdPhBJd5mHTB",
        "colab_type": "text"
      },
      "source": [
        "Compare metrics, for the sake of example we will assume that hypothesis is same as referneces"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ac6-q30DmRWT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "hypothesis = references"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TuUEydtYjBRy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "outputId": "032338a1-8a3b-4249-dca3-27a62a30b048"
      },
      "source": [
        "vizseq.view_scores(references, hypothesis, ['bleu', 'meteor'])"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<!DOCTYPE html>\n",
              "<html lang=\"en\">\n",
              "<head>\n",
              "    <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css\"\n",
              "          integrity=\"sha384-GJzZqFGwb1QTTN6wy59ffF1BuGJpLSa9DkKMp0DgiMDm4iYMj70gZWKYbI706tWS\" crossorigin=\"anonymous\">\n",
              "\n",
              "    <style>\n",
              "        body {\n",
              "            padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */\n",
              "        }\n",
              "    </style>\n",
              "</head>\n",
              "\n",
              "<body>\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "<script>\n",
              "    function visualizeDict(dict) {\n",
              "        let values = Object.keys(dict).map(k => dict[k]);\n",
              "        let worse_v = Math.min(...values);\n",
              "        let best_v = Math.max(...values);\n",
              "        let visualized = {};\n",
              "        Object.keys(dict).forEach(function (k) {\n",
              "            if (dict[k] === worse_v) {\n",
              "                visualized[k] = '<span style=\"font-style:italic\"><u>' + dict[k] + '</u></span>';\n",
              "            } else if (dict[k] === best_v) {\n",
              "                visualized[k] = '<span style=\"font-weight:bold\">' + dict[k] + '</span>';\n",
              "            } else {\n",
              "                visualized[k] = dict[k];\n",
              "            }\n",
              "        });\n",
              "        return visualized;\n",
              "    }\n",
              "</script>\n",
              "\n",
              "\n",
              "<script>\n",
              "    function copyToClipboard(elementId) {\n",
              "        let $temp = $(\"<input>\");\n",
              "        $(\"body\").append($temp);\n",
              "        $temp.val($(elementId).text()).select();\n",
              "        document.execCommand(\"copy\");\n",
              "        $temp.remove();\n",
              "    }\n",
              "</script>\n",
              "\n",
              "\n",
              "\n",
              "<div class=\"container-fluid\">\n",
              "    <h4>BLEU</h4>\n",
              "\n",
              "    <div class=\"row\">\n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr>\n",
              "                    <th scope=\"col\">\n",
              "                        <button type=\"button\" class=\"btn btn-secondary btn-sm\" data-toggle=\"tooltip\"\n",
              "                                data-placement=\"bottom\" title=\"Copy to clipboard\"\n",
              "                                onclick=\"copyToClipboard('#csv_bleu')\">CSV</button>\n",
              "                        <button type=\"button\" class=\"btn btn-secondary btn-sm\" data-toggle=\"tooltip\"\n",
              "                                data-placement=\"bottom\" title=\"Copy to clipboard\"\n",
              "                                onclick=\"copyToClipboard('#latex_bleu')\">LaTeX</button>\n",
              "                    </th>\n",
              "                     <th scope=\"col\" style=\"word-wrap: break-word\">fra.txt</th> \n",
              "                </tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                <tr>\n",
              "                    <th scope=\"row\">All</th> <td>100.0</td> \n",
              "                </tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "    </div>\n",
              "\n",
              "    <span id=\"latex_bleu\" style=\"display:none\">\\begin{tabular}{lr}\n",
              "\\toprule\n",
              "{} &amp;  fra.txt \\\\\n",
              "\\midrule\n",
              "All &amp;   100.00 \\\\\n",
              "\\bottomrule\n",
              "\\end{tabular}\n",
              "</span>\n",
              "    <span id=\"csv_bleu\" style=\"display:none\">,fra.txt\n",
              "All,100.0\n",
              "</span>\n",
              "</div>\n",
              "\n",
              "<div class=\"container-fluid\">\n",
              "    <h4>METEOR</h4>\n",
              "\n",
              "    <div class=\"row\">\n",
              "        <div class=\"col\">\n",
              "            <table class=\"table table-hover table-sm table-bordered\">\n",
              "                <thead class=\"thead-light\">\n",
              "                <tr>\n",
              "                    <th scope=\"col\">\n",
              "                        <button type=\"button\" class=\"btn btn-secondary btn-sm\" data-toggle=\"tooltip\"\n",
              "                                data-placement=\"bottom\" title=\"Copy to clipboard\"\n",
              "                                onclick=\"copyToClipboard('#csv_meteor')\">CSV</button>\n",
              "                        <button type=\"button\" class=\"btn btn-secondary btn-sm\" data-toggle=\"tooltip\"\n",
              "                                data-placement=\"bottom\" title=\"Copy to clipboard\"\n",
              "                                onclick=\"copyToClipboard('#latex_meteor')\">LaTeX</button>\n",
              "                    </th>\n",
              "                     <th scope=\"col\" style=\"word-wrap: break-word\">fra.txt</th> \n",
              "                </tr>\n",
              "                </thead>\n",
              "                <tbody>\n",
              "                <tr>\n",
              "                    <th scope=\"row\">All</th> <td>0.994</td> \n",
              "                </tr>\n",
              "                \n",
              "                </tbody>\n",
              "            </table>\n",
              "        </div>\n",
              "    </div>\n",
              "\n",
              "    <span id=\"latex_meteor\" style=\"display:none\">\\begin{tabular}{lr}\n",
              "\\toprule\n",
              "{} &amp;  fra.txt \\\\\n",
              "\\midrule\n",
              "All &amp;     0.99 \\\\\n",
              "\\bottomrule\n",
              "\\end{tabular}\n",
              "</span>\n",
              "    <span id=\"csv_meteor\" style=\"display:none\">,fra.txt\n",
              "All,0.994\n",
              "</span>\n",
              "</div>\n",
              "\n",
              "\n",
              "<script src=\"https://code.jquery.com/jquery-3.3.1.slim.min.js\"\n",
              "        integrity=\"sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js\"\n",
              "        integrity=\"sha384-wHAiFfRlMFy6i5SRaxvfOCifBUQy1xHdJ/yoi7FRNXMRBu5WHdZYu1hA6ZOblgut\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js\"\n",
              "        integrity=\"sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "\n",
              "</body>\n",
              "</html>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0I5kDrCbm6xE",
        "colab_type": "text"
      },
      "source": [
        "### Examples inspection\n",
        "VizSeq let you manually inspect the examples with `vizseq.view_examples`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OVuyQe8cmAmp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 706
        },
        "outputId": "b0243ebf-77a7-47db-c1e1-061da83ffb49"
      },
      "source": [
        "vizseq.view_examples(sources, references, hypothesis, ['bleu'], page_sz=2, page_no=12)"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<!DOCTYPE html>\n",
              "<html lang=\"en\">\n",
              "<head>\n",
              "    <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css\"\n",
              "          integrity=\"sha384-GJzZqFGwb1QTTN6wy59ffF1BuGJpLSa9DkKMp0DgiMDm4iYMj70gZWKYbI706tWS\" crossorigin=\"anonymous\">\n",
              "\n",
              "    <style>\n",
              "        body {\n",
              "            padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */\n",
              "        }\n",
              "    </style>\n",
              "</head>\n",
              "\n",
              "<body>\n",
              "\n",
              "\n",
              "<script>\n",
              "    function highlight_span(spanNode, trgSpanId) {\n",
              "        trgSpanNode = document.getElementById(trgSpanId);\n",
              "        trgSpanNode.style.backgroundColor = '#D6EAF8';\n",
              "        spanNode.style.backgroundColor = '#D6EAF8';\n",
              "    }\n",
              "    function dehighlight_span(spanNode, trgSpanId) {\n",
              "        trgSpanNode = document.getElementById(trgSpanId);\n",
              "        trgSpanNode.style.backgroundColor = trgSpanNode.parentNode.style.backgroundColor;\n",
              "        spanNode.style.backgroundColor = spanNode.parentNode.style.backgroundColor;\n",
              "    }\n",
              "</script>\n",
              "\n",
              "\n",
              "\n",
              "<div class=\"card\">\n",
              "    <div class=\"card-header\">\n",
              "        <div class=\"row\">\n",
              "            <div class=\"col\">\n",
              "                <b># 1 / 2 / 174481 ( 23 / 174481 )</b>\n",
              "            </div>\n",
              "            <div class=\"col\" align=\"right\">\n",
              "            </div>\n",
              "        </div>\n",
              "    </div>\n",
              "    <div class=\"card-body\">\n",
              "        <table class=\"table table-hover table-sm table-borderless\">\n",
              "            <tbody>\n",
              "            \n",
              "            \n",
              "            <tr style=\"background-color:#FDF2E9\" id=\"srcRow00\">\n",
              "                <th scope=\"row\" class=\"text-left\" width=\"20%\">Source fra.txt</th>\n",
              "                \n",
              "                <td class=\"text-left\"><span id=\"src_0_0_0\">J'ai</span> <span id=\"src_0_0_1\">gagné</span> <span id=\"src_0_0_2\">!</span></td>\n",
              "                \n",
              "            </tr>\n",
              "            \n",
              "\n",
              "            <tr style=\"background-color:#EAF2F8\">\n",
              "            \n",
              "            </tr>\n",
              "\n",
              "            \n",
              "            \n",
              "            \n",
              "            <tr style=\"background-color:#FEF9E7\">\n",
              "                <th scope=\"row\" class=\"text-left\">Reference fra.txt</th>\n",
              "                <td class=\"text-left\"><span id=\"ref_0_0_0\" >I</span> <span id=\"ref_0_0_1\" >won!</span></td>\n",
              "            </tr>\n",
              "            \n",
              "            \n",
              "\n",
              "            \n",
              "            <tr>\n",
              "                <th scope=\"row\" class=\"text-left\">fra.txt</th>\n",
              "                <td class=\"text-left\"><span id=\"hypo_fra.txt_0_0\" onmouseover=\"javascript:highlight_span(this, &quot;ref_0_0_0&quot;)\" onmouseout=\"javascript:dehighlight_span(this, &quot;ref_0_0_0&quot;)\" style=\"color:#5499C7;font-weight:bold\" >I</span> <span id=\"hypo_fra.txt_0_1\" onmouseover=\"javascript:highlight_span(this, &quot;ref_0_0_1&quot;)\" onmouseout=\"javascript:dehighlight_span(this, &quot;ref_0_0_1&quot;)\" style=\"color:#5499C7;font-weight:bold\" >won!</span></td>\n",
              "            </tr>\n",
              "            \n",
              "            </tbody>\n",
              "        </table>\n",
              "\n",
              "        \n",
              "        <table class=\"table table-hover table-sm table-bordered\">\n",
              "            <thead class=\"thead-light\">\n",
              "            <tr>\n",
              "                <th scope=\"col\">Model</th><th scope=\"col\">bleu</th>\n",
              "            </tr>\n",
              "            </thead>\n",
              "            <tbody>\n",
              "            \n",
              "            <tr>\n",
              "                <td>fra.txt</td><td><span style=\"font-weight:bold\">100.0</span></td>\n",
              "            </tr>\n",
              "            \n",
              "            </tbody>\n",
              "        </table>\n",
              "        \n",
              "    </div>\n",
              "</div>\n",
              "<br>\n",
              "\n",
              "<div class=\"card\">\n",
              "    <div class=\"card-header\">\n",
              "        <div class=\"row\">\n",
              "            <div class=\"col\">\n",
              "                <b># 2 / 2 / 174481 ( 24 / 174481 )</b>\n",
              "            </div>\n",
              "            <div class=\"col\" align=\"right\">\n",
              "            </div>\n",
              "        </div>\n",
              "    </div>\n",
              "    <div class=\"card-body\">\n",
              "        <table class=\"table table-hover table-sm table-borderless\">\n",
              "            <tbody>\n",
              "            \n",
              "            \n",
              "            <tr style=\"background-color:#FDF2E9\" id=\"srcRow01\">\n",
              "                <th scope=\"row\" class=\"text-left\" width=\"20%\">Source fra.txt</th>\n",
              "                \n",
              "                <td class=\"text-left\"><span id=\"src_0_1_0\">Je</span> <span id=\"src_0_1_1\">l'ai</span> <span id=\"src_0_1_2\">emporté</span> <span id=\"src_0_1_3\">!</span></td>\n",
              "                \n",
              "            </tr>\n",
              "            \n",
              "\n",
              "            <tr style=\"background-color:#EAF2F8\">\n",
              "            \n",
              "            </tr>\n",
              "\n",
              "            \n",
              "            \n",
              "            \n",
              "            <tr style=\"background-color:#FEF9E7\">\n",
              "                <th scope=\"row\" class=\"text-left\">Reference fra.txt</th>\n",
              "                <td class=\"text-left\"><span id=\"ref_0_1_0\" >I</span> <span id=\"ref_0_1_1\" >won!</span></td>\n",
              "            </tr>\n",
              "            \n",
              "            \n",
              "\n",
              "            \n",
              "            <tr>\n",
              "                <th scope=\"row\" class=\"text-left\">fra.txt</th>\n",
              "                <td class=\"text-left\"><span id=\"hypo_fra.txt_1_0\" onmouseover=\"javascript:highlight_span(this, &quot;ref_0_1_0&quot;)\" onmouseout=\"javascript:dehighlight_span(this, &quot;ref_0_1_0&quot;)\" style=\"color:#5499C7;font-weight:bold\" >I</span> <span id=\"hypo_fra.txt_1_1\" onmouseover=\"javascript:highlight_span(this, &quot;ref_0_1_1&quot;)\" onmouseout=\"javascript:dehighlight_span(this, &quot;ref_0_1_1&quot;)\" style=\"color:#5499C7;font-weight:bold\" >won!</span></td>\n",
              "            </tr>\n",
              "            \n",
              "            </tbody>\n",
              "        </table>\n",
              "\n",
              "        \n",
              "        <table class=\"table table-hover table-sm table-bordered\">\n",
              "            <thead class=\"thead-light\">\n",
              "            <tr>\n",
              "                <th scope=\"col\">Model</th><th scope=\"col\">bleu</th>\n",
              "            </tr>\n",
              "            </thead>\n",
              "            <tbody>\n",
              "            \n",
              "            <tr>\n",
              "                <td>fra.txt</td><td><span style=\"font-weight:bold\">100.0</span></td>\n",
              "            </tr>\n",
              "            \n",
              "            </tbody>\n",
              "        </table>\n",
              "        \n",
              "    </div>\n",
              "</div>\n",
              "<br>\n",
              "\n",
              "\n",
              "<script src=\"https://code.jquery.com/jquery-3.3.1.slim.min.js\"\n",
              "        integrity=\"sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js\"\n",
              "        integrity=\"sha384-wHAiFfRlMFy6i5SRaxvfOCifBUQy1xHdJ/yoi7FRNXMRBu5WHdZYu1hA6ZOblgut\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "<script src=\"https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js\"\n",
              "        integrity=\"sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k\" crossorigin=\"anonymous\">\n",
              "</script>\n",
              "</body>\n",
              "</html>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2tKg_C_onPkQ",
        "colab_type": "text"
      },
      "source": [
        "Check official VizSeq website for more details - [link](https://facebookresearch.github.io/vizseq/)."
      ]
    }
  ]
}