tutorials/data_handling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.059593Z",
     "start_time": "2019-04-03T09:48:27.403792Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "import pandas as pd\n",
    "print(mz.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataPack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Structure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`matchzoo.DataPack` is a MatchZoo native data structure that most MatchZoo data handling processes build upon. A `matchzoo.DataPack` consists of three parts: `left`, `right` and `relation`, each one of is a `pandas.DataFrame`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.086451Z",
     "start_time": "2019-04-03T09:48:30.061756Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_pack = mz.datasets.toy.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.108737Z",
     "start_time": "2019-04-03T09:48:30.088302Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_left</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_left</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q1</th>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q5</th>\n",
       "      <td>how did apollo creed die</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q6</th>\n",
       "      <td>how long is the term for federal judges</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q7</th>\n",
       "      <td>how a beretta model 21 pistols magazines works</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_left\n",
       "id_left                                                   \n",
       "Q1                           how are glacier caves formed?\n",
       "Q2       How are the directions of the velocity and for...\n",
       "Q5                                how did apollo creed die\n",
       "Q6                 how long is the term for federal judges\n",
       "Q7          how a beretta model 21 pistols magazines works"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.left.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.119367Z",
     "start_time": "2019-04-03T09:48:30.113090Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_right</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1-0</th>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-1</th>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-2</th>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-3</th>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-4</th>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_right\n",
       "id_right                                                   \n",
       "D1-0      A partly submerged glacier cave on Perito More...\n",
       "D1-1              The ice facade is approximately 60 m high\n",
       "D1-2              Ice formations in the Titlis glacier cave\n",
       "D1-3      A glacier cave is a cave formed within the ice...\n",
       "D1-4      Glacier caves are often called ice caves , but..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.right.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.129987Z",
     "start_time": "2019-04-03T09:48:30.122949Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left id_right  label\n",
       "0      Q1     D1-0    0.0\n",
       "1      Q1     D1-1    0.0\n",
       "2      Q1     D1-2    0.0\n",
       "3      Q1     D1-3    1.0\n",
       "4      Q1     D1-4    0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.relation.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The main reason for using a `matchzoo.DataPack` instead of `pandas.DataFrame` is efficiency: we save space from storing duplicate texts and save time from processing duplicate texts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-13T03:12:15.986951Z",
     "start_time": "2018-12-13T03:12:15.984894Z"
    }
   },
   "source": [
    "## DataPack.FrameView"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, since a big table is easier to understand and manage, we provide the `frame` that merges three parts into a single `pandas.DataFrame` when called."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.147628Z",
     "start_time": "2019-04-03T09:48:30.132138Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    0.0  \n",
       "1          The ice facade is approximately 60 m high    0.0  \n",
       "2          Ice formations in the Titlis glacier cave    0.0  \n",
       "3  A glacier cave is a cave formed within the ice...    1.0  \n",
       "4  Glacier caves are often called ice caves , but...    0.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that `frame` is not a method, but a property that returns a `matchzoo.DataPack.FrameView` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.152848Z",
     "start_time": "2019-04-03T09:48:30.149232Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matchzoo.data_pack.data_pack.DataPack.FrameView"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data_pack.frame)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This view reflects changes in the data pack, and can be called to create a `pandas.DataFrame` at any time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.158711Z",
     "start_time": "2019-04-03T09:48:30.154324Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frame = data_pack.frame\n",
    "data_pack.relation['label'] = data_pack.relation['label'] + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.175473Z",
     "start_time": "2019-04-03T09:48:30.159993Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    1.0  \n",
       "1          The ice facade is approximately 60 m high    1.0  \n",
       "2          Ice formations in the Titlis glacier cave    1.0  \n",
       "3  A glacier cave is a cave formed within the ice...    2.0  \n",
       "4  Glacier caves are often called ice caves , but...    1.0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slicing a DataPack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You may use `[]` to slice a `matchzoo.DataPack` similar to slicing a `list`. This also returns a shallow copy of the sliced data like slicing a `list`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.181630Z",
     "start_time": "2019-04-03T09:48:30.176857Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_slice = data_pack[5:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A sliced data pack's `relation` will directly reflect the slicing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.191098Z",
     "start_time": "2019-04-03T09:48:30.183339Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left id_right  label\n",
       "0      Q2     D2-0    1.0\n",
       "1      Q2     D2-1    1.0\n",
       "2      Q2     D2-2    1.0\n",
       "3      Q2     D2-3    1.0\n",
       "4      Q2     D2-4    1.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.relation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, `left` and `right` will be processed so only relevant information are kept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.197471Z",
     "start_time": "2019-04-03T09:48:30.192792Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_left</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_left</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_left\n",
       "id_left                                                   \n",
       "Q2       How are the directions of the velocity and for..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.206259Z",
     "start_time": "2019-04-03T09:48:30.199187Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_right</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D2-0</th>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-1</th>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-2</th>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-3</th>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-4</th>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_right\n",
       "id_right                                                   \n",
       "D2-0      In physics , circular motion is a movement of ...\n",
       "D2-1      It can be uniform, with constant angular rate ...\n",
       "D2-2      The rotation around a fixed axis of a three-di...\n",
       "D2-3      The equations of motion describe the movement ...\n",
       "D2-4      Examples of circular motion include: an artifi..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.right"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to slice a frame view object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.221433Z",
     "start_time": "2019-04-03T09:48:30.207711Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0      Q2  How are the directions of the velocity and for...     D2-0   \n",
       "1      Q2  How are the directions of the velocity and for...     D2-1   \n",
       "2      Q2  How are the directions of the velocity and for...     D2-2   \n",
       "3      Q2  How are the directions of the velocity and for...     D2-3   \n",
       "4      Q2  How are the directions of the velocity and for...     D2-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  In physics , circular motion is a movement of ...    1.0  \n",
       "1  It can be uniform, with constant angular rate ...    1.0  \n",
       "2  The rotation around a fixed axis of a three-di...    1.0  \n",
       "3  The equations of motion describe the movement ...    1.0  \n",
       "4  Examples of circular motion include: an artifi...    1.0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.frame[5:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And this is equivalent to slicing the data pack first, then the frame, since both of them are based on the `relation` column. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.252181Z",
     "start_time": "2019-04-03T09:48:30.223605Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_left  text_left  id_right  text_right  label\n",
       "0     True       True      True        True   True\n",
       "1     True       True      True        True   True\n",
       "2     True       True      True        True   True\n",
       "3     True       True      True        True   True\n",
       "4     True       True      True        True   True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.frame() == data_pack.frame[5:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Slicing is extremely useful for partitioning data for training vs testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.261839Z",
     "start_time": "2019-04-03T09:48:30.253455Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_train = int(len(data_pack) * 0.8)\n",
    "data_pack.shuffle(inplace=True)\n",
    "train_slice = data_pack[:num_train]\n",
    "test_slice = data_pack[num_train:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforming Texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `apply_on_text` to transform texts in a `matchzoo.DataPack`. Check the documentation for more information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.287943Z",
     "start_time": "2019-04-03T09:48:30.263323Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with len: 100%|██████████| 1/1 [00:00<00:00, 2347.12it/s]\n",
      "Processing text_right with len: 100%|██████████| 5/5 [00:00<00:00, 10270.09it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>99</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left  text_left id_right  text_right  label\n",
       "0      Q2         85     D2-0         126    1.0\n",
       "1      Q2         85     D2-1         128    1.0\n",
       "2      Q2         85     D2-2          99    1.0\n",
       "3      Q2         85     D2-3          78    1.0\n",
       "4      Q2         85     D2-4         312    1.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.apply_on_text(len).frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.317246Z",
     "start_time": "2019-04-03T09:48:30.292585Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing left_length with len: 100%|██████████| 1/1 [00:00<00:00, 1883.39it/s]\n",
      "Processing right_length with len: 100%|██████████| 5/5 [00:00<00:00, 14276.05it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>left_length</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>right_length</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "      <td>126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "      <td>99</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "      <td>78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left  left_length  \\\n",
       "0      Q2  How are the directions of the velocity and for...           85   \n",
       "1      Q2  How are the directions of the velocity and for...           85   \n",
       "2      Q2  How are the directions of the velocity and for...           85   \n",
       "3      Q2  How are the directions of the velocity and for...           85   \n",
       "4      Q2  How are the directions of the velocity and for...           85   \n",
       "\n",
       "  id_right                                         text_right  right_length  \\\n",
       "0     D2-0  In physics , circular motion is a movement of ...           126   \n",
       "1     D2-1  It can be uniform, with constant angular rate ...           128   \n",
       "2     D2-2  The rotation around a fixed axis of a three-di...            99   \n",
       "3     D2-3  The equations of motion describe the movement ...            78   \n",
       "4     D2-4  Examples of circular motion include: an artifi...           312   \n",
       "\n",
       "   label  \n",
       "0    1.0  \n",
       "1    1.0  \n",
       "2    1.0  \n",
       "3    1.0  \n",
       "4    1.0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.apply_on_text(len, rename=('left_length', 'right_length')).frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since adding a column indicating text length is a quite common usage, you may simply do:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.347241Z",
     "start_time": "2019-04-03T09:48:30.322020Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing length_left with len: 100%|██████████| 1/1 [00:00<00:00, 2288.22it/s]\n",
      "Processing length_right with len: 100%|██████████| 5/5 [00:00<00:00, 7361.01it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>length_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>length_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "      <td>126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "      <td>99</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "      <td>78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left  length_left  \\\n",
       "0      Q2  How are the directions of the velocity and for...           85   \n",
       "1      Q2  How are the directions of the velocity and for...           85   \n",
       "2      Q2  How are the directions of the velocity and for...           85   \n",
       "3      Q2  How are the directions of the velocity and for...           85   \n",
       "4      Q2  How are the directions of the velocity and for...           85   \n",
       "\n",
       "  id_right                                         text_right  length_right  \\\n",
       "0     D2-0  In physics , circular motion is a movement of ...           126   \n",
       "1     D2-1  It can be uniform, with constant angular rate ...           128   \n",
       "2     D2-2  The rotation around a fixed axis of a three-di...            99   \n",
       "3     D2-3  The equations of motion describe the movement ...            78   \n",
       "4     D2-4  Examples of circular motion include: an artifi...           312   \n",
       "\n",
       "   label  \n",
       "0    1.0  \n",
       "1    1.0  \n",
       "2    1.0  \n",
       "3    1.0  \n",
       "4    1.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.append_text_length().frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To one-hot encode the labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.365410Z",
     "start_time": "2019-04-03T09:48:30.348612Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q17</td>\n",
       "      <td>how much are the harry potter movies worth</td>\n",
       "      <td>D17-8</td>\n",
       "      <td>According to Rowling, the main theme is death.</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q5</td>\n",
       "      <td>how did apollo creed die</td>\n",
       "      <td>D5-1</td>\n",
       "      <td>He was played by Carl Weathers .</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q9</td>\n",
       "      <td>how a vul works</td>\n",
       "      <td>D9-1</td>\n",
       "      <td>In a VUL, the cash value can be invested in a ...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q5</td>\n",
       "      <td>how did apollo creed die</td>\n",
       "      <td>D5-6</td>\n",
       "      <td>Rocky Balboa is often wrongly credited with po...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0     Q17         how much are the harry potter movies worth    D17-8   \n",
       "1      Q2  How are the directions of the velocity and for...     D2-1   \n",
       "2      Q5                           how did apollo creed die     D5-1   \n",
       "3      Q9                                    how a vul works     D9-1   \n",
       "4      Q5                           how did apollo creed die     D5-6   \n",
       "\n",
       "                                          text_right      label  \n",
       "0     According to Rowling, the main theme is death.  [0, 1, 0]  \n",
       "1  It can be uniform, with constant angular rate ...  [0, 1, 0]  \n",
       "2                   He was played by Carl Weathers .  [0, 1, 0]  \n",
       "3  In a VUL, the cash value can be invested in a ...  [0, 1, 0]  \n",
       "4  Rocky Balboa is often wrongly credited with po...  [0, 1, 0]  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.relation['label'] = data_pack.relation['label'].astype(int)\n",
    "data_pack.one_hot_encode_label(num_classes=3).frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Your own DataPack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `matchzoo.pack` to build your own data pack. Check documentation for more information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.385111Z",
     "start_time": "2019-04-03T09:48:30.367282Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-0</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L-1</td>\n",
       "      <td>R</td>\n",
       "      <td>R-1</td>\n",
       "      <td>r</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L-2</td>\n",
       "      <td>S</td>\n",
       "      <td>R-2</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-3</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-4</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>L-1</td>\n",
       "      <td>R</td>\n",
       "      <td>R-5</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>L-2</td>\n",
       "      <td>S</td>\n",
       "      <td>R-6</td>\n",
       "      <td>u</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-2</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left text_left id_right text_right\n",
       "0     L-0         A      R-0          a\n",
       "1     L-1         R      R-1          r\n",
       "2     L-2         S      R-2          s\n",
       "3     L-0         A      R-3          t\n",
       "4     L-0         A      R-4          e\n",
       "5     L-1         R      R-5          n\n",
       "6     L-2         S      R-6          u\n",
       "7     L-0         A      R-2          s"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame({\n",
    "    'text_left': list('ARSAARSA'),\n",
    "    'text_right': list('arstenus')\n",
    "})\n",
    "my_pack = mz.pack(data)\n",
    "my_pack.frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unpack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Format data in a way so that MatchZoo models can directly fit it. For more details, consult `matchzoo/tutorials/models.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.398913Z",
     "start_time": "2019-04-03T09:48:30.386558Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x, y = data_pack[:3].unpack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.404010Z",
     "start_time": "2019-04-03T09:48:30.400535Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id_left': array(['Q17', 'Q2', 'Q5'], dtype='<U3'),\n",
       " 'text_left': array(['how much are the harry potter movies worth',\n",
       "        'How are the directions of the velocity and force vectors related in a circular motion',\n",
       "        'how did apollo creed die'], dtype='<U85'),\n",
       " 'id_right': array(['D17-8', 'D2-1', 'D5-1'], dtype='<U5'),\n",
       " 'text_right': array(['According to Rowling, the main theme is death.',\n",
       "        'It can be uniform, with constant angular rate of rotation (and constant speed), or non-uniform with a changing rate of rotation.',\n",
       "        'He was played by Carl Weathers .'], dtype='<U128')}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.408655Z",
     "start_time": "2019-04-03T09:48:30.405628Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1],\n",
       "       [1],\n",
       "       [1]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MatchZoo incorporates various datasets that can be loaded as MatchZoo native data structures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.414136Z",
     "start_time": "2019-04-03T09:48:30.410120Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['toy', 'wiki_qa', 'embeddings', 'quora_qp', 'snli']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.datasets.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The toy dataset doesn't need to be downloaded and can be directly used. It's the best choice to get things rolling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.436266Z",
     "start_time": "2019-04-03T09:48:30.415607Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    0.0  \n",
       "1          The ice facade is approximately 60 m high    0.0  \n",
       "2          Ice formations in the Titlis glacier cave    0.0  \n",
       "3  A glacier cave is a cave formed within the ice...    1.0  \n",
       "4  Glacier caves are often called ice caves , but...    0.0  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toy_train_rank = mz.datasets.toy.load_data()\n",
    "toy_train_rank.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.459760Z",
     "start_time": "2019-04-03T09:48:30.437823Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>[0, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right   label  \n",
       "0  A partly submerged glacier cave on Perito More...  [1, 0]  \n",
       "1          The ice facade is approximately 60 m high  [1, 0]  \n",
       "2          Ice formations in the Titlis glacier cave  [1, 0]  \n",
       "3  A glacier cave is a cave formed within the ice...  [0, 1]  \n",
       "4  Glacier caves are often called ice caves , but...  [1, 0]  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toy_dev_classification, classes = mz.datasets.toy.load_data(\n",
    "    stage='train', task='classification', return_classes=True)\n",
    "toy_dev_classification.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.465063Z",
     "start_time": "2019-04-03T09:48:30.461565Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[False, True]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other larger datasets will be automatically downloaded the first time you use it. Run the following lines to trigger downloading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:30.648869Z",
     "start_time": "2019-04-03T09:48:30.466465Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-0</td>\n",
       "      <td>Cross section of sclerenchyma fibers in plant ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-1</td>\n",
       "      <td>Microscopic view of a histologic specimen of h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-2</td>\n",
       "      <td>In Biology , Tissue is a cellular organization...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-3</td>\n",
       "      <td>A tissue is an ensemble of similar cells from ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-4</td>\n",
       "      <td>Organs are then formed by the functional group...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                    text_left id_right  \\\n",
       "0      Q8  How are epithelial tissues joined together?     D8-0   \n",
       "1      Q8  How are epithelial tissues joined together?     D8-1   \n",
       "2      Q8  How are epithelial tissues joined together?     D8-2   \n",
       "3      Q8  How are epithelial tissues joined together?     D8-3   \n",
       "4      Q8  How are epithelial tissues joined together?     D8-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  Cross section of sclerenchyma fibers in plant ...      0  \n",
       "1  Microscopic view of a histologic specimen of h...      0  \n",
       "2  In Biology , Tissue is a cellular organization...      0  \n",
       "3  A tissue is an ensemble of similar cells from ...      0  \n",
       "4  Organs are then formed by the functional group...      0  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wiki_dev_entailment_rank = mz.datasets.wiki_qa.load_data(stage='dev')\n",
    "wiki_dev_entailment_rank.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:34.860314Z",
     "start_time": "2019-04-03T09:48:30.651265Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L-0</td>\n",
       "      <td>This church choir sings to the masses as they ...</td>\n",
       "      <td>R-0</td>\n",
       "      <td>The church has cracks in the ceiling.</td>\n",
       "      <td>[0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L-0</td>\n",
       "      <td>This church choir sings to the masses as they ...</td>\n",
       "      <td>R-1</td>\n",
       "      <td>The church is filled with song.</td>\n",
       "      <td>[1, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L-0</td>\n",
       "      <td>This church choir sings to the masses as they ...</td>\n",
       "      <td>R-2</td>\n",
       "      <td>A choir singing at a baseball game.</td>\n",
       "      <td>[0, 1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L-1</td>\n",
       "      <td>A woman with a green headscarf, blue shirt and...</td>\n",
       "      <td>R-3</td>\n",
       "      <td>The woman is young.</td>\n",
       "      <td>[0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L-1</td>\n",
       "      <td>A woman with a green headscarf, blue shirt and...</td>\n",
       "      <td>R-4</td>\n",
       "      <td>The woman is very happy.</td>\n",
       "      <td>[1, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0     L-0  This church choir sings to the masses as they ...      R-0   \n",
       "1     L-0  This church choir sings to the masses as they ...      R-1   \n",
       "2     L-0  This church choir sings to the masses as they ...      R-2   \n",
       "3     L-1  A woman with a green headscarf, blue shirt and...      R-3   \n",
       "4     L-1  A woman with a green headscarf, blue shirt and...      R-4   \n",
       "\n",
       "                              text_right         label  \n",
       "0  The church has cracks in the ceiling.  [0, 0, 1, 0]  \n",
       "1        The church is filled with song.  [1, 0, 0, 0]  \n",
       "2    A choir singing at a baseball game.  [0, 1, 0, 0]  \n",
       "3                    The woman is young.  [0, 0, 1, 0]  \n",
       "4               The woman is very happy.  [1, 0, 0, 0]  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snli_test_classification, classes = mz.datasets.snli.load_data(\n",
    "    stage='test', task='classification', return_classes=True)\n",
    "snli_test_classification.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:34.864841Z",
     "start_time": "2019-04-03T09:48:34.861733Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['entailment', 'contradiction', 'neutral', '-']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T09:59:51.361693Z",
     "start_time": "2019-01-11T09:59:51.359438Z"
    }
   },
   "source": [
    "## Preprocessors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`matchzoo.preprocessors` are responsible for transforming data into correct forms that `matchzoo.models`. `BasicPreprocessor` is used for models with common forms, and some other models have customized preprocessors made just for them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:34.872121Z",
     "start_time": "2019-04-03T09:48:34.866900Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.preprocessors.dssm_preprocessor.DSSMPreprocessor,\n",
       " matchzoo.preprocessors.naive_preprocessor.NaivePreprocessor,\n",
       " matchzoo.preprocessors.basic_preprocessor.BasicPreprocessor,\n",
       " matchzoo.preprocessors.cdssm_preprocessor.CDSSMPreprocessor]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.preprocessors.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When in doubt, use the default preprocessor a model class provides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:34.878403Z",
     "start_time": "2019-04-03T09:48:34.875228Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.models.Naive.get_default_preprocessor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A preprocessor should be used in two steps. First, `fit`, then, `transform`. `fit` collects information into `context`, which includes everything the preprocessor needs to `transform` together with other useful information for later use. `fit` will only change the preprocessor's inner state but not the input data. In contrast, `transform` returns a modified copy of the input data without changing the preprocessor's inner state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:34.963013Z",
     "start_time": "2019-04-03T09:48:34.880336Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 13/13 [00:00<00:00, 1283.21it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 100/100 [00:00<00:00, 4023.97it/s]\n",
      "Processing text_right with append: 100%|██████████| 100/100 [00:00<00:00, 98527.23it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 100/100 [00:00<00:00, 33645.95it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 105596.78it/s]\n",
      "Processing text_left with extend: 100%|██████████| 13/13 [00:00<00:00, 27962.03it/s]\n",
      "Processing text_right with extend: 100%|██████████| 100/100 [00:00<00:00, 106725.29it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 1665/1665 [00:00<00:00, 1289661.34it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x1331c92e8>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x133f96e80>,\n",
       " 'vocab_size': 285,\n",
       " 'embedding_input_dim': 285,\n",
       " 'input_shapes': [(30,), (30,)]}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_raw = mz.datasets.toy.load_data('train', 'ranking')\n",
    "test_raw = mz.datasets.toy.load_data('test', 'ranking')\n",
    "preprocessor.fit(train_raw)\n",
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.049852Z",
     "start_time": "2019-04-03T09:48:34.965170Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 13/13 [00:00<00:00, 7532.25it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 100/100 [00:00<00:00, 4073.29it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 79182.63it/s]\n",
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 22174.03it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 55180.95it/s]\n",
      "Processing length_left with len: 100%|██████████| 13/13 [00:00<00:00, 22776.09it/s]\n",
      "Processing length_right with len: 100%|██████████| 100/100 [00:00<00:00, 171546.18it/s]\n",
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 15552.18it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 73856.38it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 3/3 [00:00<00:00, 2962.08it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 20/20 [00:00<00:00, 4877.66it/s]\n",
      "Processing text_right with transform: 100%|██████████| 20/20 [00:00<00:00, 32896.50it/s]\n",
      "Processing text_left with transform: 100%|██████████| 3/3 [00:00<00:00, 4703.89it/s]\n",
      "Processing text_right with transform: 100%|██████████| 20/20 [00:00<00:00, 37600.22it/s]\n",
      "Processing length_left with len: 100%|██████████| 3/3 [00:00<00:00, 9612.61it/s]\n",
      "Processing length_right with len: 100%|██████████| 20/20 [00:00<00:00, 22221.48it/s]\n",
      "Processing text_left with transform: 100%|██████████| 3/3 [00:00<00:00, 4048.56it/s]\n",
      "Processing text_right with transform: 100%|██████████| 20/20 [00:00<00:00, 20198.91it/s]\n"
     ]
    }
   ],
   "source": [
    "train_preprocessed = preprocessor.transform(train_raw)\n",
    "test_preprocessed = preprocessor.transform(test_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.354275Z",
     "start_time": "2019-04-03T09:48:35.052022Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter \"task\" set to Ranking Task.\n",
      "Parameter \"input_shapes\" set to [(30,), (30,)].\n",
      "Epoch 1/1\n",
      "100/100 [==============================] - 0s 970us/step - loss: 31720.4902\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{mean_average_precision(0.0): 0.08333333333333333}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = mz.models.Naive()\n",
    "model.guess_and_fill_missing_params()\n",
    "model.build()\n",
    "model.compile()\n",
    "x_train, y_train = train_preprocessed.unpack()\n",
    "model.fit(x_train, y_train)\n",
    "x_test, y_test = test_preprocessed.unpack()\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Processor Units"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessors utilize `mz.processor_units` to transform data. Processor units correspond to specific transformations and you may use them independently to preprocess a data pack. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.384276Z",
     "start_time": "2019-04-03T09:48:35.356163Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    0.0  \n",
       "1          The ice facade is approximately 60 m high    0.0  \n",
       "2          Ice formations in the Titlis glacier cave    0.0  \n",
       "3  A glacier cave is a cave formed within the ice...    1.0  \n",
       "4  Glacier caves are often called ice caves , but...    0.0  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack = mz.datasets.toy.load_data()\n",
    "data_pack.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.431269Z",
     "start_time": "2019-04-03T09:48:35.386044Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 7794.99it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 5312.00it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[A, partly, submerged, glacier, cave, on, Peri...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[The, ice, facade, is, approximately, 60, m, h...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[Ice, formations, in, the, Titlis, glacier, cave]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[A, glacier, cave, is, a, cave, formed, within...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[Glacier, caves, are, often, called, ice, cave...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                              text_left id_right  \\\n",
       "0      Q1  [how, are, glacier, caves, formed, ?]     D1-0   \n",
       "1      Q1  [how, are, glacier, caves, formed, ?]     D1-1   \n",
       "2      Q1  [how, are, glacier, caves, formed, ?]     D1-2   \n",
       "3      Q1  [how, are, glacier, caves, formed, ?]     D1-3   \n",
       "4      Q1  [how, are, glacier, caves, formed, ?]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [A, partly, submerged, glacier, cave, on, Peri...    0.0  \n",
       "1  [The, ice, facade, is, approximately, 60, m, h...    0.0  \n",
       "2  [Ice, formations, in, the, Titlis, glacier, cave]    0.0  \n",
       "3  [A, glacier, cave, is, a, cave, formed, within...    1.0  \n",
       "4  [Glacier, caves, are, often, called, ice, cave...    0.0  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = mz.preprocessors.units.Tokenize()\n",
    "data_pack.apply_on_text(tokenizer.transform, inplace=True)\n",
    "data_pack.frame[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.462052Z",
     "start_time": "2019-04-03T09:48:35.433979Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 17737.79it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 63811.11it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[a, partly, submerged, glacier, cave, on, peri...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[the, ice, facade, is, approximately, 60, m, h...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[ice, formations, in, the, titlis, glacier, cave]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[a, glacier, cave, is, a, cave, formed, within...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[glacier, caves, are, often, called, ice, cave...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                              text_left id_right  \\\n",
       "0      Q1  [how, are, glacier, caves, formed, ?]     D1-0   \n",
       "1      Q1  [how, are, glacier, caves, formed, ?]     D1-1   \n",
       "2      Q1  [how, are, glacier, caves, formed, ?]     D1-2   \n",
       "3      Q1  [how, are, glacier, caves, formed, ?]     D1-3   \n",
       "4      Q1  [how, are, glacier, caves, formed, ?]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [a, partly, submerged, glacier, cave, on, peri...    0.0  \n",
       "1  [the, ice, facade, is, approximately, 60, m, h...    0.0  \n",
       "2  [ice, formations, in, the, titlis, glacier, cave]    0.0  \n",
       "3  [a, glacier, cave, is, a, cave, formed, within...    1.0  \n",
       "4  [glacier, caves, are, often, called, ice, cave...    0.0  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lower_caser = mz.preprocessors.units.Lowercase()\n",
    "data_pack.apply_on_text(lower_caser.transform, inplace=True)\n",
    "data_pack.frame[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or use `chain_transform` to apply multiple processor units at one time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.516262Z",
     "start_time": "2019-04-03T09:48:35.463597Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase: 100%|██████████| 13/13 [00:00<00:00, 6659.25it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase: 100%|██████████| 100/100 [00:00<00:00, 4481.48it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[a, partly, submerged, glacier, cave, on, peri...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[the, ice, facade, is, approximately, 60, m, h...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[ice, formations, in, the, titlis, glacier, cave]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[a, glacier, cave, is, a, cave, formed, within...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[glacier, caves, are, often, called, ice, cave...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                              text_left id_right  \\\n",
       "0      Q1  [how, are, glacier, caves, formed, ?]     D1-0   \n",
       "1      Q1  [how, are, glacier, caves, formed, ?]     D1-1   \n",
       "2      Q1  [how, are, glacier, caves, formed, ?]     D1-2   \n",
       "3      Q1  [how, are, glacier, caves, formed, ?]     D1-3   \n",
       "4      Q1  [how, are, glacier, caves, formed, ?]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [a, partly, submerged, glacier, cave, on, peri...    0.0  \n",
       "1  [the, ice, facade, is, approximately, 60, m, h...    0.0  \n",
       "2  [ice, formations, in, the, titlis, glacier, cave]    0.0  \n",
       "3  [a, glacier, cave, is, a, cave, formed, within...    1.0  \n",
       "4  [glacier, caves, are, often, called, ice, cave...    0.0  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack = mz.datasets.toy.load_data()\n",
    "chain = mz.chain_transform([mz.preprocessors.units.Tokenize(),\n",
    "                           mz.preprocessors.units.Lowercase()])\n",
    "data_pack.apply_on_text(chain, inplace=True)\n",
    "data_pack.frame[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that some processor units are stateful so we have to `fit` them before using their `transform`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.520779Z",
     "start_time": "2019-04-03T09:48:35.517524Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matchzoo.preprocessors.units.stateful_unit.StatefulUnit"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.preprocessors.units.Vocabulary.__base__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.534682Z",
     "start_time": "2019-04-03T09:48:35.522220Z"
    }
   },
   "outputs": [],
   "source": [
    "vocab_unit = mz.preprocessors.units.Vocabulary()\n",
    "texts = data_pack.frame()[['text_left', 'text_right']]\n",
    "all_tokens = texts.sum().sum()\n",
    "vocab_unit.fit(all_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Such `StatefulProcessorUnit` will save information in its `state` when `fit`, similar to the `context` of a preprocessor. In our case here, the vocabulary unit will save a term to index mapping, and a index to term mapping, called `term_index` and `index_term` respectively. Then we can proceed transforming a data pack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.539922Z",
     "start_time": "2019-04-03T09:48:35.536354Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "how 153\n",
      "are 604\n",
      "glacier 55\n"
     ]
    }
   ],
   "source": [
    "for vocab in 'how', 'are', 'glacier':\n",
    "    print(vocab, vocab_unit.state['term_index'][vocab])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.567959Z",
     "start_time": "2019-04-03T09:48:35.542411Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 18211.74it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 58408.36it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[153, 604, 55, 448, 752, 629]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[688, 278, 896, 55, 165, 25, 493, 851, 55, 509]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[153, 604, 55, 448, 752, 629]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[371, 800, 827, 185, 87, 76, 901, 639]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[153, 604, 55, 448, 752, 629]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[800, 378, 394, 371, 213, 55, 165]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[153, 604, 55, 448, 752, 629]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[688, 55, 165, 185, 688, 165, 752, 712, 371, 8...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[153, 604, 55, 448, 752, 629]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[55, 448, 604, 856, 389, 800, 448, 808, 72, 33...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  [153, 604, 55, 448, 752, 629]     D1-0   \n",
       "1      Q1  [153, 604, 55, 448, 752, 629]     D1-1   \n",
       "2      Q1  [153, 604, 55, 448, 752, 629]     D1-2   \n",
       "3      Q1  [153, 604, 55, 448, 752, 629]     D1-3   \n",
       "4      Q1  [153, 604, 55, 448, 752, 629]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0    [688, 278, 896, 55, 165, 25, 493, 851, 55, 509]    0.0  \n",
       "1             [371, 800, 827, 185, 87, 76, 901, 639]    0.0  \n",
       "2                 [800, 378, 394, 371, 213, 55, 165]    0.0  \n",
       "3  [688, 55, 165, 185, 688, 165, 752, 712, 371, 8...    1.0  \n",
       "4  [55, 448, 604, 856, 389, 800, 448, 808, 72, 33...    0.0  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.apply_on_text(vocab_unit.transform, inplace=True)\n",
    "data_pack.frame()[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since this usage is quite common, we wrapped a function to do the same thing. For other stateful units, consult their documentations and try `mz.build_unit_from_data_pack`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.618784Z",
     "start_time": "2019-04-03T09:48:35.569635Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with extend: 100%|██████████| 13/13 [00:00<00:00, 18205.66it/s]\n",
      "Processing text_right with extend: 100%|██████████| 100/100 [00:00<00:00, 146398.05it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 13893/13893 [00:00<00:00, 3564222.00it/s]\n",
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 12546.24it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 23825.86it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[12, 29, 66, 24, 72, 54, 33, 51, 29, 21, 25, 1...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[37, 4, 32, 29, 7, 49, 32, 29, 68, 24, 49, 24,...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[8, 49, 32, 29, 68, 1, 72, 70, 24, 54, 7, 1, 2...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[12, 29, 11, 33, 24, 49, 7, 32, 72, 29, 49, 24...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[6, 33, 24, 49, 7, 32, 72, 29, 49, 24, 14, 32,...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0      Q1  [4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...     D1-0   \n",
       "1      Q1  [4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...     D1-1   \n",
       "2      Q1  [4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...     D1-2   \n",
       "3      Q1  [4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...     D1-3   \n",
       "4      Q1  [4, 1, 55, 29, 24, 72, 32, 29, 11, 33, 24, 49,...     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [12, 29, 66, 24, 72, 54, 33, 51, 29, 21, 25, 1...    0.0  \n",
       "1  [37, 4, 32, 29, 7, 49, 32, 29, 68, 24, 49, 24,...    0.0  \n",
       "2  [8, 49, 32, 29, 68, 1, 72, 70, 24, 54, 7, 1, 2...    0.0  \n",
       "3  [12, 29, 11, 33, 24, 49, 7, 32, 72, 29, 49, 24...    1.0  \n",
       "4  [6, 33, 24, 49, 7, 32, 72, 29, 49, 24, 14, 32,...    0.0  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack = mz.datasets.toy.load_data()\n",
    "vocab_unit = mz.build_vocab_unit(data_pack)\n",
    "data_pack.apply_on_text(vocab_unit.transform).frame[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T14:34:11.732867Z",
     "start_time": "2019-01-11T14:34:11.730142Z"
    }
   },
   "source": [
    "# DataGenerator\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some MatchZoo models (e.g. DRMM, MatchPyramid) require batch-wise information for training so using `fit_generator` instead of using `fit` is necessary. In addition, sometimes your memory just can't hold all transformed data so to delay a part of the preprocessing process is necessary. \n",
    "\n",
    "MatchZoo provides `DataGenerator` as an alternative. Instead of `fit`, you may do a `fit_generator` that takes a data generator that `unpack` data on the fly.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.638900Z",
     "start_time": "2019-04-03T09:48:35.620736Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "\r",
      "100/100 [==============================] - 0s 18us/step - loss: 30618.9355\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1085f8438>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train, y_train = train_preprocessed.unpack()\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.779263Z",
     "start_time": "2019-04-03T09:48:35.640340Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "\r",
      "1/1 [==============================] - 0s 18ms/step - loss: 29543.1270\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x10860dba8>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_gen = mz.DataGenerator(train_preprocessed)\n",
    "model.fit_generator(data_gen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data preprocessing of `DSSM` eats a lot of memory, but we can workaround that using the callback hook of `DataGenerator`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.855202Z",
     "start_time": "2019-04-03T09:48:35.780878Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.preprocessors.DSSMPreprocessor(with_word_hashing=False)\n",
    "data = preprocessor.fit_transform(train_raw, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.983719Z",
     "start_time": "2019-04-03T09:48:35.857115Z"
    }
   },
   "outputs": [],
   "source": [
    "dssm = mz.models.DSSM()\n",
    "dssm.params['task'] = mz.tasks.Ranking()\n",
    "dssm.params.update(preprocessor.context)\n",
    "dssm.build()\n",
    "dssm.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:35.989452Z",
     "start_time": "2019-04-03T09:48:35.985687Z"
    }
   },
   "outputs": [],
   "source": [
    "term_index = preprocessor.context['vocab_unit'].state['term_index']\n",
    "hashing_unit = mz.preprocessors.units.WordHashing(term_index)\n",
    "data_generator = mz.DataGenerator(\n",
    "    data,\n",
    "    batch_size=4,\n",
    "    callbacks=[\n",
    "        mz.data_generator.callbacks.LambdaCallback(\n",
    "            on_batch_data_pack=lambda dp: dp.apply_on_text(\n",
    "                hashing_unit.transform, inplace=True, verbose=0)\n",
    "        )\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:37.417180Z",
     "start_time": "2019-04-03T09:48:35.991315Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "25/25 [==============================] - 1s 33ms/step - loss: 0.0471\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1337f78d0>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dssm.fit_generator(data_generator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, losses like `RankHingeLoss` and `RankCrossEntropyLoss` have to be used with `DataGenerator` with `mode='pair'`, since batch-wise information are needed and computed on the fly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:37.518125Z",
     "start_time": "2019-04-03T09:48:37.419095Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 13/13 [00:00<00:00, 3234.62it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 100/100 [00:00<00:00, 4000.52it/s]\n",
      "Processing text_right with append: 100%|██████████| 100/100 [00:00<00:00, 194993.21it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 100/100 [00:00<00:00, 98898.94it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 89698.55it/s]\n",
      "Processing text_left with extend: 100%|██████████| 13/13 [00:00<00:00, 28622.55it/s]\n",
      "Processing text_right with extend: 100%|██████████| 100/100 [00:00<00:00, 195995.51it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 1665/1665 [00:00<00:00, 2223341.66it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 13/13 [00:00<00:00, 6020.98it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 100/100 [00:00<00:00, 5079.76it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 106373.42it/s]\n",
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 21274.27it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 56397.79it/s]\n",
      "Processing length_left with len: 100%|██████████| 13/13 [00:00<00:00, 24863.64it/s]\n",
      "Processing length_right with len: 100%|██████████| 100/100 [00:00<00:00, 142228.01it/s]\n",
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 21526.23it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 84767.66it/s]\n"
     ]
    }
   ],
   "source": [
    "num_neg = 4\n",
    "task = mz.tasks.Ranking(loss=mz.losses.RankHingeLoss(num_neg=num_neg))\n",
    "preprocessor = model.get_default_preprocessor()\n",
    "train_processed = preprocessor.fit_transform(train_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:37.588378Z",
     "start_time": "2019-04-03T09:48:37.519898Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = mz.models.Naive()\n",
    "model.params['task'] = task\n",
    "model.params.update(preprocessor.context)\n",
    "model.build()\n",
    "model.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:37.627721Z",
     "start_time": "2019-04-03T09:48:37.590091Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_gen = mz.DataGenerator(\n",
    "    train_processed,\n",
    "    mode='pair',\n",
    "    num_neg=num_neg,\n",
    "    num_dup=2,\n",
    "    batch_size=32\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:48:38.015010Z",
     "start_time": "2019-04-03T09:48:37.629412Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1/1 [==============================] - 0s 222ms/step - loss: 28.1434\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1332f9ba8>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(data_gen)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "matchzoo",
   "language": "python",
   "name": "matchzoo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "notify_time": "5",
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "171px",
    "left": "2190px",
    "right": "20px",
    "top": "120px",
    "width": "349px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}