seq2seq_fastai_custom_character_tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text.all import *\n",
    "import os\n",
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=np.VisibleDeprecationWarning)\n",
    "%config Completer.use_jedi = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharacterTokenizer():\n",
    "        \n",
    "    def __call__(self, items):\n",
    "        \n",
    "        # List where I temporarly store the tokens ['xxbos', 'h', 'e', 'l', 'l', 'o', 'xxeos'] as \n",
    "        # they are being parsed.\n",
    "        final_list = []\n",
    "        \n",
    "        # We don't want to mess with the special fastai tokens\n",
    "        special_chars = ['xxunk','xxpad','xxbos','xxeos','xxfld','xxrep','xxwrep','xxup','xxmaj']\n",
    "        \n",
    "        # Break up string into words, if word in special_chars dont touch it. Otherwise break up each\n",
    "        # word into each character.\n",
    "        for words in items:\n",
    "            tmp = []\n",
    "            for word in words.split():\n",
    "                if word not in special_chars:\n",
    "                    for char in word:\n",
    "                        tmp.append(char)\n",
    "                else:\n",
    "                    tmp.append(word)\n",
    "            # tmp has each token 'xxbos', 'xxmaj', 'h', 'e', 'l', 'l', 'o', ',', 'w', 'h', ....]\n",
    "            # We need to put the tmp list into another list to generate a generator below\n",
    "            final_list.append(tmp)\n",
    "        \n",
    "        # Returns a generator\n",
    "        return (t for t in final_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_txt</th>\n",
       "      <th>to_txt</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hello how are you?</td>\n",
       "      <td>I am doing fine.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Is it going to rain today?</td>\n",
       "      <td>Let me pull up the weather.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How do fastai DataBlocks work?</td>\n",
       "      <td>Not sure, I'm still learning.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         from_txt                         to_txt  is_valid\n",
       "0              Hello how are you?               I am doing fine.      True\n",
       "1      Is it going to rain today?    Let me pull up the weather.      True\n",
       "2  How do fastai DataBlocks work?  Not sure, I'm still learning.      True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example of a validation dataset. In this case it is a list of python dictionaries.\n",
    "# You should be able to adapt this to your usecase if you can read in a file using Pandas.\n",
    "valid_jsons = [\n",
    "    {\"from_txt\":\"Hello how are you?\",\"to_txt\":\"I am doing fine.\"},\n",
    "    {\"from_txt\":\"Is it going to rain today?\",\"to_txt\":\"Let me pull up the weather.\"},\n",
    "    {\"from_txt\":\"How do fastai DataBlocks work?\",\"to_txt\":\"Not sure, I'm still learning.\"}\n",
    "]\n",
    "\n",
    "# Read the list of dicts into a Data Frame\n",
    "df_valid = pd.DataFrame(valid_jsons)\n",
    "\n",
    "# Add a is_valid colum. The ColSplitter() function below expects this column.\n",
    "df_valid['is_valid'] = True\n",
    "\n",
    "# View our work\n",
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_txt</th>\n",
       "      <th>to_txt</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hello, where is the closest McDonald's?</td>\n",
       "      <td>Let me find you that on Google Maps.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Is it going to snow today?</td>\n",
       "      <td>Let me pull up the weather.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How much coffee is safe to drink?</td>\n",
       "      <td>As much as you need to learn the Fastai Library.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  from_txt  \\\n",
       "0  Hello, where is the closest McDonald's?   \n",
       "1               Is it going to snow today?   \n",
       "2        How much coffee is safe to drink?   \n",
       "\n",
       "                                             to_txt  is_valid  \n",
       "0              Let me find you that on Google Maps.     False  \n",
       "1                       Let me pull up the weather.     False  \n",
       "2  As much as you need to learn the Fastai Library.     False  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example of a test dataset. In this case it is a list of python dictionaries.\n",
    "# You should be able to adapt this to your usecase if you can read in a file using Pandas.\n",
    "test_jsons = [\n",
    "    {\"from_txt\":\"Hello, where is the closest McDonald's?\",\"to_txt\":\"Let me find you that on Google Maps.\"},\n",
    "    {\"from_txt\":\"Is it going to snow today?\",\"to_txt\":\"Let me pull up the weather.\"},\n",
    "    {\"from_txt\":\"How much coffee is safe to drink?\",\"to_txt\":\"As much as you need to learn the Fastai Library.\"}\n",
    "]\n",
    "\n",
    "# Read the list of dicts into a Data Frame\n",
    "df_train = pd.DataFrame(test_jsons)\n",
    "\n",
    "\n",
    "# Add a is_valid colum\n",
    "df_train['is_valid'] = False\n",
    "\n",
    "# View our work\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>from_txt</th>\n",
       "      <th>to_txt</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hello, where is the closest McDonald's?</td>\n",
       "      <td>Let me find you that on Google Maps.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Is it going to snow today?</td>\n",
       "      <td>Let me pull up the weather.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How much coffee is safe to drink?</td>\n",
       "      <td>As much as you need to learn the Fastai Library.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Hello how are you?</td>\n",
       "      <td>I am doing fine.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is it going to rain today?</td>\n",
       "      <td>Let me pull up the weather.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  from_txt  \\\n",
       "0  Hello, where is the closest McDonald's?   \n",
       "1               Is it going to snow today?   \n",
       "2        How much coffee is safe to drink?   \n",
       "3                       Hello how are you?   \n",
       "4               Is it going to rain today?   \n",
       "\n",
       "                                             to_txt  is_valid  \n",
       "0              Let me find you that on Google Maps.     False  \n",
       "1                       Let me pull up the weather.     False  \n",
       "2  As much as you need to learn the Fastai Library.     False  \n",
       "3                                  I am doing fine.      True  \n",
       "4                       Let me pull up the weather.      True  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Stack the two dataframes into one.\n",
    "df = pd.concat([df_train,df_valid], ignore_index=True)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "logs = DataBlock(\n",
    "        \n",
    "    blocks=(\n",
    "        TextBlock.from_df('from_txt', is_lm=False, tok=CharacterTokenizer()),\n",
    "        TextBlock.from_df('to_txt'  , is_lm=False, tok=CharacterTokenizer())),\n",
    "    \n",
    "    # The TestBlock tokenization process puts tokenized inputs into a column called text. \n",
    "    # The ColReader for get_x will always reference text, even if the original text inputs \n",
    "    # were in a column with another name in the dataframe.\n",
    "    get_x=ColReader('text'),\n",
    "    get_y=ColReader('text'),\n",
    "    \n",
    "    # The dataframe needs to have a is_valid column for this to work.\n",
    "    splitter=ColSplitter()\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#logs.summary(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls = logs.dataloaders(df,bs=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xxbos xxmaj h e l l o xxunk w h e r e i s t h e c l o s e s t xxunk c d o n a l d xxunk s ?</td>\n",
       "      <td>xxbos xxmaj l e t m e f i n d y o u t h a t o n xxmaj g o o g l e xxmaj m a p s .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xxbos xxmaj h o w xxunk xxunk c h c o f f e e i s s a f e t o d r i n k ?</td>\n",
       "      <td>xxbos xxmaj a s m u xxunk h a s y o u n e e d t o l e a r n t h e xxmaj f a s t a i xxmaj l i xxunk r a r y .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch(max_n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deeplearning_pytorch",
   "language": "python",
   "name": "deeplearning_pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}