char-level-cnn/notebooks/char-level-text-preprocess-with-keras-summary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this note, I will write how to preprocess the char level text with keras. \n",
    "\n",
    "First we preprocess the data to get training texts and labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Wall St. Bears Claw Back Into the Black (Reuters)</td>\n",
       "      <td>Reuters - Short-sellers, Wall Street's dwindli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>Carlyle Looks Toward Commercial Aerospace (Reu...</td>\n",
       "      <td>Reuters - Private investment firm Carlyle Grou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Oil and Economy Cloud Stocks' Outlook (Reuters)</td>\n",
       "      <td>Reuters - Soaring crude prices plus worries\\ab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Iraq Halts Oil Exports from Main Southern Pipe...</td>\n",
       "      <td>Reuters - Authorities have halted oil export\\f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>Oil prices soar to all-time record, posing new...</td>\n",
       "      <td>AFP - Tearaway world oil prices, toppling reco...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0                                                  1  \\\n",
       "0  3  Wall St. Bears Claw Back Into the Black (Reuters)   \n",
       "1  3  Carlyle Looks Toward Commercial Aerospace (Reu...   \n",
       "2  3    Oil and Economy Cloud Stocks' Outlook (Reuters)   \n",
       "3  3  Iraq Halts Oil Exports from Main Southern Pipe...   \n",
       "4  3  Oil prices soar to all-time record, posing new...   \n",
       "\n",
       "                                                   2  \n",
       "0  Reuters - Short-sellers, Wall Street's dwindli...  \n",
       "1  Reuters - Private investment firm Carlyle Grou...  \n",
       "2  Reuters - Soaring crude prices plus worries\\ab...  \n",
       "3  Reuters - Authorities have halted oil export\\f...  \n",
       "4  AFP - Tearaway world oil prices, toppling reco...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "data_source = '../data/ag_news_csv/train.csv'\n",
    "\n",
    "train_df = pd.read_csv(data_source, header=None)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Wall St. Bears Claw Back Into the Black (Reute...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>Carlyle Looks Toward Commercial Aerospace (Reu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Oil and Economy Cloud Stocks' Outlook (Reuters...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Iraq Halts Oil Exports from Main Southern Pipe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>Oil prices soar to all-time record, posing new...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0                                                  1\n",
       "0  3  Wall St. Bears Claw Back Into the Black (Reute...\n",
       "1  3  Carlyle Looks Toward Commercial Aerospace (Reu...\n",
       "2  3  Oil and Economy Cloud Stocks' Outlook (Reuters...\n",
       "3  3  Iraq Halts Oil Exports from Main Southern Pipe...\n",
       "4  3  Oil prices soar to all-time record, posing new..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate column 1 and column 2 as one text\n",
    "train_df[1] = train_df[1] + train_df[2]\n",
    "train_df = train_df.drop([2], axis=1)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train data to index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120000\n",
      "[\"wall st. bears claw back into the black (reuters)reuters - short-sellers, wall street's dwindling\\\\band of ultra-cynics, are seeing green again.\", 'carlyle looks toward commercial aerospace (reuters)reuters - private investment firm carlyle group,\\\\which has a reputation for making well-timed and occasionally\\\\controversial plays in the defense industry, has quietly placed\\\\its bets on another part of the market.']\n"
     ]
    }
   ],
   "source": [
    "texts = train_df[1].values # texts contrain all setences\n",
    "texts = [s.lower() for s in texts] # convert to lower case \n",
    "print(len(texts))\n",
    "print(texts[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When initializing the `Tokenizer`, there are only two parameter important.\n",
    "- `char_level=True`: this can make later `tk.texts_to_sequences()` to process sentence in char level.\n",
    "- `oov_token='UNK'`: this will add a `UNK` key in the vocabulary. We can call it by `tk.oov_token`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialization\n",
    "tk = Tokenizer(num_words=None, char_level=True, oov_token='UNK')\n",
    "# Fitting\n",
    "tk.fit_on_texts(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' ': 1,\n",
       " '!': 54,\n",
       " '\"': 48,\n",
       " '#': 34,\n",
       " '$': 50,\n",
       " '&': 42,\n",
       " \"'\": 39,\n",
       " '(': 36,\n",
       " ')': 37,\n",
       " '*': 56,\n",
       " ',': 25,\n",
       " '-': 26,\n",
       " '.': 22,\n",
       " '/': 44,\n",
       " '0': 29,\n",
       " '1': 35,\n",
       " '2': 38,\n",
       " '3': 28,\n",
       " '4': 46,\n",
       " '5': 45,\n",
       " '6': 47,\n",
       " '7': 49,\n",
       " '8': 51,\n",
       " '9': 31,\n",
       " ':': 43,\n",
       " ';': 27,\n",
       " '=': 52,\n",
       " '?': 53,\n",
       " 'UNK': 57,\n",
       " '\\\\': 41,\n",
       " '_': 55,\n",
       " 'a': 3,\n",
       " 'b': 21,\n",
       " 'c': 13,\n",
       " 'd': 11,\n",
       " 'e': 2,\n",
       " 'f': 18,\n",
       " 'g': 17,\n",
       " 'h': 12,\n",
       " 'i': 5,\n",
       " 'j': 32,\n",
       " 'k': 24,\n",
       " 'l': 10,\n",
       " 'm': 16,\n",
       " 'n': 8,\n",
       " 'o': 7,\n",
       " 'p': 15,\n",
       " 'q': 33,\n",
       " 'r': 9,\n",
       " 's': 6,\n",
       " 't': 4,\n",
       " 'u': 14,\n",
       " 'v': 23,\n",
       " 'w': 20,\n",
       " 'x': 30,\n",
       " 'y': 19,\n",
       " 'z': 40}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List the vocabulary\n",
    "tk.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But we already have a character list, so we do ont use `tk.word_index` as our vocabulary. We will replace it with `char_dict`.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'!': 41,\n",
       " '\"': 45,\n",
       " '#': 51,\n",
       " '$': 52,\n",
       " '%': 53,\n",
       " '&': 55,\n",
       " \"'\": 44,\n",
       " '(': 64,\n",
       " ')': 65,\n",
       " '*': 56,\n",
       " '+': 59,\n",
       " ',': 38,\n",
       " '-': 60,\n",
       " '.': 40,\n",
       " '/': 46,\n",
       " '0': 27,\n",
       " '1': 28,\n",
       " '2': 29,\n",
       " '3': 30,\n",
       " '4': 31,\n",
       " '5': 32,\n",
       " '6': 33,\n",
       " '7': 34,\n",
       " '8': 35,\n",
       " '9': 36,\n",
       " ':': 43,\n",
       " ';': 39,\n",
       " '<': 62,\n",
       " '=': 61,\n",
       " '>': 63,\n",
       " '?': 42,\n",
       " '@': 50,\n",
       " '[': 66,\n",
       " '\\\\': 47,\n",
       " ']': 67,\n",
       " '^': 54,\n",
       " '_': 49,\n",
       " '`': 58,\n",
       " 'a': 1,\n",
       " 'b': 2,\n",
       " 'c': 3,\n",
       " 'd': 4,\n",
       " 'e': 5,\n",
       " 'f': 6,\n",
       " 'g': 7,\n",
       " 'h': 8,\n",
       " 'i': 9,\n",
       " 'j': 10,\n",
       " 'k': 11,\n",
       " 'l': 12,\n",
       " 'm': 13,\n",
       " 'n': 14,\n",
       " 'o': 15,\n",
       " 'p': 16,\n",
       " 'q': 17,\n",
       " 'r': 18,\n",
       " 's': 19,\n",
       " 't': 20,\n",
       " 'u': 21,\n",
       " 'v': 22,\n",
       " 'w': 23,\n",
       " 'x': 24,\n",
       " 'y': 25,\n",
       " 'z': 26,\n",
       " '{': 68,\n",
       " '|': 48,\n",
       " '}': 69,\n",
       " '~': 57}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphabet=\"abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\\\"/\\\\|_@#$%^&*~`+-=<>()[]{}\"\n",
    "char_dict = {}\n",
    "for i, char in enumerate(alphabet):\n",
    "    char_dict[char] = i + 1\n",
    "char_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69\n"
     ]
    }
   ],
   "source": [
    "# max index value\n",
    "print(max(char_dict.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use char_dict to replace the tk.word_index\n",
    "tk.word_index = char_dict \n",
    "# Add 'UNK' to the vocabulary \n",
    "tk.word_index[tk.oov_token] = max(char_dict.values()) + 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'!': 41,\n",
       " '\"': 45,\n",
       " '#': 51,\n",
       " '$': 52,\n",
       " '%': 53,\n",
       " '&': 55,\n",
       " \"'\": 44,\n",
       " '(': 64,\n",
       " ')': 65,\n",
       " '*': 56,\n",
       " '+': 59,\n",
       " ',': 38,\n",
       " '-': 60,\n",
       " '.': 40,\n",
       " '/': 46,\n",
       " '0': 27,\n",
       " '1': 28,\n",
       " '2': 29,\n",
       " '3': 30,\n",
       " '4': 31,\n",
       " '5': 32,\n",
       " '6': 33,\n",
       " '7': 34,\n",
       " '8': 35,\n",
       " '9': 36,\n",
       " ':': 43,\n",
       " ';': 39,\n",
       " '<': 62,\n",
       " '=': 61,\n",
       " '>': 63,\n",
       " '?': 42,\n",
       " '@': 50,\n",
       " 'UNK': 70,\n",
       " '[': 66,\n",
       " '\\\\': 47,\n",
       " ']': 67,\n",
       " '^': 54,\n",
       " '_': 49,\n",
       " '`': 58,\n",
       " 'a': 1,\n",
       " 'b': 2,\n",
       " 'c': 3,\n",
       " 'd': 4,\n",
       " 'e': 5,\n",
       " 'f': 6,\n",
       " 'g': 7,\n",
       " 'h': 8,\n",
       " 'i': 9,\n",
       " 'j': 10,\n",
       " 'k': 11,\n",
       " 'l': 12,\n",
       " 'm': 13,\n",
       " 'n': 14,\n",
       " 'o': 15,\n",
       " 'p': 16,\n",
       " 'q': 17,\n",
       " 'r': 18,\n",
       " 's': 19,\n",
       " 't': 20,\n",
       " 'u': 21,\n",
       " 'v': 22,\n",
       " 'w': 23,\n",
       " 'x': 24,\n",
       " 'y': 25,\n",
       " 'z': 26,\n",
       " '{': 68,\n",
       " '|': 48,\n",
       " '}': 69,\n",
       " '~': 57}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tk.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After get the right vocabulary, we can convert all texts to index representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wall st. bears claw back into the black (reuters)reuters - short-sellers, wall street's dwindling\\band of ultra-cynics, are seeing green again.\n",
      "[23, 1, 12, 12, 70, 19, 20, 40, 70, 2, 5, 1, 18, 19, 70, 3, 12, 1, 23, 70, 2, 1, 3, 11, 70, 9, 14, 20, 15, 70, 20, 8, 5, 70, 2, 12, 1, 3, 11, 70, 64, 18, 5, 21, 20, 5, 18, 19, 65, 18, 5, 21, 20, 5, 18, 19, 70, 60, 70, 19, 8, 15, 18, 20, 60, 19, 5, 12, 12, 5, 18, 19, 38, 70, 23, 1, 12, 12, 70, 19, 20, 18, 5, 5, 20, 44, 19, 70, 4, 23, 9, 14, 4, 12, 9, 14, 7, 47, 2, 1, 14, 4, 70, 15, 6, 70, 21, 12, 20, 18, 1, 60, 3, 25, 14, 9, 3, 19, 38, 70, 1, 18, 5, 70, 19, 5, 5, 9, 14, 7, 70, 7, 18, 5, 5, 14, 70, 1, 7, 1, 9, 14, 40]\n"
     ]
    }
   ],
   "source": [
    "sequences = tk.texts_to_sequences(texts)\n",
    "print(texts[0])\n",
    "print(sequences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "143\n",
      "265\n",
      "231\n",
      "255\n",
      "233\n",
      "238\n"
     ]
    }
   ],
   "source": [
    "# List top 5 sentence length\n",
    "for i, s in enumerate(sequences):\n",
    "    print(len(s))\n",
    "    if i > 4:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because the sentence have different lengths, we have to make all sentence as a same length, so the CNN could handle the batch data. Here as the paper proposed, we set the sentence length as `1014`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pad_sequences(sequences, maxlen=1014, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23, 1, 12, 12, 70, 19, 20, 40, 70, 2, 5, 1, 18, 19, 70, 3, 12, 1, 23, 70, 2, 1, 3, 11, 70, 9, 14, 20, 15, 70, 20, 8, 5, 70, 2, 12, 1, 3, 11, 70, 64, 18, 5, 21, 20, 5, 18, 19, 65, 18, 5, 21, 20, 5, 18, 19, 70, 60, 70, 19, 8, 15, 18, 20, 60, 19, 5, 12, 12, 5, 18, 19, 38, 70, 23, 1, 12, 12, 70, 19, 20, 18, 5, 5, 20, 44, 19, 70, 4, 23, 9, 14, 4, 12, 9, 14, 7, 47, 2, 1, 14, 4, 70, 15, 6, 70, 21, 12, 20, 18, 1, 60, 3, 25, 14, 9, 3, 19, 38, 70, 1, 18, 5, 70, 19, 5, 5, 9, 14, 7, 70, 7, 18, 5, 5, 14, 70, 1, 7, 1, 9, 14, 40]\n",
      "\n",
      "[23  1 12 12 70 19 20 40 70  2  5  1 18 19 70  3 12  1 23 70  2  1  3 11\n",
      " 70  9 14 20 15 70 20  8  5 70  2 12  1  3 11 70 64 18  5 21 20  5 18 19\n",
      " 65 18  5 21 20  5 18 19 70 60 70 19  8 15 18 20 60 19  5 12 12  5 18 19\n",
      " 38 70 23  1 12 12 70 19 20 18  5  5 20 44 19 70  4 23  9 14  4 12  9 14\n",
      "  7 47  2  1 14  4 70 15  6 70 21 12 20 18  1 60  3 25 14  9  3 19 38 70\n",
      "  1 18  5 70 19  5  5  9 14  7 70  7 18  5  5 14 70  1  7  1  9 14 40  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0]\n"
     ]
    }
   ],
   "source": [
    "print(sequences[0][:160])\n",
    "print()\n",
    "print(data[0][:160])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(120000, 1014)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.array(data)\n",
    "data.shape # All sentence length are 1014"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get classes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 4 2 1]\n",
      "120000\n",
      "[3 3 3 3 3 3 3 3 3 3]\n",
      "{0, 1, 2, 3}\n",
      "[2, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n"
     ]
    }
   ],
   "source": [
    "print(train_df[0].unique()) # how many unique classes\n",
    "class_list = train_df[0].values\n",
    "print(len(class_list)) # how many length\n",
    "print(class_list[:10]) # print top 10 samples classes\n",
    "# get class\n",
    "class_list = [x-1 for x in class_list] # change index from [1, 2, 3, 4] to [0, 1, 2, 3] in order to use to_categorical()\n",
    "print(set(class_list)) \n",
    "print(class_list[:10]) # print top 10 samples classes to comfirm the changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 0. 1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import to_categorical\n",
    "classes = to_categorical(class_list)\n",
    "print(classes[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mclasses.txt\u001b[m\u001b[m* \u001b[31mreadme.txt\u001b[m\u001b[m*  \u001b[31mtest.csv\u001b[m\u001b[m*    \u001b[31mtrain.csv\u001b[m\u001b[m*\r\n"
     ]
    }
   ],
   "source": [
    "ls ../data/ag_news_csv/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write all code in one cell \n",
    "\n",
    "#========================Load data=========================\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "train_data_source = '../data/ag_news_csv/train.csv'\n",
    "test_data_source = '../data/ag_news_csv/test.csv'\n",
    "\n",
    "train_df = pd.read_csv(train_data_source, header=None)\n",
    "test_df = pd.read_csv(test_data_source, header=None)\n",
    "\n",
    "# concatenate column 1 and column 2 as one text\n",
    "for df in [train_df, test_df]:\n",
    "    df[1] = df[1] + df[2]\n",
    "    df = df.drop([2], axis=1)\n",
    "    \n",
    "# convert string to lower case \n",
    "train_texts = train_df[1].values \n",
    "train_texts = [s.lower() for s in train_texts] \n",
    "\n",
    "test_texts = test_df[1].values \n",
    "test_texts = [s.lower() for s in test_texts] \n",
    "\n",
    "#=======================Convert string to index================\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Tokenizer\n",
    "tk = Tokenizer(num_words=None, char_level=True, oov_token='UNK')\n",
    "tk.fit_on_texts(train_texts)\n",
    "# If we already have a character list, then replace the tk.word_index\n",
    "# If not, just skip below part\n",
    "\n",
    "#-----------------------Skip part start--------------------------\n",
    "# construct a new vocabulary \n",
    "alphabet=\"abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\\\"/\\\\|_@#$%^&*~`+-=<>()[]{}\"\n",
    "char_dict = {}\n",
    "for i, char in enumerate(alphabet):\n",
    "    char_dict[char] = i + 1\n",
    "    \n",
    "# Use char_dict to replace the tk.word_index\n",
    "tk.word_index = char_dict \n",
    "# Add 'UNK' to the vocabulary \n",
    "tk.word_index[tk.oov_token] = max(char_dict.values()) + 1\n",
    "#-----------------------Skip part end----------------------------\n",
    "\n",
    "# Convert string to index \n",
    "train_sequences = tk.texts_to_sequences(train_texts)\n",
    "test_texts = tk.texts_to_sequences(test_texts)\n",
    "\n",
    "# Padding\n",
    "train_data = pad_sequences(train_sequences, maxlen=1014, padding='post')\n",
    "test_data = pad_sequences(test_texts, maxlen=1014, padding='post')\n",
    "\n",
    "# Convert to numpy array\n",
    "train_data = np.array(train_data)\n",
    "test_data = np.array(test_data)\n",
    "\n",
    "#=======================Get classes================\n",
    "train_classes = train_df[0].values\n",
    "train_class_list = [x-1 for x in train_classes]\n",
    "\n",
    "test_classes = test_df[0].values\n",
    "test_class_list = [x-1 for x in test_classes]\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "train_classes = to_categorical(train_class_list)\n",
    "test_classes = to_categorical(test_class_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py36]",
   "language": "python",
   "name": "conda-env-py36-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}