Language identification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Project: language identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas.plotting import scatter_matrix\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import string\n",
    "import collections\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing stresses\n",
    "In order to work with only ASCII characters for the chinese language _*pinyin*_ is used instead of _*hanzi*_, and every in every word stresses are removed.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_stress(word):\n",
    "    table = collections.defaultdict(lambda: None)\n",
    "    table.update({\n",
    "        ord('é'): 'e',\n",
    "        ord('ô'): 'o',\n",
    "        ord('è'): 'e',\n",
    "        ord('à'): 'a',\n",
    "        ord('ì'): 'i',\n",
    "        ord('ù'): 'u',\n",
    "        ord('\\n'): '',\n",
    "    })\n",
    "    table.update(dict(zip(map(ord,string.ascii_uppercase), string.ascii_lowercase)))\n",
    "    table.update(dict(zip(map(ord,string.ascii_lowercase), string.ascii_lowercase)))\n",
    "    table.update(dict(zip(map(ord,string.digits), string.digits)))\n",
    "    return word.translate(table,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text to feature vector\n",
    "This function converts every word into a feature vector where each feature is the amout of times a certain letter appears, if `scale` is set to `True` then the features will be scaled by dividing them by the total length of the word. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_string(word, lang = None, scale = False):\n",
    "   \n",
    "    str(word)\n",
    "    word = strip_stress(word.lower())\n",
    "    length = len(word)\n",
    "    LetterFreq={}\n",
    "    for letter in string.ascii_lowercase:\n",
    "        LetterFreq[letter] = 0\n",
    "    for letter in word.lower():\n",
    "        LetterFreq[letter] += 1\n",
    "    features = list(LetterFreq.values())\n",
    "    \n",
    "    if(length < 1 or scale == False):\n",
    "        features = [float(x) for x in features]\n",
    "    else:\n",
    "        features = [float(x)/length for x in features]\n",
    "    \n",
    "    if(lang != None):\n",
    "        features.append(lang)\n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset\n",
    "We generate three separate DataFrames due to memory constraints, else we incurr into a `MemoryError`, the starting data is in the format:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> ./Data/chinese.txt <==\n",
      "Longhai\n",
      "LongWang\n",
      "Longxin\n",
      "Longyouxian\n",
      "longtou\n",
      "Longfengqu\n",
      "Pangshipianju\n",
      "guierzi\n",
      "guibeizhu\n",
      "hei\n",
      "\n",
      "==> ./Data/english.txt <==\n",
      "zealand\n",
      "zeds\n",
      "zero\n",
      "ziggurats\n",
      "zinc\n",
      "zipped\n",
      "zloty\n",
      "zoologists\n",
      "zounds\n",
      "�lan\n",
      "\n",
      "==> ./Data/italian.txt <==\n",
      "zoofili\n",
      "zoppa\n",
      "zoppicate\n",
      "zotici\n",
      "zuccherato\n",
      "zucchini\n",
      "zuffolassi\n",
      "zufolare\n",
      "�rdono\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!tail ./Data/*.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',\n",
    "         'U', 'V', 'W','X', 'Y', 'Z', 'class']\n",
    "\n",
    "dataset_it = pd.DataFrame((parse_string(word,'Italian') for word in open('./Data/italian.txt', encoding = \"ISO-8859-1\")), columns = names)\n",
    "dataset_en = pd.DataFrame(([parse_string(word,'English') for word in open('./Data/english.txt', encoding = \"ISO-8859-1\")]), columns=names)\n",
    "dataset_zh = pd.DataFrame(([parse_string(word,'Chinese') for word in open('./Data/chinese.txt', encoding = \"ISO-8859-1\")]) , columns = names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we join the data from the three datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.DataFrame()\n",
    "dataset = dataset.append(dataset_it)\n",
    "dataset = dataset.append(dataset_en)\n",
    "dataset = dataset.append(dataset_zh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Visualization on the different letter frequency in the analyzed languages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "# Code used to generate the histograms\n",
    "dataset_it.hist()\n",
    "dataset_en.hist()\n",
    "dataset_zh.hist()\n",
    "\n",
    "pyplot.show()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Italian | English | Chinese\n",
    " - | - | -  \n",
    " ![IT](./media/hist_it.png) |![EN](./media/hist_en.png) |![ZH](./media/hist_zh.png) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zoomed version:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "  Italian:\n",
    "  ![IT](./media/hist_it.png)\n",
    "  \n",
    "  English:\n",
    "  ![EN](./media/hist_en.png)\n",
    " \n",
    "  Chinese:\n",
    " ![ZH](./media/hist_zh.png) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split-out validation dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = dataset.values\n",
    "X = array[:,0:26]\n",
    "Y = array[:,26]\n",
    "validation_size = 0.20\n",
    "seed = 7\n",
    "X_train, X_validation, Y_train, Y_validation = train_test_split(X,Y,test_size=validation_size, random_state=seed, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spot-Check Algorithms\n",
    "Here we train many different models with the training set as to compare their performances and pick the most promising one to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "models=[]\n",
    "models.append(('LR', LogisticRegression(multi_class='auto', solver='liblinear')))\n",
    "models.append(('LDA',LinearDiscriminantAnalysis()))\n",
    "models.append(('KNN', KNeighborsClassifier()))\n",
    "models.append(('CART', DecisionTreeClassifier()))\n",
    "models.append(('NB', GaussianNB()))\n",
    "models.append(('SVM', SVC(gamma='scale')))\n",
    "models.append(('MLP', MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(30,10), random_state=3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evalutate each model in turn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "names = []\n",
    "for name, model in models:\n",
    "    kfold = KFold(n_splits=10, random_state=seed) \n",
    "    cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring='accuracy')\n",
    "    results.append(cv_results)\n",
    "    names.append(name)\n",
    "    msg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n",
    "    print(msg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "# Code used to generate the boxplots \n",
    "fig = pyplot.figure()\n",
    "fig.suptitle('Algorithm comparison')\n",
    "ax = fig.add_subplot(111)\n",
    "ax.boxplot(results)\n",
    "ax.set_xticklabels(names)\n",
    "pyplot.show()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model |With unscaled features: | With scaled features:\n",
    "- | - | - \n",
    " | ![unscaled features](./media/algorithm_comparison_unscaled_features.png) | ![scaled features](./media/algorithm_comparison_scaled_features.png)\n",
    "LR | 0.786885 (0.009271)| 0.767812 (0.006941)\n",
    " LDA | 0.774308 (0.005538) | 0.763458 (0.005461)\n",
    "KNN | 0.740793 (0.008171) | 0.760417 (0.012712)\n",
    "CART | 0.705410 (0.008459)| 0.712391 (0.005213)\n",
    "NB | 0.691453 (0.013798) | 0.684612 (0.013949)\n",
    "SVM | 0.831181 (0.011519) | 0.791100 (0.009735)\n",
    "MLP | 0.819709 (0.011321) | 0.801397 (0.010456)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make predictions with chosen model on validation dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Select a model (Support Vector) and train it with all the training data\n",
    "We then make prediction for the validation set to extimate the final model's accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SVC(gamma='scale', probability=True)\n",
    "model.fit(X_train, Y_train)\n",
    "predictions = model.predict(X_validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final accuracy analysis \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: SVC \n",
      "\n",
      "Accuracy score:\n",
      " 0.8366500829187397\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "\n",
      "Predicted  Chinese  English  Italian\n",
      "Actual                              \n",
      "Chinese       1429       31       31\n",
      "English         84      698      258\n",
      "Italian         51      136      900\n",
      "\n",
      "\n",
      "Classification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "     Chinese       0.91      0.96      0.94      1491\n",
      "     English       0.81      0.67      0.73      1040\n",
      "     Italian       0.76      0.83      0.79      1087\n",
      "\n",
      "   micro avg       0.84      0.84      0.84      3618\n",
      "   macro avg       0.83      0.82      0.82      3618\n",
      "weighted avg       0.84      0.84      0.83      3618\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_name = str(model).split('(')[0]\n",
    "print(\"Model:\",model_name,\"\\n\")\n",
    "print(\"Accuracy score:\\n\",accuracy_score(Y_validation, predictions))\n",
    "print(\"\\n\\nConfusion matrix:\\n\")\n",
    "data = {'y_Predicted': predictions,\n",
    "        'y_Actual':    Y_validation\n",
    "        }\n",
    "\n",
    "df = pd.DataFrame(data, columns=['y_Actual','y_Predicted'])\n",
    "\n",
    "confusion_matrix = pd.crosstab(df['y_Actual'], df['y_Predicted'], rownames=['Actual'], colnames=['Predicted'])\n",
    "print (confusion_matrix)\n",
    "print(\"\\n\\nClassification report:\\n\",classification_report(Y_validation, predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save current model to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='scale', kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False)\n"
     ]
    }
   ],
   "source": [
    "filename = 'finalized_model.sav'\n",
    "pickle.dump(model, open(filename, 'wb'))\n",
    "print(\"Saved\", model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'finalized_model.sav'\n",
    "model = pickle.load(open(filename, 'rb'))\n",
    "print(\"Loaded\", model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Practical demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The word \"\u001b[31mTest\u001b[0m\" was classified as \u001b[31mEnglish\u001b[0m\n",
      "\tChinese: 0.258 %\n",
      "\tEnglish: 79.301 %\n",
      "\tItalian: 20.442 %\n"
     ]
    }
   ],
   "source": [
    "test_string = \"Test\"\n",
    "\n",
    "classes = [\"Chinese\",\"English\", \"Italian\"]\n",
    "\n",
    "predictions = model.predict_proba([parse_string(test_string)]).tolist()[0]\n",
    "best_guess = predictions.index(max(predictions))\n",
    "\n",
    "print(\"The word\", \"\\\"\\x1b[31m\"+test_string+\"\\x1b[0m\\\"\",  \"was classified as\", \"\\x1b[31m\"+ classes[best_guess] +\"\\x1b[0m\")\n",
    "for i in range (0,3):\n",
    "    print(\"\\t\"+classes[i]+\":\",round(predictions[i]*100, 3),\"%\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}