Multinomial Naive Bayes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "    \n",
    "# class MultinomialNB:\n",
    "    \n",
    "#     def fit(self, X, y):\n",
    "#         self.y_classes, y_counts = np.unique(y, return_counts=True)\n",
    "#         self.x_classes = np.array([np.unique(x) for x in X.T])\n",
    "#         self.phi_y = 1.0 * y_counts/y_counts.sum()\n",
    "#         self.phi_x = self.mean_x(X, y)\n",
    "#         return self\n",
    "    \n",
    "#     def mean_x(self, X, y):\n",
    "#         return [[(X[:,j][y==k].reshape(-1,1) == self.x_classes[j]).mean(axis=0)\n",
    "#                        for j in range(len(self.x_classes))]\n",
    "#                       for k in self.y_classes]\n",
    "    \n",
    "#     def predict(self, X):\n",
    "#         return np.apply_along_axis(lambda x: self.compute_probs(x), 1, X)\n",
    "    \n",
    "#     def compute_probs(self, x):\n",
    "#         probs = np.array([self.compute_prob(x, y) for y in range(len(self.y_classes))])\n",
    "#         return self.y_classes[np.argmax(probs)]\n",
    "    \n",
    "#     def compute_prob(self, x, y):\n",
    "#         Pxy = 1\n",
    "#         for j in range(len(x)):\n",
    "#             i = list(self.x_classes[j]).index(x[j])\n",
    "#             Pxy *= self.phi_x[y][j][i] # p(xj|y)\n",
    "#         return Pxy * self.phi_y[y]\n",
    "    \n",
    "#     def evaluate(self, X, y):\n",
    "#         return (self.predict(X) == y).mean()\n",
    "\n",
    "class MultinomialNB:\n",
    "    \n",
    "    def fit(self, X, y, ls=0.01):\n",
    "        self.ls = ls\n",
    "        self.y_classes, y_counts = np.unique(y, return_counts=True)\n",
    "        self.x_classes = [np.unique(x) for x in X.T]\n",
    "        self.phi_y = 1.0 * y_counts/y_counts.sum()\n",
    "        self.phi_x = self.mean_X(X, y)\n",
    "        self.c_x = self.count_x(X, y)\n",
    "        return self\n",
    "    \n",
    "    def mean_X(self, X, y):\n",
    "        return [[self.ls_mean_x(X, y, k, j) for j in range(len(self.x_classes))] for k in self.y_classes]\n",
    "    \n",
    "    def ls_mean_x(self, X, y, k, j):\n",
    "        x_data = (X[:,j][y==k].reshape(-1,1) == self.x_classes[j])\n",
    "        return (x_data.sum(axis=0) + self.ls ) / (len(x_data) + (len(self.x_classes) * self.ls))\n",
    "    \n",
    "    def get_mean_x(self, y, j):\n",
    "        return 1 + self.ls / (self.c_x[y][j] + (len(self.x_classes) * self.ls))\n",
    "        \n",
    "    def count_x(self, X, y):\n",
    "        return [[len(X[:,j][y==k].reshape(-1,1) == self.x_classes[j])\n",
    "                       for j in range(len(self.x_classes))]\n",
    "                      for k in self.y_classes]\n",
    "\n",
    "    def predict(self, X):\n",
    "        return np.apply_along_axis(lambda x: self.compute_probs(x), 1, X)\n",
    "    \n",
    "    def compute_probs(self, x):\n",
    "        probs = np.array([self.compute_prob(x, y) for y in range(len(self.y_classes))])\n",
    "        return self.y_classes[np.argmax(probs)]\n",
    "    \n",
    "    def compute_prob(self, x, y):\n",
    "        Pxy = 1\n",
    "        for j in range(len(x)):\n",
    "            x_clas = self.x_classes[j]\n",
    "            if x[j] in x_clas:\n",
    "                i = list(x_clas).index(x[j])\n",
    "                p_x_j_y = self.phi_x[y][j][i] # p(xj|y)\n",
    "                Pxy *= p_x_j_y\n",
    "            else:\n",
    "                Pxy *= get_mean_x(y, j)\n",
    "        return Pxy * self.phi_y[y]\n",
    "    \n",
    "    def evaluate(self, X, y):\n",
    "        return (self.predict(X) == y).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9666666666666667"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "from utils import accuracy_score\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data  \n",
    "y = iris.target\n",
    "nb = MultinomialNB().fit(X, y)\n",
    "nb.evaluate(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.33333333, 0.33333333, 0.33333333])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb.phi_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[array([0.02018385, 0.06015188, 0.02018385, 0.08013589, 0.04016787,\n",
       "         0.1001199 , 0.08013589, 0.16007194, 0.16007194, 0.06015188,\n",
       "         0.02018385, 0.1001199 , 0.04016787, 0.00019984, 0.04016787,\n",
       "         0.02018385, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984]),\n",
       "  array([0.00019984, 0.00019984, 0.02018385, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.02018385, 0.12010392,\n",
       "         0.08013589, 0.1001199 , 0.04016787, 0.18005596, 0.12010392,\n",
       "         0.06015188, 0.06015188, 0.08013589, 0.04016787, 0.02018385,\n",
       "         0.02018385, 0.02018385, 0.02018385]),\n",
       "  array([2.01838529e-02, 2.01838529e-02, 4.01678657e-02, 1.40087930e-01,\n",
       "         2.59992006e-01, 2.59992006e-01, 1.40087930e-01, 8.01358913e-02,\n",
       "         4.01678657e-02, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04]),\n",
       "  array([1.00119904e-01, 5.79736211e-01, 1.40087930e-01, 1.40087930e-01,\n",
       "         2.01838529e-02, 2.01838529e-02, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04])],\n",
       " [array([0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.02018385, 0.04016787, 0.02018385, 0.02018385,\n",
       "         0.00019984, 0.02018385, 0.1001199 , 0.1001199 , 0.1001199 ,\n",
       "         0.06015188, 0.04016787, 0.08013589, 0.08013589, 0.04016787,\n",
       "         0.06015188, 0.04016787, 0.02018385, 0.04016787, 0.06015188,\n",
       "         0.02018385, 0.02018385, 0.02018385, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984]),\n",
       "  array([0.02018385, 0.04016787, 0.06015188, 0.06015188, 0.08013589,\n",
       "         0.06015188, 0.1001199 , 0.12010392, 0.14008793, 0.16007194,\n",
       "         0.06015188, 0.06015188, 0.02018385, 0.02018385, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984]),\n",
       "  array([0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.02018385,\n",
       "         0.04016787, 0.04016787, 0.02018385, 0.02018385, 0.02018385,\n",
       "         0.06015188, 0.1001199 , 0.06015188, 0.08013589, 0.04016787,\n",
       "         0.08013589, 0.14008793, 0.06015188, 0.1001199 , 0.04016787,\n",
       "         0.04016787, 0.02018385, 0.02018385, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984]),\n",
       "  array([1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.40087930e-01, 6.01518785e-02,\n",
       "         1.00119904e-01, 2.59992006e-01, 1.40087930e-01, 2.00039968e-01,\n",
       "         6.01518785e-02, 2.01838529e-02, 2.01838529e-02, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04])],\n",
       " [array([0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.02018385, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.02018385, 0.02018385,\n",
       "         0.06015188, 0.02018385, 0.04016787, 0.04016787, 0.04016787,\n",
       "         0.12010392, 0.1001199 , 0.08013589, 0.00019984, 0.1001199 ,\n",
       "         0.04016787, 0.06015188, 0.00019984, 0.02018385, 0.06015188,\n",
       "         0.02018385, 0.02018385, 0.02018385, 0.08013589, 0.02018385]),\n",
       "  array([1.99840128e-04, 2.01838529e-02, 1.99840128e-04, 1.99840128e-04,\n",
       "         8.01358913e-02, 4.01678657e-02, 8.01358913e-02, 1.60071942e-01,\n",
       "         4.01678657e-02, 2.40007994e-01, 8.01358913e-02, 1.00119904e-01,\n",
       "         6.01518785e-02, 4.01678657e-02, 1.99840128e-04, 2.01838529e-02,\n",
       "         1.99840128e-04, 4.01678657e-02, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04]),\n",
       "  array([0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.00019984, 0.00019984, 0.00019984, 0.00019984,\n",
       "         0.00019984, 0.02018385, 0.00019984, 0.00019984, 0.04016787,\n",
       "         0.06015188, 0.06015188, 0.14008793, 0.04016787, 0.04016787,\n",
       "         0.04016787, 0.06015188, 0.12010392, 0.06015188, 0.06015188,\n",
       "         0.04016787, 0.04016787, 0.06015188, 0.02018385, 0.02018385,\n",
       "         0.02018385, 0.04016787, 0.02018385]),\n",
       "  array([1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 1.99840128e-04, 1.99840128e-04,\n",
       "         1.99840128e-04, 1.99840128e-04, 2.01838529e-02, 4.01678657e-02,\n",
       "         2.01838529e-02, 2.01838529e-02, 2.20023981e-01, 1.00119904e-01,\n",
       "         1.20103917e-01, 1.20103917e-01, 6.01518785e-02, 1.60071942e-01,\n",
       "         6.01518785e-02, 6.01518785e-02])]]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb.phi_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 2 2 2]\n"
     ]
    }
   ],
   "source": [
    "X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])\n",
    "Y = np.array([1, 1, 1, 2, 2, 2])\n",
    "clf = MultinomialNB().fit(X, Y)\n",
    "print(clf.predict(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9755147468002225"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target\n",
    "MultinomialNB().fit(X, y).evaluate(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}