DL/Softmax.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Softmax from scratch.\n",
    "\n",
    "- stochastic gradient descent\n",
    "- epochs, mini-batches.\n",
    "- pre-process the features.\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "class Softmax:\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "        \n",
    "    def forward(self, X):\n",
    "        pred = np.dot(X, self.W) + self.b # N * label_dim.\n",
    "        z = pred - pred.max(axis=1).reshape([-1, 1]) # stable softmax, avoid overflow. \n",
    "        prob = np.exp(z) / np.sum(np.exp(z), axis=1, keepdims=True)\n",
    "        return prob\n",
    "        \n",
    "    def compute_loss(self, y, y_hat):\n",
    "        \"\"\"\n",
    "        y: ground truth labels. N * 1\n",
    "        y_hat: pred output. N * k\n",
    "        \"\"\"\n",
    "        N = y.shape[0]\n",
    "        log_likelihood = -np.log(y_hat[range(N), y])\n",
    "        loss = np.sum(log_likelihood) / N\n",
    "        return loss\n",
    "    \n",
    "    def compute_grad(self, y, y_hat, x, reg_penalty):\n",
    "        N = y.shape[0]\n",
    "        y_hat[range(N), y] -= 1 \n",
    "        dW = (x.T.dot(y_hat) / N) + (reg_penalty * self.W)\n",
    "        return dW\n",
    "        \n",
    "    \n",
    "    def train(self, X, Y, X_test, Y_test, epochs = 1000, batch_size = 32, reg_penalty = 1e-4, momentum_rate = 0.6, learning_rate = 2e-1, plot = False):\n",
    "        \n",
    "        self.num_data = X.shape[0]\n",
    "        label_dim = np.max(Y) + 1 # label dimension. NOTE: labels must be zero-aligned!\n",
    "        \n",
    "        input_dim = X.shape[1] # input feature dimension.\n",
    "        self.W = np.random.randn(input_dim, label_dim) / np.sqrt(input_dim/2)\n",
    "        self.b = np.random.randn(1, label_dim) / np.sqrt(input_dim/2)\n",
    "        self.epochs = epochs\n",
    "        self.batch_size = batch_size\n",
    "        self.reg_penalty = reg_penalty\n",
    "        \n",
    "        \n",
    "        # history.\n",
    "        train_loss_lst = []\n",
    "        train_accuracy_lst = []\n",
    "        test_loss_lst = []\n",
    "        test_accuracy_lst = []\n",
    "        \n",
    "        # epoch_step\n",
    "        epoch_step = 50\n",
    "        \n",
    "        for epoch in range(self.epochs):\n",
    "            \n",
    "            # record last time momentum vector.\n",
    "            momentum = 0 \n",
    "            loss_mean = []\n",
    "            \n",
    "            # step through training sample by batch_size.\n",
    "            for i in range(0, X.shape[0], self.batch_size):\n",
    "                X_mini = X[i: i + batch_size]\n",
    "                Y_mini = Y[i: i + batch_size]\n",
    "                \n",
    "                # computer estimate for batch.\n",
    "                y_hat = self.forward(X_mini)\n",
    "                \n",
    "                # computer loss for batch.\n",
    "                loss = self.compute_loss(Y_mini, y_hat)\n",
    "                loss_mean.append(loss)\n",
    "                \n",
    "                # compute gradient for dw.\n",
    "                dW = self.compute_grad(Y_mini, y_hat, X_mini, reg_penalty)\n",
    "                \n",
    "                # update dw.\n",
    "                momentum = (momentum_rate * momentum - learning_rate * dW) \n",
    "                self.W += momentum \n",
    "             \n",
    "            if epoch % epoch_step == 0:\n",
    "                \n",
    "                # calculate train loss.\n",
    "                train_loss = np.mean(np.array(loss_mean))\n",
    "\n",
    "                # calculate train accuracy.\n",
    "                train_pred = self.predict(X)\n",
    "                train_accuracy = np.mean(np.equal(Y, train_pred))\n",
    "\n",
    "                # calculate test loss.\n",
    "                test_loss = self.compute_loss(Y_test, self.forward(X_test))\n",
    "                \n",
    "                # calculate test accuracy.\n",
    "                test_pred = self.predict(X_test)\n",
    "                test_accuracy = np.mean(np.equal(Y_test, test_pred))\n",
    "\n",
    "                if plot:\n",
    "                    train_loss_lst.append(train_loss)\n",
    "                    train_accuracy_lst.append(train_accuracy)\n",
    "                    test_loss_lst.append(test_loss)\n",
    "                    test_accuracy_lst.append(test_accuracy)\n",
    "                \n",
    "                print('Epoch: %s \\n'\\\n",
    "                      'Train Loss: %s, Train Accuracy: %s \\n'\\\n",
    "                      'Test Loss: %s, Test Accuracy: %s.' % (epoch, train_loss, train_accuracy, test_loss, test_accuracy))\n",
    "                print()\n",
    "        \n",
    "        \n",
    "        # plot loss graphs.\n",
    "        if plot:\n",
    "            epoch_count = range(0, epochs, epoch_step)\n",
    "            plt.plot(epoch_count, train_loss_lst, 'r--')\n",
    "            plt.plot(epoch_count, test_loss_lst, 'b-')\n",
    "            plt.legend(['Training Loss', 'Test Loss'])\n",
    "            plt.xlabel('Epoch')\n",
    "            plt.ylabel('Loss')\n",
    "            plt.show()\n",
    "\n",
    "            # plot accuracy graph.\n",
    "            plt.plot(epoch_count, train_accuracy_lst, 'r--')\n",
    "            plt.plot(epoch_count, test_accuracy_lst, 'b-')\n",
    "            plt.legend(['Training Accuracy', 'Test Accuracy'])\n",
    "            plt.xlabel('Epoch')\n",
    "            plt.ylabel('Accuracy')\n",
    "            plt.show()\n",
    "\n",
    "        \n",
    "    def predict(self, X):\n",
    "        # return predicted class for test set. argmax() return index.\n",
    "        return np.argmax(X.dot(self.W), 1)\n",
    "        \n",
    "\"\"\"\n",
    "train_data: N * D,\n",
    "train_labels: N * 1: class label, from 1 to 3. \n",
    "\"\"\"\n",
    "def train_iris():\n",
    "    sm = Softmax()\n",
    "    \n",
    "    # re-align labels to be zero-aligned.\n",
    "    Y = train_labels - np.min(train_labels) \n",
    "    Y_test = test_labels - np.min(test_labels) \n",
    "    \n",
    "    sm.train(train_data, Y, test_data, Y_test, plot = True)\n",
    "\n",
    "# train_iris()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}