08_multinomial_logistic_regression__wine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sklearn.metrics\n",
    "import sklearn.datasets\n",
    "import tensorflow as tf\n",
    "\n",
    "# Prints numpy arrays nicer\n",
    "np.set_printoptions(precision=2, suppress=True, linewidth=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wine = sklearn.datasets.load_wine()\n",
    "# Uncomment the following line for a description of the dataset.\n",
    "# print(wine['DESCR'])\n",
    "xs = wine.data\n",
    "ys = wine.target\n",
    "\n",
    "standardize_xs = True\n",
    "if standardize_xs:\n",
    "    xs = (xs - np.mean(xs, axis=0)) / np.std(xs, axis=0)\n",
    "\n",
    "data = list(zip(xs, ys))\n",
    "\n",
    "# Need to shuffle data before split, because it's ordered after ys by default.\n",
    "np.random.shuffle(data)\n",
    "\n",
    "# Perform 60% / 40% training/test split\n",
    "split_index = int(len(data) * 0.6)\n",
    "train_data = data[:split_index]\n",
    "test_data = data[split_index:]\n",
    "print('Num training examples:', len(train_data))\n",
    "print('Num testing examples:', len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "learning_rate = 0.00005 if not standardize_xs else 50\n",
    "num_epochs = 10000 if not standardize_xs else 100\n",
    "num_features = len(train_data[0][0])\n",
    "num_classes = 3\n",
    "batch_size = 10\n",
    "\n",
    "# Model Definition\n",
    "batch_x = tf.placeholder(tf.float32, shape=[None, num_features])\n",
    "batch_y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "W = tf.Variable(tf.random_normal(shape=[num_features, num_classes],\n",
    "                                 mean=0, stddev=1))\n",
    "b = tf.Variable(tf.zeros(shape=[num_classes]))\n",
    "\n",
    "logits = tf.matmul(batch_x, W) + b\n",
    "y_prediction = tf.argmax(logits, axis=-1, output_type=tf.int32)\n",
    "\n",
    "loss = tf.losses.sparse_softmax_cross_entropy(batch_y, logits)\n",
    "train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # Training\n",
    "    time_before = time.time()\n",
    "    losses = []  # Storing losses so we can plot them later\n",
    "    for epoch in range(num_epochs):\n",
    "        np.random.shuffle(train_data)\n",
    "        cumulative_loss = 0\n",
    "        for i in range(0, len(train_data), batch_size):\n",
    "            _batch_x, _batch_y = zip(*train_data[i:i + batch_size])\n",
    "            _loss, _train_op = sess.run(\n",
    "                (loss, train_op),\n",
    "                feed_dict={batch_x: _batch_x, batch_y: _batch_y})\n",
    "            cumulative_loss += _loss * len(_batch_x)\n",
    "        average_loss = cumulative_loss / len(train_data)\n",
    "        if epoch % 20 == 19:\n",
    "            print('Epoch: {}, Loss: {}'.format(epoch + 1, average_loss))\n",
    "        losses.append(average_loss)\n",
    "    time_after = time.time()\n",
    "    print('Training took {:.2f}s.'.format(time_after - time_before))\n",
    "\n",
    "    # Prediction\n",
    "    train_xs, train_ys = zip(*train_data)\n",
    "    train_ys_prediction = sess.run(y_prediction, feed_dict={batch_x: train_xs})\n",
    "\n",
    "    test_xs, test_ys = zip(*test_data)\n",
    "    test_ys_prediction = sess.run(y_prediction, feed_dict={batch_x: test_xs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=150)\n",
    "plt.title('Loss over Time')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.plot(range(len(losses)), losses, color='#458588')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Precision on Training data:',\n",
    "      sklearn.metrics.precision_score(train_ys, train_ys_prediction,\n",
    "                                      average='macro'))\n",
    "print('Recall on Training data:',\n",
    "      sklearn.metrics.recall_score(train_ys, train_ys_prediction,\n",
    "                                   average='macro'))\n",
    "print('F1-Score on Training data:',\n",
    "      sklearn.metrics.f1_score(train_ys, train_ys_prediction, average='macro'))\n",
    "print('Accuracy on Training data:',\n",
    "      sklearn.metrics.accuracy_score(train_ys, train_ys_prediction))\n",
    "print()\n",
    "print('Precision on Testing data:',\n",
    "      sklearn.metrics.precision_score(test_ys, test_ys_prediction,\n",
    "                                      average='macro'))\n",
    "print('Recall on Testing data:',\n",
    "      sklearn.metrics.recall_score(test_ys, test_ys_prediction,\n",
    "                                   average='macro'))\n",
    "print('F1-Score on Testing data:',\n",
    "      sklearn.metrics.f1_score(test_ys, test_ys_prediction, average='macro'))\n",
    "print('Accuracy on Testing data:',\n",
    "      sklearn.metrics.accuracy_score(test_ys, test_ys_prediction))\n",
    "\n",
    "train_num_labels = []\n",
    "train_num_labels_prediction = []\n",
    "test_num_labels = []\n",
    "test_num_labels_prediction = []\n",
    "for i in range(3):\n",
    "    train_num_labels.append(np.sum(np.equal(train_ys, i)))\n",
    "    train_num_labels_prediction.append(np.sum(np.equal(train_ys_prediction, i)))\n",
    "    test_num_labels.append(np.sum(np.equal(test_ys, i)))\n",
    "    test_num_labels_prediction.append(np.sum(np.equal(test_ys_prediction, i)))\n",
    "\n",
    "plt.figure(dpi=150)\n",
    "plt.title('Class Distribution Actual vs Predicted: Training Data')\n",
    "plt.ylabel('Frequency')\n",
    "plt.bar([1, 2, 3.5, 4.5, 6, 7],\n",
    "        [train_num_labels[0], train_num_labels_prediction[0],\n",
    "         train_num_labels[1], train_num_labels_prediction[1],\n",
    "         train_num_labels[2], train_num_labels_prediction[2]],\n",
    "        tick_label=['Actual A', 'Predicted A',\n",
    "                    'Actual B', 'Predicted B',\n",
    "                    'Actual C', 'Predicted C'],\n",
    "        color=['#458588', '#CC241D'])\n",
    "plt.show()\n",
    "\n",
    "plt.figure(dpi=150)\n",
    "plt.title('Class Distribution Actual vs Predicted: Testing Data')\n",
    "plt.ylabel('Frequency')\n",
    "plt.bar([1, 2, 3.5, 4.5, 6, 7],\n",
    "        [test_num_labels[0], test_num_labels_prediction[0],\n",
    "         test_num_labels[1], test_num_labels_prediction[1],\n",
    "         test_num_labels[2], test_num_labels_prediction[2]],\n",
    "        tick_label=['Actual A', 'Predicted A',\n",
    "                    'Actual B', 'Predicted B',\n",
    "                    'Actual C', 'Predicted C'],\n",
    "        color=['#458588', '#CC241D'])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}