notebooks/Lecture 04 - Part 4 - Artificial Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "443a57e3",
   "metadata": {},
   "source": [
    "# **SIN 393 – Introduction to Computer Vision (2023)**\n",
    "\n",
    "# Lecture 04 - Part 4 - Artificial Neural Networks\n",
    "\n",
    "Prof. João Fernando Mari ([*joaofmari.github.io*](https://joaofmari.github.io/))\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82143f90",
   "metadata": {},
   "source": [
    "## Importing the required libraries\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c3f119c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "\n",
    "import ipywidgets as widgets\n",
    "\n",
    "from sklearn import datasets, metrics, preprocessing, model_selection\n",
    "\n",
    "### %matplotlib widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f2980b90-c765-4564-8698-13e0bc527314",
   "metadata": {},
   "outputs": [],
   "source": [
    "def error_brute_force(X, y, w0=[-1, 1], w1=[-1, 1], b=[-1, 1], step=0.1):\n",
    "    \"\"\"\n",
    "    20 x 20 x 20 = 8,000 parameter combinations\n",
    "    \"\"\"\n",
    "    # Initialize error space\n",
    "    error_space = np.zeros([len(np.arange(w0[0], w0[1], step)),\n",
    "                            len(np.arange(w1[0], w1[1], step)),\n",
    "                            len(np.arange(b[0], b[1], step))])\n",
    "\n",
    "    # Iterate along w0\n",
    "    for i, w0_ in enumerate(np.arange(w0[0], w0[1], step)):\n",
    "        # Iterate along w1\n",
    "        for j, w1_ in enumerate(np.arange(w1[0], w1[1], step)):\n",
    "            # Iterate along b\n",
    "            for k, b_ in enumerate(np.arange(b[0], b[1], step)):\n",
    "                # Initialize epoch error\n",
    "                erro_epoca = 0.\n",
    "                w = [w0_, w1_]\n",
    "                # Iterate along dataset (one epoch)\n",
    "                for x_, y_ in zip(X, y):\n",
    "                    # Innet product\n",
    "                    v = np.dot(x_, w) + b_\n",
    "                    # Activation function\n",
    "                    y_out = np.where(v >= 0., 1, 0)\n",
    "\n",
    "                    # Error\n",
    "                    erro = y_ - y_out\n",
    "                    # Update epoch error\n",
    "                    erro_epoca = erro_epoca + erro**2\n",
    "            \n",
    "                # Erro total da época.\n",
    "                erro_epoca = erro_epoca / 2.\n",
    "                error_space[i, j, k] = erro_epoca      \n",
    "\n",
    "    return error_space "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3234925b",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dc5f177b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "X_bin = np.array([[0, 0],\n",
    "                  [0, 1],\n",
    "                  [1, 0],\n",
    "                  [1, 1]])\n",
    "\n",
    "# Binary functions\n",
    "# ================\n",
    "# AND\n",
    "y_and = np.array([0, 0, 0, 1])\n",
    "\n",
    "# OR\n",
    "y_or = np.array([0, 1, 1, 1])\n",
    "\n",
    "# XOR\n",
    "y_xor = np.array([0, 1, 1, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "351992a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the IRIS dataset\n",
    "# ========================\n",
    "iris = datasets.load_iris()\n",
    "\n",
    "# Selects only the Setosa (0) and Virginica (1) classes.\n",
    "# Select only 2 attributes: sepal length (0) and sepal width (1)\n",
    "X_iris = iris.data[iris.target < 2, :2]\n",
    "y_iris = iris.target[iris.target < 2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91f669a6-2da4-4d68-9fa6-5a66dc125702",
   "metadata": {},
   "source": [
    "## Select the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f4f8f3f3-df3d-4ef1-a219-50db2ff2c7bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X_bin\n",
    "y = y_and\n",
    "str_title = 'AND'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a367c6fc-2e38-468a-b3a5-c3d46264575e",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46165b1d-d7f3-49c1-8621-124c6aee6a16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 20, 20)\n"
     ]
    }
   ],
   "source": [
    "error_space = error_brute_force(X, y)\n",
    "\n",
    "print(error_space.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "71aa58cd-4fc3-4031-946a-07a040925aee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_error_a(error_space, w0_, w1_, b_, w0=[-1, 1], w1=[-1, 1], b=[-1, 1], step=0.1, surf=True):\n",
    "\n",
    "    # Convert index to value.\n",
    "    w0_r = np.arange(w0[0], w0[1], step)\n",
    "    w1_r = np.arange(w1[0], w1[1], step)\n",
    "    b_r = np.arange(b[0], b[1], step)\n",
    "\n",
    "    w0_v = w0_r[w0_]\n",
    "    w1_v = w1_r[w1_]\n",
    "    b_v = b_r[b_]\n",
    "\n",
    "    # Avoid zero division\n",
    "    if w1_v == w0_v or w1_v == b_v:\n",
    "        w1_v = w1_v + 0.01\n",
    "\n",
    "    w = [w0_v, w1_v]\n",
    "\n",
    "    # Define os pontos extremos do segmento da superficie de decisão plotado.\n",
    "    # p0 = (x0_min, f_x0_min); p1 = (x0_max, f_x0_max)\n",
    "    # ----------\n",
    "    x0_min = X.min() - 1.\n",
    "    x0_max = X.max() + 1.\n",
    "    # ----------\n",
    "    f_x0_min = -(w[0] / w[1]) * x0_min - (b_v / w[1])\n",
    "    f_x0_max = -(w[0] / w[1]) * x0_max - (b_v / w[1])\n",
    "\n",
    "    colors = ['r', 'g', 'b', 'y', 'c', 'm']\n",
    "\n",
    "    print(f'w0: {w0_v:.2f} w1: {w1_v:.2f} b: {b_v:.2f} Error: {error_space[w0_,w1_,b_]}')\n",
    "\n",
    "    # Plot\n",
    "    fig = plt.figure(figsize=(12, 3))\n",
    "\n",
    "    # Feature space\n",
    "    ax1 = fig.add_subplot(1, 4, 1)\n",
    "    for y_ in np.unique(y):\n",
    "        ax1.scatter(X[y==y_][:,0], X[y==y_][:,1], color=colors[y_], label=str(y_))\n",
    " \n",
    "    ax1.set_xlabel('$x_0$')\n",
    "    ax1.set_ylabel('$x_1$')\n",
    "    ax1.legend()\n",
    "    ax1.set_title(str_title)\n",
    "\n",
    "    ax1.set_xlim(X.min()-.5, X.max()+.5)\n",
    "    ax1.set_ylim(X.min()-.5, X.max()+.5)\n",
    "\n",
    "    # Plot decision surface\n",
    "    ax1.plot([x0_min, x0_max], [f_x0_min, f_x0_max], color='b')\n",
    "    ax1.set_aspect('equal')\n",
    "\n",
    "    # w0 Vs. w1\n",
    "    if surf:\n",
    "        ax2 = fig.add_subplot(1, 4, 2, projection='3d')\n",
    "        xx, yy = np.mgrid[0:error_space[:,:,b_].shape[0], 0:error_space[:,:,b_].shape[1]]\n",
    "        ax2.plot_surface(xx, yy, error_space[:,:,b_] ,rstride=1, cstride=1, linewidth=0)\n",
    "        ax2.plot([w0_, w0_], [w1_, w1_], [0, np.max(error_space[:,:,b_]) + 1.0], 'r')\n",
    "    else:\n",
    "        ax2 = fig.add_subplot(1, 4, 2)\n",
    "        im2 = ax2.imshow(error_space[:,:,b_], cmap='gray')\n",
    "        ax2.plot(w0_, w1_, marker='o', color='r')\n",
    "        # Colorbar\n",
    "        divider = make_axes_locatable(ax2)\n",
    "        cax = divider.append_axes('right', size='5%', pad=0.05)\n",
    "        fig.colorbar(im2, cax=cax, orientation='vertical')\n",
    "    ax2.set_title('$w_0$ Vs. $w_1$')\n",
    "    ax2.set_xlabel('$w_0$')\n",
    "    ax2.set_ylabel('$w_1$')\n",
    "\n",
    "    # w0 Vs. b\n",
    "    if surf:\n",
    "        ax3 = fig.add_subplot(1, 4, 3, projection='3d')\n",
    "        # create the x and y coordinate arrays (here we just use pixel indices)\n",
    "        xx, yy = np.mgrid[0:error_space[:,w1_,:].shape[0], 0:error_space[:,:,b_].shape[1]]\n",
    "        ax3.plot_surface(xx, yy, error_space[:,w1_,:] ,rstride=1, cstride=1, linewidth=0)\n",
    "        ax3.plot([w0_, w0_], [b_, b_], [0, np.max(error_space[:,w1_,:]) + 1.0], 'r')\n",
    "    else:\n",
    "        ax3 = fig.add_subplot(1, 4, 3)\n",
    "        im3 = ax3.imshow(error_space[:,w1_,:].T, cmap='gray')\n",
    "        ax3.plot(w0_, b_,  marker='o', color='r')\n",
    "        # Colorbar\n",
    "        divider = make_axes_locatable(ax3)\n",
    "        cax = divider.append_axes('right', size='5%', pad=0.05)\n",
    "        fig.colorbar(im3, cax=cax, orientation='vertical')\n",
    "    ax3.set_title('$w_0$ Vs. $b$')\n",
    "    ax3.set_xlabel('$w_0$')\n",
    "    ax3.set_ylabel('$b$')\n",
    "    \n",
    "    # w1 Vs. b\n",
    "    if surf:\n",
    "        ax4 = fig.add_subplot(1, 4, 4, projection='3d')\n",
    "        # create the x and y coordinate arrays (here we just use pixel indices)\n",
    "        xx, yy = np.mgrid[0:error_space[w0_,:,:].shape[0], 0:error_space[:,:,b_].shape[1]]\n",
    "        ax4.plot_surface(xx, yy, error_space[w0_,:,:] ,rstride=1, cstride=1, linewidth=0)\n",
    "        ax4.plot([w1_, w1_], [b_, b_], [0, np.max(error_space[w0_,:,:]) + 1.0], 'r')\n",
    "    else:\n",
    "        ax4 = fig.add_subplot(1, 4, 4)\n",
    "        im4 = ax4.imshow(error_space[w0_,:,:].T, cmap='gray')\n",
    "        ax4.plot(w1_, b_,  marker='o', color='r')\n",
    "        # Colorbar\n",
    "        divider = make_axes_locatable(ax4)\n",
    "        cax = divider.append_axes('right', size='5%', pad=0.05)\n",
    "        fig.colorbar(im4, cax=cax, orientation='vertical')\n",
    "    ax4.set_title('$w_1$ Vs. $b$')\n",
    "    ax4.set_xlabel('$w_1$')\n",
    "    ax4.set_ylabel('$b$')\n",
    "\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fcec5368-0b16-4a27-a9eb-336399b834b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_error_b(error_space, w0_, w1_, b_, w0=[-1, 1], w1=[-1, 1], b=[-1, 1], step=0.1, surf=True):\n",
    "\n",
    "    # Convert index to value.\n",
    "    w0_r = np.arange(w0[0], w0[1], step)\n",
    "    w1_r = np.arange(w1[0], w1[1], step)\n",
    "    b_r = np.arange(b[0], b[1], step)\n",
    "\n",
    "    w0_v = w0_r[w0_]\n",
    "    w1_v = w1_r[w1_]\n",
    "    b_v = b_r[b_]\n",
    "\n",
    "    # Avoid zero division\n",
    "    if w1_v == w0_v or w1_v == b_v:\n",
    "        w1_v = w1_v + 0.01\n",
    "\n",
    "    w = [w0_v, w1_v]\n",
    "\n",
    "    # Define os pontos extremos do segmento da superficie de decisão plotado.\n",
    "    # p0 = (x0_min, f_x0_min); p1 = (x0_max, f_x0_max)\n",
    "    # ----------\n",
    "    x0_min = X.min() - 1.\n",
    "    x0_max = X.max() + 1.\n",
    "    # ----------\n",
    "    f_x0_min = -(w[0] / w[1]) * x0_min - (b_v / w[1])\n",
    "    f_x0_max = -(w[0] / w[1]) * x0_max - (b_v / w[1])\n",
    "\n",
    "    colors = ['r', 'g', 'b', 'y', 'c', 'm']\n",
    "\n",
    "    print(f'w0: {w0_v:.2f} w1: {w1_v:.2f} b: {b_v:.2f} Error: {error_space[w0_,w1_,b_]}')\n",
    "\n",
    "    fig = plt.figure(figsize=(12, 6))\n",
    "\n",
    "    # Feature space\n",
    "    ax1 = fig.add_subplot(2, 3, 1)\n",
    "    for y_ in np.unique(y):\n",
    "        ax1.scatter(X[y==y_][:,0], X[y==y_][:,1], color=colors[y_], label=str(y_))\n",
    " \n",
    "    ax1.set_xlabel('$x_0$')\n",
    "    ax1.set_ylabel('$x_1$')\n",
    "    ax1.legend()\n",
    "    ax1.set_title(str_title)\n",
    "\n",
    "    ax1.set_xlim(X.min()-.5, X.max()+.5)\n",
    "    ax1.set_ylim(X.min()-.5, X.max()+.5)\n",
    "\n",
    "    # Plot decision surface\n",
    "    ax1.plot([x0_min, x0_max], [f_x0_min, f_x0_max], color='b')\n",
    "    ax1.set_aspect('equal')\n",
    "\n",
    "    ax1a = fig.add_subplot(2, 3, 2)\n",
    "    ax1a.axis('off')\n",
    "    ax1b = fig.add_subplot(2, 3, 3)\n",
    "    ax1b.axis('off')\n",
    "\n",
    "    # w0 Vs. w1\n",
    "    if surf:\n",
    "        ax2 = fig.add_subplot(2, 3, 4, projection='3d')\n",
    "        # create the x and y coordinate arrays (here we just use pixel indices)\n",
    "        xx, yy = np.mgrid[0:error_space[:,:,b_].shape[0], 0:error_space[:,:,b_].shape[1]]\n",
    "        ax2.plot_surface(xx, yy, error_space[:,:,b_] ,rstride=1, cstride=1, linewidth=0)\n",
    "        ax2.plot([w0_, w0_], [w1_, w1_], [0, np.max(error_space[:,:,b_]) + 1.0], 'r')\n",
    "    else:\n",
    "        ax2 = fig.add_subplot(2, 3, 4)\n",
    "        im2 = ax2.imshow(error_space[:,:,b_], cmap='gray')\n",
    "        ax2.plot(w0_, w1_,  marker='o', color='r')\n",
    "        # Colorbar\n",
    "        divider = make_axes_locatable(ax2)\n",
    "        cax = divider.append_axes('right', size='5%', pad=0.05)\n",
    "        fig.colorbar(im2, cax=cax, orientation='vertical')\n",
    "    ax2.set_title('$w_0$ Vs. $w_1$')\n",
    "    ax2.set_xlabel('$w_0$')\n",
    "    ax2.set_ylabel('$w_1$')\n",
    "\n",
    "    # w0 Vs. b\n",
    "    if surf:\n",
    "        ax3 = fig.add_subplot(2, 3, 5, projection='3d')\n",
    "        # create the x and y coordinate arrays (here we just use pixel indices)\n",
    "        xx, yy = np.mgrid[0:error_space[:,w1_,:].shape[0], 0:error_space[:,:,b_].shape[1]]\n",
    "        ax3.plot_surface(xx, yy, error_space[:,w1_,:] ,rstride=1, cstride=1, linewidth=0)\n",
    "        ax3.plot([w0_, w0_], [b_, b_], [0, np.max(error_space[:,w1_,:]) + 1.0], 'r')\n",
    "    else:\n",
    "        ax3 = fig.add_subplot(2, 3, 5)\n",
    "        im3 = ax3.imshow(error_space[:,w1_,:].T, cmap='gray')\n",
    "        ax3.plot(w0_, b_,  marker='o', color='r')\n",
    "        # Colorbar\n",
    "        divider = make_axes_locatable(ax3)\n",
    "        cax = divider.append_axes('right', size='5%', pad=0.05)\n",
    "        fig.colorbar(im3, cax=cax, orientation='vertical')\n",
    "    ax3.set_title('$w_0$ Vs. $b$')\n",
    "    ax3.set_xlabel('$w_0$')\n",
    "    ax3.set_ylabel('$b$')\n",
    "\n",
    "    # w1 Vs. b\n",
    "    if surf:\n",
    "        ax4 = fig.add_subplot(2, 3, 6, projection='3d')\n",
    "        xx, yy = np.mgrid[0:error_space[w0_,:,:].shape[0], 0:error_space[:,:,b_].shape[1]]\n",
    "        ax4.plot_surface(xx, yy, error_space[w0_,:,:] ,rstride=1, cstride=1, linewidth=0)\n",
    "        ax4.plot([w1_, w1_], [b_, b_], [0, np.max(error_space[w0_,:,:]) + 1.0], 'r')\n",
    "    else:\n",
    "        ax4 = fig.add_subplot(2, 3, 6)\n",
    "        im4 = ax4.imshow(error_space[w0_,:,:].T, cmap='gray')\n",
    "        ax4.plot(w1_, b_,  marker='o', color='r')\n",
    "        # Colorbar\n",
    "        divider = make_axes_locatable(ax4)\n",
    "        cax = divider.append_axes('right', size='5%', pad=0.05)\n",
    "        fig.colorbar(im4, cax=cax, orientation='vertical')\n",
    "    ax4.set_title('$w_1$ Vs. $b$')\n",
    "    ax4.set_xlabel('$w_1$')\n",
    "    ax4.set_ylabel('$b$')\n",
    "    \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "64a5ae94-9832-46e1-aac5-23433832a6ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdf8063e25c14896bb2a62859aa67101",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=10, description='w0_', max=20), IntSlider(value=10, description='w1_', m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_error_a(error_space, w0_, w1_, b_, w0=[-1, 1], w1=[-1, 1], b=[-1, 1], step=0.1, surf=True)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "slider_w0 = widgets.IntSlider(value=10, min=0, max=20)\n",
    "slider_w1 = widgets.IntSlider(value=10, min=0, max=20)\n",
    "slider_b = widgets.IntSlider(value=10, min=0, max=20)\n",
    "\n",
    "widgets.interact(plot_error_a, error_space=widgets.fixed(error_space), \n",
    "                 w0_=slider_w0, w1_=slider_w1, b_=slider_b, \n",
    "                 w0=widgets.fixed([-1, 1]), w1=widgets.fixed([-1, 1]), b=widgets.fixed([-1, 1]), \n",
    "                 step=widgets.fixed(0.1), surf=widgets.fixed(False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5875b50",
   "metadata": {},
   "source": [
    "## Bibliography\n",
    "---\n",
    "\n",
    "* GONZALEZ, R.C.; WOODS, R.E. Digital Image Processing. 3rd ed. Pearson, 2007."
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Índice",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "261.2px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}