notebook/Autograd_Engine_&_NN_Library.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "class Value:\n",
        "  \n",
        "  def __init__(self, data, _children=(), _op='', label=''):\n",
        "    # Convert data to a NumPy array\n",
        "    if isinstance(data, np.ndarray):\n",
        "      self.data = data\n",
        "    else:\n",
        "      self.data = np.array(data)\n",
        "      \n",
        "    # Initialize the gradient as an array of zeros with the same shape as the data\n",
        "    self.grad = np.zeros_like(self.data)\n",
        "    \n",
        "    # Placeholder for the backward function\n",
        "    self._backward = lambda: None\n",
        "    \n",
        "    # Set of previous nodes (parents)\n",
        "    self._prev = set(_children)\n",
        "    \n",
        "    # Operation associated with the current node\n",
        "    self._op = _op\n",
        "    \n",
        "    # Optional label for the node\n",
        "    self.label = label\n",
        "\n",
        "  \n",
        "  def __add__(self, other):\n",
        "    # Convert other to a Value if it is not already\n",
        "    other = other if isinstance(other, Value) else Value(other)\n",
        "    \n",
        "    # Create a new Value instance representing the addition operation\n",
        "    out = Value(self.data + other.data, (self, other), '+')\n",
        "\n",
        "    def _backward():\n",
        "      # Calculate the gradients using the chain rule and update the gradients of the operands\n",
        "      self.grad = self.grad + (1.0 * out.grad)\n",
        "      other.grad = other.grad + (1.0 * out.grad)\n",
        "\n",
        "    # Assign the backward function to the new Value instance\n",
        "    out._backward = _backward\n",
        "    return out\n",
        "\n",
        "\n",
        "  def __mul__(self, other):\n",
        "    # Convert other to a Value if it is not already\n",
        "    other = other if isinstance(other, Value) else Value(other)\n",
        "    \n",
        "    # Create a new Value instance representing the multiplication operation\n",
        "    out = Value(self.data * other.data, (self, other), '*')\n",
        "\n",
        "    def _backward():\n",
        "      # Calculate the gradients using the chain rule and update the gradients of the operands\n",
        "      self.grad = self.grad + (other.data * out.grad)\n",
        "      other.grad = other.grad + (self.data * out.grad)\n",
        "\n",
        "    # Assign the backward function to the new Value instance\n",
        "    out._backward = _backward\n",
        "    return out\n",
        "\n",
        "\n",
        "  def __pow__(self, other):\n",
        "    if isinstance(other, (int, float)):\n",
        "      # If the exponent is a scalar, perform element-wise power operation\n",
        "      out_data = np.power(self.data, other)\n",
        "      out = Value(out_data, (self,), f'**{other}')\n",
        "\n",
        "      def _backward():\n",
        "        # Calculate the gradients using the chain rule and update the gradients of the operand\n",
        "        self.grad = self.grad + ((other * np.power(self.data, other - 1)) * out.grad)\n",
        "\n",
        "      out._backward = _backward\n",
        "      return out\n",
        "    elif isinstance(other, Value):\n",
        "      # If the exponent is a Value instance, perform element-wise power operation\n",
        "      out_data = np.power(self.data, other.data)\n",
        "      out = Value(out_data, (self, other), f'**')\n",
        "\n",
        "      def _backward():\n",
        "        # Calculate the gradients using the chain rule and update the gradients of the operands\n",
        "        self.grad = self.grad + ((other.data * np.power(self.data * other.data - 1)) * out.grad)\n",
        "        other.grad = other.grad + (np.log(self.data) * out.grad)\n",
        "\n",
        "      out._backward = _backward\n",
        "      return out\n",
        "    else:\n",
        "      raise TypeError(\"Unsupported operand type(s) for **: 'Value' and '{}'\".format(type(other).__name__))\n",
        "\n",
        "\n",
        "  def __radd__(self, other):\n",
        "    # Perform right addition by the Value instance\n",
        "    return self + other\n",
        "\n",
        "\n",
        "  def __rmul__(self, other):\n",
        "    # Perform right multiplication by the Value instance\n",
        "    return self * other\n",
        "\n",
        "\n",
        "  def __truediv__(self, other):\n",
        "    # Perform true division by the Value instance\n",
        "    return self * other**-1\n",
        "\n",
        "\n",
        "  def __neg__(self):\n",
        "    # Perform negation of the Value instance\n",
        "    return self * -1\n",
        "\n",
        "\n",
        "  def __sub__(self, other):\n",
        "    # Perform subtraction of a Value instance\n",
        "    return self + (-other)\n",
        "\n",
        "\n",
        "  def exp(self):\n",
        "    # Compute the element-wise exponential of the Value instance\n",
        "    x = self.data\n",
        "    out = Value(np.exp(x), (self, ), 'exp')\n",
        "\n",
        "    def _backward():\n",
        "        # Calculate the gradients using the chain rule\n",
        "        self.grad = self.grad + (out.data * out.grad)\n",
        "\n",
        "    # Assign the backward function to the new Value instance\n",
        "    out._backward = _backward\n",
        "    return out\n",
        "\n",
        "\n",
        "  def tanh(self):\n",
        "    # Compute the element-wise hyperbolic tangent of the Value instance\n",
        "    x = self.data\n",
        "    t = np.tanh(x)\n",
        "    out = Value(t, (self, ), 'tanh')\n",
        "\n",
        "    def _backward():\n",
        "        # Calculate the gradients using the chain rule\n",
        "        self.grad = self.grad + ((1 - t**2) * out.grad)\n",
        "\n",
        "    # Assign the backward function to the new Value instance\n",
        "    out._backward = _backward\n",
        "    return out\n",
        "\n",
        "\n",
        "  def backward(self):\n",
        "    # Perform backpropagation to compute gradients\n",
        "    topo = []\n",
        "    visited = set()\n",
        "\n",
        "    def build_topo(v):\n",
        "        if v not in visited:\n",
        "            visited.add(v)\n",
        "            for child in v._prev:\n",
        "                build_topo(child)\n",
        "            topo.append(v)\n",
        "\n",
        "    # Build the topological order of nodes\n",
        "    build_topo(self)\n",
        "\n",
        "    # Set the gradient of the output to ones (assuming scalar loss)\n",
        "    self.grad = np.ones_like(self.data)\n",
        "\n",
        "    # Perform backward pass through the nodes in reverse topological order\n",
        "    for node in reversed(topo):\n",
        "        node._backward()\n",
        "\n",
        "\n",
        "  def __repr__(self):\n",
        "    # String representation of the Value instance\n",
        "    return f\"Value(data={self.data})\""
      ],
      "metadata": {
        "id": "V1WaNmBccejW"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from graphviz import Digraph\n",
        "\n",
        "def trace(root):\n",
        "    # Initialize empty sets for nodes and edges\n",
        "    nodes, edges = set(), set()\n",
        "\n",
        "    # Recursive function to build the set of nodes and edges\n",
        "    def build(v):\n",
        "        # If the current node is not in the set of nodes\n",
        "        if v not in nodes:\n",
        "            # Add the current node to the set of nodes\n",
        "            nodes.add(v)\n",
        "            \n",
        "            # Iterate over the previous nodes (parents) of the current node\n",
        "            for child in v._prev:\n",
        "                # Add an edge from the previous node to the current node\n",
        "                edges.add((child, v))\n",
        "                \n",
        "                # Recursively call build() on the previous node\n",
        "                build(child)\n",
        "\n",
        "    # Start building the set of nodes and edges from the root node\n",
        "    build(root)\n",
        "\n",
        "    # Return the set of nodes and edges\n",
        "    return nodes, edges\n",
        "\n",
        "\n",
        "def draw_dot(root):\n",
        "    # Create a Digraph object with SVG format and set the graph attribute 'rankdir' to 'LR'\n",
        "    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'})\n",
        "\n",
        "    # Retrieve the nodes and edges from the root\n",
        "    nodes, edges = trace(root)\n",
        "\n",
        "    # Iterate over the nodes\n",
        "    for n in nodes:\n",
        "        uid = str(id(n))\n",
        "        # Convert the data array to a string representation\n",
        "        data_str = np.array2string(n.data, precision=4, separator=',', suppress_small=True)\n",
        "        # Convert the gradient array to a string representation\n",
        "        grad_str = np.array2string(n.grad, precision=4, separator=',', suppress_small=True)\n",
        "        # Create the label for the node using the node's label, data, and gradient strings\n",
        "        label = \"{ %s | data %s | grad %s }\" % (n.label, data_str, grad_str)\n",
        "        # Add a node to the graph with a unique ID, label, and shape 'record'\n",
        "        dot.node(name=uid, label=label, shape='record')\n",
        "        \n",
        "        # If the node has an operation associated with it\n",
        "        if n._op:\n",
        "            # Add a node to the graph with a unique ID based on the node's ID and operation, and label it with the operation\n",
        "            dot.node(name=uid + n._op, label=n._op)\n",
        "            # Add an edge from the operation node to the current node\n",
        "            dot.edge(uid + n._op, uid)\n",
        "\n",
        "    # Iterate over the edges\n",
        "    for n1, n2 in edges:\n",
        "        # Add an edge from the first node's ID to the second node's ID concatenated with the operation name\n",
        "        dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
        "\n",
        "    # Return the generated graph\n",
        "    return dot"
      ],
      "metadata": {
        "id": "fd168_ch2LSO"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# constructing and visualizing the computation graph for a simple neural network forward pass\n",
        "\n",
        "# inputs x1, x2\n",
        "x1 = Value([[2.0, 2.0], [2.0, 2.0]], label=\"x1\")\n",
        "x2 = Value([[0.0, 0.0], [0.0, 0.0]], label=\"x2\")\n",
        "\n",
        "# weights w1, w2\n",
        "w1 = Value([[-3.0, -3.0], [-3.0, -3.0]], label=\"w1\")\n",
        "w2 = Value([[1.0, 1.0], [1.0, 1.0]], label=\"w2\")\n",
        "\n",
        "# bias of the neurons\n",
        "b = Value([[6.8813735870195432, 6.8813735870195432], [6.8813735870195432, 6.8813735870195432]], label=\"b\")\n",
        "\n",
        "# x1*w1 + x2*w2 + b\n",
        "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
        "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
        "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1w1 + x2w2'\n",
        "n = x1w1x2w2 + b; n.label = 'n'\n",
        "\n",
        "# apply the hyperbolic tangent function to n\n",
        "o = n.tanh(); o.label = 'o'\n",
        "\n",
        "# visualize the computation graph\n",
        "draw_dot(o)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321
        },
        "id": "A4-4na2ILMAh",
        "outputId": "0e818212-52d9-4522-de63-2e4d3071c508"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.43.0 (0)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"2302pt\" height=\"210pt\"\n viewBox=\"0.00 0.00 2302.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n<title>%3</title>\n<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 2298,-206 2298,4 -4,4\"/>\n<!-- 139966976160768 -->\n<g id=\"node1\" class=\"node\">\n<title>139966976160768</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"441,-55.5 441,-91.5 749,-91.5 749,-55.5 441,-55.5\"/>\n<text text-anchor=\"middle\" x=\"467.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"494,-55.5 494,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"557.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.,0.], [0.,0.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"621,-55.5 621,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"685\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976162016+ -->\n<g id=\"node8\" class=\"node\">\n<title>139966976162016+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"821\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"821\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976160768&#45;&gt;139966976162016+ -->\n<g id=\"edge14\" class=\"edge\">\n<title>139966976160768&#45;&gt;139966976162016+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M745.41,-91.51C759.52,-93.21 772.74,-94.8 784.01,-96.16\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"783.83,-99.67 794.18,-97.39 784.67,-92.72 783.83,-99.67\"/>\n</g>\n<!-- 139966976160768* -->\n<g id=\"node2\" class=\"node\">\n<title>139966976160768*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"369\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"369\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966976160768*&#45;&gt;139966976160768 -->\n<g id=\"edge1\" class=\"edge\">\n<title>139966976160768*&#45;&gt;139966976160768</title>\n<path fill=\"none\" stroke=\"black\" d=\"M396.13,-73.5C405.79,-73.5 417.58,-73.5 430.53,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"430.66,-77 440.66,-73.5 430.66,-70 430.66,-77\"/>\n</g>\n<!-- 139966976159280 -->\n<g id=\"node3\" class=\"node\">\n<title>139966976159280</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1396,-109.5 1396,-145.5 1782,-145.5 1782,-109.5 1396,-109.5\"/>\n<text text-anchor=\"middle\" x=\"1407.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1419,-109.5 1419,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"1536.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.8814,0.8814], [0.8814,0.8814]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1654,-109.5 1654,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"1718\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976161776tanh -->\n<g id=\"node15\" class=\"node\">\n<title>139966976161776tanh</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1845\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1845\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n</g>\n<!-- 139966976159280&#45;&gt;139966976161776tanh -->\n<g id=\"edge12\" class=\"edge\">\n<title>139966976159280&#45;&gt;139966976161776tanh</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1782.19,-127.5C1791.45,-127.5 1800.08,-127.5 1807.77,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1808,-131 1818,-127.5 1808,-124 1808,-131\"/>\n</g>\n<!-- 139966976159280+ -->\n<g id=\"node4\" class=\"node\">\n<title>139966976159280+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1333\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1333\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976159280+&#45;&gt;139966976159280 -->\n<g id=\"edge2\" class=\"edge\">\n<title>139966976159280+&#45;&gt;139966976159280</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1360.23,-127.5C1367.56,-127.5 1376.15,-127.5 1385.62,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1385.79,-131 1395.79,-127.5 1385.79,-124 1385.79,-131\"/>\n</g>\n<!-- 139966991612464 -->\n<g id=\"node5\" class=\"node\">\n<title>139966991612464</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"9,-55.5 9,-91.5 297,-91.5 297,-55.5 9,-55.5\"/>\n<text text-anchor=\"middle\" x=\"25.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"42,-55.5 42,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"105.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[1.,1.], [1.,1.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"169,-55.5 169,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"233\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966991612464&#45;&gt;139966976160768* -->\n<g id=\"edge13\" class=\"edge\">\n<title>139966991612464&#45;&gt;139966976160768*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M297.04,-73.5C309.63,-73.5 321.46,-73.5 331.73,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"331.82,-77 341.82,-73.5 331.82,-70 331.82,-77\"/>\n</g>\n<!-- 139966991610544 -->\n<g id=\"node6\" class=\"node\">\n<title>139966991610544</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"0,-165.5 0,-201.5 306,-201.5 306,-165.5 0,-165.5\"/>\n<text text-anchor=\"middle\" x=\"16.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"33,-165.5 33,-201.5 \"/>\n<text text-anchor=\"middle\" x=\"105.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[&#45;3.,&#45;3.], [&#45;3.,&#45;3.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"178,-165.5 178,-201.5 \"/>\n<text text-anchor=\"middle\" x=\"242\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976160096* -->\n<g id=\"node12\" class=\"node\">\n<title>139966976160096*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"369\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"369\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966991610544&#45;&gt;139966976160096* -->\n<g id=\"edge8\" class=\"edge\">\n<title>139966991610544&#45;&gt;139966976160096*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M272.52,-165.42C283.93,-162.79 295.25,-159.83 306,-156.5 316.77,-153.17 328.14,-148.43 338.15,-143.79\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"339.76,-146.9 347.27,-139.43 336.74,-140.58 339.76,-146.9\"/>\n</g>\n<!-- 139966976162016 -->\n<g id=\"node7\" class=\"node\">\n<title>139966976162016</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"894.5,-82.5 894.5,-118.5 1259.5,-118.5 1259.5,-82.5 894.5,-82.5\"/>\n<text text-anchor=\"middle\" x=\"940.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1w1 + x2w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"986.5,-82.5 986.5,-118.5 \"/>\n<text text-anchor=\"middle\" x=\"1059\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[&#45;6.,&#45;6.], [&#45;6.,&#45;6.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1131.5,-82.5 1131.5,-118.5 \"/>\n<text text-anchor=\"middle\" x=\"1195.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976162016&#45;&gt;139966976159280+ -->\n<g id=\"edge7\" class=\"edge\">\n<title>139966976162016&#45;&gt;139966976159280+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1247.42,-118.51C1265.37,-120.42 1282.05,-122.19 1295.76,-123.65\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1295.78,-127.17 1306.1,-124.75 1296.52,-120.21 1295.78,-127.17\"/>\n</g>\n<!-- 139966976162016+&#45;&gt;139966976162016 -->\n<g id=\"edge3\" class=\"edge\">\n<title>139966976162016+&#45;&gt;139966976162016</title>\n<path fill=\"none\" stroke=\"black\" d=\"M848.23,-100.5C858.22,-100.5 870.54,-100.5 884.23,-100.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"884.43,-104 894.43,-100.5 884.43,-97 884.43,-104\"/>\n</g>\n<!-- 139966976152800 -->\n<g id=\"node9\" class=\"node\">\n<title>139966976152800</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"10.5,-110.5 10.5,-146.5 295.5,-146.5 295.5,-110.5 10.5,-110.5\"/>\n<text text-anchor=\"middle\" x=\"25.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"40.5,-110.5 40.5,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"104\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[2.,2.], [2.,2.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"167.5,-110.5 167.5,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"231.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976152800&#45;&gt;139966976160096* -->\n<g id=\"edge10\" class=\"edge\">\n<title>139966976152800&#45;&gt;139966976160096*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M295.55,-128.5C308.72,-128.5 321.12,-128.5 331.8,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"331.88,-132 341.88,-128.5 331.88,-125 331.88,-132\"/>\n</g>\n<!-- 139966991608816 -->\n<g id=\"node10\" class=\"node\">\n<title>139966991608816</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"884,-137.5 884,-173.5 1270,-173.5 1270,-137.5 884,-137.5\"/>\n<text text-anchor=\"middle\" x=\"895.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"907,-137.5 907,-173.5 \"/>\n<text text-anchor=\"middle\" x=\"1024.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[6.8814,6.8814], [6.8814,6.8814]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1142,-137.5 1142,-173.5 \"/>\n<text text-anchor=\"middle\" x=\"1206\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966991608816&#45;&gt;139966976159280+ -->\n<g id=\"edge9\" class=\"edge\">\n<title>139966991608816&#45;&gt;139966976159280+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1241.38,-137.49C1261.88,-135.23 1281,-133.12 1296.34,-131.43\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1296.77,-134.9 1306.33,-130.33 1296,-127.95 1296.77,-134.9\"/>\n</g>\n<!-- 139966976160096 -->\n<g id=\"node11\" class=\"node\">\n<title>139966976160096</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"432,-110.5 432,-146.5 758,-146.5 758,-110.5 432,-110.5\"/>\n<text text-anchor=\"middle\" x=\"458.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"485,-110.5 485,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"557.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[&#45;6.,&#45;6.], [&#45;6.,&#45;6.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"630,-110.5 630,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"694\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976160096&#45;&gt;139966976162016+ -->\n<g id=\"edge11\" class=\"edge\">\n<title>139966976160096&#45;&gt;139966976162016+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M740.08,-110.49C756.27,-108.47 771.48,-106.56 784.2,-104.97\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"784.78,-108.43 794.27,-103.72 783.91,-101.48 784.78,-108.43\"/>\n</g>\n<!-- 139966976160096*&#45;&gt;139966976160096 -->\n<g id=\"edge4\" class=\"edge\">\n<title>139966976160096*&#45;&gt;139966976160096</title>\n<path fill=\"none\" stroke=\"black\" d=\"M396.13,-128.5C403.5,-128.5 412.11,-128.5 421.52,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"421.62,-132 431.62,-128.5 421.62,-125 421.62,-132\"/>\n</g>\n<!-- 139966991611312 -->\n<g id=\"node13\" class=\"node\">\n<title>139966991611312</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"10.5,-0.5 10.5,-36.5 295.5,-36.5 295.5,-0.5 10.5,-0.5\"/>\n<text text-anchor=\"middle\" x=\"25.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"40.5,-0.5 40.5,-36.5 \"/>\n<text text-anchor=\"middle\" x=\"104\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.,0.], [0.,0.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"167.5,-0.5 167.5,-36.5 \"/>\n<text text-anchor=\"middle\" x=\"231.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966991611312&#45;&gt;139966976160768* -->\n<g id=\"edge6\" class=\"edge\">\n<title>139966991611312&#45;&gt;139966976160768*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M267.99,-36.56C280.91,-39.47 293.81,-42.78 306,-46.5 316.53,-49.72 327.66,-54.2 337.54,-58.59\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"336.39,-61.91 346.94,-62.89 339.3,-55.54 336.39,-61.91\"/>\n</g>\n<!-- 139966976161776 -->\n<g id=\"node14\" class=\"node\">\n<title>139966976161776</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1908,-109.5 1908,-145.5 2294,-145.5 2294,-109.5 1908,-109.5\"/>\n<text text-anchor=\"middle\" x=\"1919.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1931,-109.5 1931,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"2048.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.7071,0.7071], [0.7071,0.7071]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2166,-109.5 2166,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"2230\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966976161776tanh&#45;&gt;139966976161776 -->\n<g id=\"edge5\" class=\"edge\">\n<title>139966976161776tanh&#45;&gt;139966976161776</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1872.23,-127.5C1879.56,-127.5 1888.15,-127.5 1897.62,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1897.79,-131 1907.79,-127.5 1897.79,-124 1897.79,-131\"/>\n</g>\n</g>\n</svg>\n",
            "text/plain": [
              "<graphviz.graphs.Digraph at 0x7f4c99e4f6d0>"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# perform backward propagation to compute gradients\n",
        "o.backward()\n",
        "\n",
        "# visualizing the updated computation graph\n",
        "draw_dot(o)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321
        },
        "id": "CX0Kr-4HMyfk",
        "outputId": "d9d5d3f5-978b-4fe1-e818-ceaf34d6d3b3"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.43.0 (0)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"2407pt\" height=\"210pt\"\n viewBox=\"0.00 0.00 2407.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n<title>%3</title>\n<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 2403,-206 2403,4 -4,4\"/>\n<!-- 139966976160768 -->\n<g id=\"node1\" class=\"node\">\n<title>139966976160768</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"465,-55.5 465,-91.5 800,-91.5 800,-55.5 465,-55.5\"/>\n<text text-anchor=\"middle\" x=\"491.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"518,-55.5 518,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"581.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.,0.], [0.,0.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"645,-55.5 645,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"722.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.5,0.5], [0.5,0.5]]</text>\n</g>\n<!-- 139966976162016+ -->\n<g id=\"node8\" class=\"node\">\n<title>139966976162016+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"872\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"872\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976160768&#45;&gt;139966976162016+ -->\n<g id=\"edge14\" class=\"edge\">\n<title>139966976160768&#45;&gt;139966976162016+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M791.92,-91.51C807.71,-93.3 822.45,-94.98 834.82,-96.39\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"834.71,-99.9 845.05,-97.55 835.51,-92.94 834.71,-99.9\"/>\n</g>\n<!-- 139966976160768* -->\n<g id=\"node2\" class=\"node\">\n<title>139966976160768*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"393\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"393\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966976160768*&#45;&gt;139966976160768 -->\n<g id=\"edge1\" class=\"edge\">\n<title>139966976160768*&#45;&gt;139966976160768</title>\n<path fill=\"none\" stroke=\"black\" d=\"M420.32,-73.5C429.95,-73.5 441.69,-73.5 454.66,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"454.81,-77 464.81,-73.5 454.81,-70 454.81,-77\"/>\n</g>\n<!-- 139966976159280 -->\n<g id=\"node3\" class=\"node\">\n<title>139966976159280</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1474,-109.5 1474,-145.5 1887,-145.5 1887,-109.5 1474,-109.5\"/>\n<text text-anchor=\"middle\" x=\"1485.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1497,-109.5 1497,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"1614.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.8814,0.8814], [0.8814,0.8814]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1732,-109.5 1732,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"1809.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.5,0.5], [0.5,0.5]]</text>\n</g>\n<!-- 139966976161776tanh -->\n<g id=\"node15\" class=\"node\">\n<title>139966976161776tanh</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1950\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1950\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n</g>\n<!-- 139966976159280&#45;&gt;139966976161776tanh -->\n<g id=\"edge12\" class=\"edge\">\n<title>139966976159280&#45;&gt;139966976161776tanh</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1887.29,-127.5C1896.5,-127.5 1905.06,-127.5 1912.69,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1912.83,-131 1922.83,-127.5 1912.83,-124 1912.83,-131\"/>\n</g>\n<!-- 139966976159280+ -->\n<g id=\"node4\" class=\"node\">\n<title>139966976159280+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1411\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1411\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976159280+&#45;&gt;139966976159280 -->\n<g id=\"edge2\" class=\"edge\">\n<title>139966976159280+&#45;&gt;139966976159280</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1438.14,-127.5C1445.5,-127.5 1454.15,-127.5 1463.7,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1463.98,-131 1473.98,-127.5 1463.98,-124 1463.98,-131\"/>\n</g>\n<!-- 139966991612464 -->\n<g id=\"node5\" class=\"node\">\n<title>139966991612464</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"21,-55.5 21,-91.5 309,-91.5 309,-55.5 21,-55.5\"/>\n<text text-anchor=\"middle\" x=\"37.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"54,-55.5 54,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"117.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[1.,1.], [1.,1.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"181,-55.5 181,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"245\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.,0.], [0.,0.]]</text>\n</g>\n<!-- 139966991612464&#45;&gt;139966976160768* -->\n<g id=\"edge13\" class=\"edge\">\n<title>139966991612464&#45;&gt;139966976160768*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M309.13,-73.5C326.27,-73.5 342.42,-73.5 355.82,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"355.95,-77 365.95,-73.5 355.95,-70 355.95,-77\"/>\n</g>\n<!-- 139966991610544 -->\n<g id=\"node6\" class=\"node\">\n<title>139966991610544</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"12,-165.5 12,-201.5 318,-201.5 318,-165.5 12,-165.5\"/>\n<text text-anchor=\"middle\" x=\"28.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"45,-165.5 45,-201.5 \"/>\n<text text-anchor=\"middle\" x=\"117.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[&#45;3.,&#45;3.], [&#45;3.,&#45;3.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"190,-165.5 190,-201.5 \"/>\n<text text-anchor=\"middle\" x=\"254\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[1.,1.], [1.,1.]]</text>\n</g>\n<!-- 139966976160096* -->\n<g id=\"node12\" class=\"node\">\n<title>139966976160096*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"393\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"393\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966991610544&#45;&gt;139966976160096* -->\n<g id=\"edge8\" class=\"edge\">\n<title>139966991610544&#45;&gt;139966976160096*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M295.64,-165.41C307.35,-162.8 318.95,-159.84 330,-156.5 340.79,-153.24 352.16,-148.51 362.17,-143.87\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"363.79,-146.98 371.29,-139.5 360.76,-140.66 363.79,-146.98\"/>\n</g>\n<!-- 139966976162016 -->\n<g id=\"node7\" class=\"node\">\n<title>139966976162016</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"945.5,-82.5 945.5,-118.5 1337.5,-118.5 1337.5,-82.5 945.5,-82.5\"/>\n<text text-anchor=\"middle\" x=\"991.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1w1 + x2w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1037.5,-82.5 1037.5,-118.5 \"/>\n<text text-anchor=\"middle\" x=\"1110\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[&#45;6.,&#45;6.], [&#45;6.,&#45;6.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1182.5,-82.5 1182.5,-118.5 \"/>\n<text text-anchor=\"middle\" x=\"1260\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.5,0.5], [0.5,0.5]]</text>\n</g>\n<!-- 139966976162016&#45;&gt;139966976159280+ -->\n<g id=\"edge7\" class=\"edge\">\n<title>139966976162016&#45;&gt;139966976159280+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1320.93,-118.51C1340.82,-120.52 1359.23,-122.38 1374.08,-123.87\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1373.94,-127.38 1384.24,-124.9 1374.64,-120.41 1373.94,-127.38\"/>\n</g>\n<!-- 139966976162016+&#45;&gt;139966976162016 -->\n<g id=\"edge3\" class=\"edge\">\n<title>139966976162016+&#45;&gt;139966976162016</title>\n<path fill=\"none\" stroke=\"black\" d=\"M899.14,-100.5C909.08,-100.5 921.35,-100.5 935.05,-100.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"935.27,-104 945.27,-100.5 935.27,-97 935.27,-104\"/>\n</g>\n<!-- 139966976152800 -->\n<g id=\"node9\" class=\"node\">\n<title>139966976152800</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 330,-146.5 330,-110.5 0,-110.5\"/>\n<text text-anchor=\"middle\" x=\"15\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"30,-110.5 30,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"93.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[2.,2.], [2.,2.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"157,-110.5 157,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"243.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[&#45;1.5,&#45;1.5], [&#45;1.5,&#45;1.5]]</text>\n</g>\n<!-- 139966976152800&#45;&gt;139966976160096* -->\n<g id=\"edge10\" class=\"edge\">\n<title>139966976152800&#45;&gt;139966976160096*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M330.24,-128.5C339.36,-128.5 347.92,-128.5 355.59,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"355.8,-132 365.8,-128.5 355.8,-125 355.8,-132\"/>\n</g>\n<!-- 139966991608816 -->\n<g id=\"node10\" class=\"node\">\n<title>139966991608816</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"935,-137.5 935,-173.5 1348,-173.5 1348,-137.5 935,-137.5\"/>\n<text text-anchor=\"middle\" x=\"946.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"958,-137.5 958,-173.5 \"/>\n<text text-anchor=\"middle\" x=\"1075.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[6.8814,6.8814], [6.8814,6.8814]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1193,-137.5 1193,-173.5 \"/>\n<text text-anchor=\"middle\" x=\"1270.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.5,0.5], [0.5,0.5]]</text>\n</g>\n<!-- 139966991608816&#45;&gt;139966976159280+ -->\n<g id=\"edge9\" class=\"edge\">\n<title>139966991608816&#45;&gt;139966976159280+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1314.56,-137.49C1336.86,-135.16 1357.6,-132.98 1374.02,-131.27\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1374.57,-134.73 1384.15,-130.21 1373.84,-127.77 1374.57,-134.73\"/>\n</g>\n<!-- 139966976160096 -->\n<g id=\"node11\" class=\"node\">\n<title>139966976160096</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"456,-110.5 456,-146.5 809,-146.5 809,-110.5 456,-110.5\"/>\n<text text-anchor=\"middle\" x=\"482.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"509,-110.5 509,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"581.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[&#45;6.,&#45;6.], [&#45;6.,&#45;6.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"654,-110.5 654,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"731.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.5,0.5], [0.5,0.5]]</text>\n</g>\n<!-- 139966976160096&#45;&gt;139966976162016+ -->\n<g id=\"edge11\" class=\"edge\">\n<title>139966976160096&#45;&gt;139966976162016+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M786.26,-110.49C804.35,-108.36 821.28,-106.36 835.18,-104.72\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"835.67,-108.19 845.19,-103.54 834.85,-101.24 835.67,-108.19\"/>\n</g>\n<!-- 139966976160096*&#45;&gt;139966976160096 -->\n<g id=\"edge4\" class=\"edge\">\n<title>139966976160096*&#45;&gt;139966976160096</title>\n<path fill=\"none\" stroke=\"black\" d=\"M420.32,-128.5C427.67,-128.5 436.26,-128.5 445.69,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"445.79,-132 455.79,-128.5 445.79,-125 445.79,-132\"/>\n</g>\n<!-- 139966991611312 -->\n<g id=\"node13\" class=\"node\">\n<title>139966991611312</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"9,-0.5 9,-36.5 321,-36.5 321,-0.5 9,-0.5\"/>\n<text text-anchor=\"middle\" x=\"24\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"39,-0.5 39,-36.5 \"/>\n<text text-anchor=\"middle\" x=\"102.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.,0.], [0.,0.]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"166,-0.5 166,-36.5 \"/>\n<text text-anchor=\"middle\" x=\"243.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[0.5,0.5], [0.5,0.5]]</text>\n</g>\n<!-- 139966991611312&#45;&gt;139966976160768* -->\n<g id=\"edge6\" class=\"edge\">\n<title>139966991611312&#45;&gt;139966976160768*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M290.75,-36.54C304.11,-39.44 317.41,-42.74 330,-46.5 340.66,-49.68 351.92,-54.2 361.87,-58.64\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"360.41,-61.82 370.96,-62.82 363.34,-55.46 360.41,-61.82\"/>\n</g>\n<!-- 139966976161776 -->\n<g id=\"node14\" class=\"node\">\n<title>139966976161776</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"2013,-109.5 2013,-145.5 2399,-145.5 2399,-109.5 2013,-109.5\"/>\n<text text-anchor=\"middle\" x=\"2024.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2036,-109.5 2036,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"2153.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data [[0.7071,0.7071], [0.7071,0.7071]]</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2271,-109.5 2271,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"2335\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad [[1.,1.], [1.,1.]]</text>\n</g>\n<!-- 139966976161776tanh&#45;&gt;139966976161776 -->\n<g id=\"edge5\" class=\"edge\">\n<title>139966976161776tanh&#45;&gt;139966976161776</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1977.23,-127.5C1984.56,-127.5 1993.15,-127.5 2002.62,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2002.79,-131 2012.79,-127.5 2002.79,-124 2002.79,-131\"/>\n</g>\n</g>\n</svg>\n",
            "text/plain": [
              "<graphviz.graphs.Digraph at 0x7f4c99e60910>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# constructing and visualizing the computation graph for a neural network forward pass and performing backward propagation\n",
        "\n",
        "# inputs x1,x2\n",
        "x1 = Value(2.0, label=\"x1\")\n",
        "x2 = Value(0.0, label=\"x2\")\n",
        "\n",
        "# weights w1, w2\n",
        "w1 = Value(-3.0, label=\"w1\")\n",
        "w2 = Value(1.0, label=\"w2\")\n",
        "\n",
        "# bias of the neurons\n",
        "b = Value(6.8813735870195432, label=\"b\")\n",
        "\n",
        "# x1*w1 + x2*w2 + b\n",
        "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
        "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
        "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1w1 + x2w2'\n",
        "n = x1w1x2w2 + b; n.label = 'n'\n",
        "\n",
        "# break tanh() into the following expression (hyperbolic tangent)\n",
        "\n",
        "# computes the value of 'e' by taking the exponential function of the expression '2*n'\n",
        "e = (2*n).exp(); e.label = 'e'\n",
        "# computes the value of 'o' using the values of 'e' & subtracts 1 from 'e' and divides it by 'e' plus 1, following a mathematical expression\n",
        "o = (e - 1) / (e + 1); o.label = 'o'\n",
        "\n",
        "# perform backward propagation to compute gradients\n",
        "o.backward()\n",
        "\n",
        "# visualizing the updated computation graph\n",
        "draw_dot(o)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        },
        "id": "8td2h163Ohv8",
        "outputId": "1c7e84f6-365e-4ec4-e74d-98ff9ce80815"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.43.0 (0)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"2722pt\" height=\"236pt\"\n viewBox=\"0.00 0.00 2722.00 236.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 232)\">\n<title>%3</title>\n<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-232 2718,-232 2718,4 -4,4\"/>\n<!-- 139966976156160 -->\n<g id=\"node1\" class=\"node\">\n<title>139966976156160</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"574,-82.5 574,-118.5 786,-118.5 786,-82.5 574,-82.5\"/>\n<text text-anchor=\"middle\" x=\"620\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1w1 + x2w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"666,-82.5 666,-118.5 \"/>\n<text text-anchor=\"middle\" x=\"695\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"724,-82.5 724,-118.5 \"/>\n<text text-anchor=\"middle\" x=\"755\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5</text>\n</g>\n<!-- 139966976154912+ -->\n<g id=\"node19\" class=\"node\">\n<title>139966976154912+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"849\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"849\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976156160&#45;&gt;139966976154912+ -->\n<g id=\"edge27\" class=\"edge\">\n<title>139966976156160&#45;&gt;139966976154912+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M786.29,-117.52C795.62,-119.03 804.56,-120.48 812.6,-121.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"812.17,-125.25 822.6,-123.39 813.28,-118.34 812.17,-125.25\"/>\n</g>\n<!-- 139966976156160+ -->\n<g id=\"node2\" class=\"node\">\n<title>139966976156160+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"511\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"511\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976156160+&#45;&gt;139966976156160 -->\n<g id=\"edge1\" class=\"edge\">\n<title>139966976156160+&#45;&gt;139966976156160</title>\n<path fill=\"none\" stroke=\"black\" d=\"M538.09,-100.5C545.65,-100.5 554.42,-100.5 563.83,-100.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"563.83,-104 573.83,-100.5 563.83,-97 563.83,-104\"/>\n</g>\n<!-- 139966976162304 -->\n<g id=\"node3\" class=\"node\">\n<title>139966976162304</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"275,-110.5 275,-146.5 448,-146.5 448,-110.5 275,-110.5\"/>\n<text text-anchor=\"middle\" x=\"301.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"328,-110.5 328,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"357\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"386,-110.5 386,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"417\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5</text>\n</g>\n<!-- 139966976162304&#45;&gt;139966976156160+ -->\n<g id=\"edge23\" class=\"edge\">\n<title>139966976162304&#45;&gt;139966976156160+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M448.3,-112.21C457.66,-110.44 466.75,-108.71 474.95,-107.15\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"475.64,-110.58 484.82,-105.28 474.34,-103.71 475.64,-110.58\"/>\n</g>\n<!-- 139966976162304* -->\n<g id=\"node4\" class=\"node\">\n<title>139966976162304*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"212\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"212\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966976162304*&#45;&gt;139966976162304 -->\n<g id=\"edge2\" class=\"edge\">\n<title>139966976162304*&#45;&gt;139966976162304</title>\n<path fill=\"none\" stroke=\"black\" d=\"M239.03,-128.5C246.62,-128.5 255.4,-128.5 264.69,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"264.93,-132 274.93,-128.5 264.93,-125 264.93,-132\"/>\n</g>\n<!-- 139966976153088 -->\n<g id=\"node5\" class=\"node\">\n<title>139966976153088</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"2,-55.5 2,-91.5 147,-91.5 147,-55.5 2,-55.5\"/>\n<text text-anchor=\"middle\" x=\"17\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"32,-55.5 32,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"58.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"85,-55.5 85,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"116\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5</text>\n</g>\n<!-- 139966976158512* -->\n<g id=\"node23\" class=\"node\">\n<title>139966976158512*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"212\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"212\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966976153088&#45;&gt;139966976158512* -->\n<g id=\"edge11\" class=\"edge\">\n<title>139966976153088&#45;&gt;139966976158512*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M147.22,-73.5C156.68,-73.5 166.04,-73.5 174.55,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"174.83,-77 184.83,-73.5 174.83,-70 174.83,-77\"/>\n</g>\n<!-- 139966976153136 -->\n<g id=\"node6\" class=\"node\">\n<title>139966976153136</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"0,-165.5 0,-201.5 149,-201.5 149,-165.5 0,-165.5\"/>\n<text text-anchor=\"middle\" x=\"15\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"30,-165.5 30,-201.5 \"/>\n<text text-anchor=\"middle\" x=\"56.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"83,-165.5 83,-201.5 \"/>\n<text text-anchor=\"middle\" x=\"116\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5</text>\n</g>\n<!-- 139966976153136&#45;&gt;139966976162304* -->\n<g id=\"edge19\" class=\"edge\">\n<title>139966976153136&#45;&gt;139966976162304*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M126.29,-165.38C133.94,-162.5 141.7,-159.48 149,-156.5 159.36,-152.27 170.54,-147.33 180.51,-142.79\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"182.02,-145.96 189.65,-138.6 179.1,-139.59 182.02,-145.96\"/>\n</g>\n<!-- 139966976152656 -->\n<g id=\"node7\" class=\"node\">\n<title>139966976152656</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1203,-136.5 1203,-172.5 1371,-172.5 1371,-136.5 1203,-136.5\"/>\n<text text-anchor=\"middle\" x=\"1213\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1223,-136.5 1223,-172.5 \"/>\n<text text-anchor=\"middle\" x=\"1263\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.7627</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1303,-136.5 1303,-172.5 \"/>\n<text text-anchor=\"middle\" x=\"1337\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.25</text>\n</g>\n<!-- 139966976152512exp -->\n<g id=\"node27\" class=\"node\">\n<title>139966976152512exp</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1434\" cy=\"-154.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1434\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">exp</text>\n</g>\n<!-- 139966976152656&#45;&gt;139966976152512exp -->\n<g id=\"edge13\" class=\"edge\">\n<title>139966976152656&#45;&gt;139966976152512exp</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1371.09,-154.5C1379.96,-154.5 1388.62,-154.5 1396.51,-154.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1396.75,-158 1406.75,-154.5 1396.75,-151 1396.75,-158\"/>\n</g>\n<!-- 139966976152656* -->\n<g id=\"node8\" class=\"node\">\n<title>139966976152656*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1140\" cy=\"-154.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1140\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966976152656*&#45;&gt;139966976152656 -->\n<g id=\"edge3\" class=\"edge\">\n<title>139966976152656*&#45;&gt;139966976152656</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1167.26,-154.5C1174.81,-154.5 1183.51,-154.5 1192.7,-154.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1192.83,-158 1202.83,-154.5 1192.83,-151 1192.83,-158\"/>\n</g>\n<!-- 139966976158800 -->\n<g id=\"node9\" class=\"node\">\n<title>139966976158800</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1511.5,-81.5 1511.5,-117.5 1667.5,-117.5 1667.5,-81.5 1511.5,-81.5\"/>\n<text text-anchor=\"middle\" x=\"1521.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1531.5,-81.5 1531.5,-117.5 \"/>\n<text text-anchor=\"middle\" x=\"1558.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1585.5,-81.5 1585.5,-117.5 \"/>\n<text text-anchor=\"middle\" x=\"1626.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.1464</text>\n</g>\n<!-- 139966976158848+ -->\n<g id=\"node12\" class=\"node\">\n<title>139966976158848+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1745\" cy=\"-126.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1745\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976158800&#45;&gt;139966976158848+ -->\n<g id=\"edge20\" class=\"edge\">\n<title>139966976158800&#45;&gt;139966976158848+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1667.7,-113.08C1681.9,-115.58 1696.09,-118.07 1708.31,-120.22\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1708.03,-123.73 1718.49,-122.01 1709.24,-116.83 1708.03,-123.73\"/>\n</g>\n<!-- 139966976150640 -->\n<g id=\"node10\" class=\"node\">\n<title>139966976150640</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"918.5,-164.5 918.5,-200.5 1070.5,-200.5 1070.5,-164.5 918.5,-164.5\"/>\n<text text-anchor=\"middle\" x=\"928.5\" y=\"-178.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"938.5,-164.5 938.5,-200.5 \"/>\n<text text-anchor=\"middle\" x=\"963.5\" y=\"-178.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"988.5,-164.5 988.5,-200.5 \"/>\n<text text-anchor=\"middle\" x=\"1029.5\" y=\"-178.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.2203</text>\n</g>\n<!-- 139966976150640&#45;&gt;139966976152656* -->\n<g id=\"edge15\" class=\"edge\">\n<title>139966976150640&#45;&gt;139966976152656*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1070.61,-167.84C1082.19,-165.59 1093.65,-163.35 1103.78,-161.37\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1104.66,-164.77 1113.81,-159.42 1103.32,-157.9 1104.66,-164.77\"/>\n</g>\n<!-- 139966976158848 -->\n<g id=\"node11\" class=\"node\">\n<title>139966976158848</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"2030,-108.5 2030,-144.5 2212,-144.5 2212,-108.5 2030,-108.5\"/>\n<text text-anchor=\"middle\" x=\"2040\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2050,-108.5 2050,-144.5 \"/>\n<text text-anchor=\"middle\" x=\"2090\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.8284</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2130,-108.5 2130,-144.5 \"/>\n<text text-anchor=\"middle\" x=\"2171\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.1464</text>\n</g>\n<!-- 139966976153376* -->\n<g id=\"node21\" class=\"node\">\n<title>139966976153376*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"2493\" cy=\"-153.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"2493\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 139966976158848&#45;&gt;139966976153376* -->\n<g id=\"edge25\" class=\"edge\">\n<title>139966976158848&#45;&gt;139966976153376*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M2212.04,-133.07C2290.41,-138.79 2399.98,-146.79 2455.96,-150.87\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2455.87,-154.37 2466.1,-151.61 2456.38,-147.39 2455.87,-154.37\"/>\n</g>\n<!-- 139966976158848+&#45;&gt;139966976158848 -->\n<g id=\"edge4\" class=\"edge\">\n<title>139966976158848+&#45;&gt;139966976158848</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1772,-126.5C1822.46,-126.5 1936.06,-126.5 2019.81,-126.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2019.87,-130 2029.87,-126.5 2019.87,-123 2019.87,-130\"/>\n</g>\n<!-- 139966976154240 -->\n<g id=\"node13\" class=\"node\">\n<title>139966976154240</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1808,-163.5 1808,-199.5 1994,-199.5 1994,-163.5 1808,-163.5\"/>\n<text text-anchor=\"middle\" x=\"1818\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1828,-163.5 1828,-199.5 \"/>\n<text text-anchor=\"middle\" x=\"1868\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8284</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1908,-163.5 1908,-199.5 \"/>\n<text text-anchor=\"middle\" x=\"1951\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.1036</text>\n</g>\n<!-- 139966976153808**&#45;1 -->\n<g id=\"node16\" class=\"node\">\n<title>139966976153808**&#45;1</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"2121\" cy=\"-181.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"2121\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">**&#45;1</text>\n</g>\n<!-- 139966976154240&#45;&gt;139966976153808**&#45;1 -->\n<g id=\"edge21\" class=\"edge\">\n<title>139966976154240&#45;&gt;139966976153808**&#45;1</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1994.14,-181.5C2025.72,-181.5 2059.24,-181.5 2083.74,-181.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2083.98,-185 2093.98,-181.5 2083.98,-178 2083.98,-185\"/>\n</g>\n<!-- 139966976154240+ -->\n<g id=\"node14\" class=\"node\">\n<title>139966976154240+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1745\" cy=\"-181.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1745\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 139966976154240+&#45;&gt;139966976154240 -->\n<g id=\"edge5\" class=\"edge\">\n<title>139966976154240+&#45;&gt;139966976154240</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1772.12,-181.5C1779.62,-181.5 1788.29,-181.5 1797.5,-181.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1797.69,-185 1807.69,-181.5 1797.69,-178 1797.69,-185\"/>\n</g>\n<!-- 139966976153808 -->\n<g id=\"node15\" class=\"node\">\n<title>139966976153808</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"2248,-161.5 2248,-197.5 2430,-197.5 2430,-161.5 2248,-161.5\"/>\n<text text-anchor=\"middle\" x=\"2258\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2268,-161.5 2268,-197.5 \"/>\n<text text-anchor=\"middle\" x=\"2308\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.1464</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2348,-161.5 2348,-197.5 \"/>\n<text text-anchor=\"middle\" x=\"2389\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.8284</text>\n</g>\n<!-- 139966976153808&#45;&gt;139966976153376* -->\n<g id=\"edge26\" class=\"edge\">\n<title>139966976153808&#45;&gt;139966976153376*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M2430.18,-164.08C2439.42,-162.49 2448.38,-160.96 2456.47,-159.58\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2457.29,-162.99 2466.56,-157.85 2456.11,-156.09 2457.29,-162.99\"/>\n</g>\n<!-- 139966976153808**&#45;1&#45;&gt;139966976153808 -->\n<g id=\"edge6\" class=\"edge\">\n<title>139966976153808**&#45;1&#45;&gt;139966976153808</title>\n<path fill=\"none\" stroke=\"black\" d=\"M2148.05,-181.26C2170.53,-181.05 2204.48,-180.74 2237.5,-180.43\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2237.74,-183.93 2247.7,-180.34 2237.67,-176.93 2237.74,-183.93\"/>\n</g>\n<!-- 139966976153856 -->\n<g id=\"node17\" class=\"node\">\n<title>139966976153856</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1511.5,-191.5 1511.5,-227.5 1667.5,-227.5 1667.5,-191.5 1511.5,-191.5\"/>\n<text text-anchor=\"middle\" x=\"1521.5\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1531.5,-191.5 1531.5,-227.5 \"/>\n<text text-anchor=\"middle\" x=\"1556.5\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1581.5,-191.5 1581.5,-227.5 \"/>\n<text text-anchor=\"middle\" x=\"1624.5\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.1036</text>\n</g>\n<!-- 139966976153856&#45;&gt;139966976154240+ -->\n<g id=\"edge18\" class=\"edge\">\n<title>139966976153856&#45;&gt;139966976154240+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1667.7,-195.42C1682.16,-192.78 1696.61,-190.14 1708.98,-187.89\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1709.62,-191.33 1718.83,-186.09 1708.36,-184.44 1709.62,-191.33\"/>\n</g>\n<!-- 139966976154912 -->\n<g id=\"node18\" class=\"node\">\n<title>139966976154912</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"912,-109.5 912,-145.5 1077,-145.5 1077,-109.5 912,-109.5\"/>\n<text text-anchor=\"middle\" x=\"923.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"935,-109.5 935,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"975\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1015,-109.5 1015,-145.5 \"/>\n<text text-anchor=\"middle\" x=\"1046\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5</text>\n</g>\n<!-- 139966976154912&#45;&gt;139966976152656* -->\n<g id=\"edge28\" class=\"edge\">\n<title>139966976154912&#45;&gt;139966976152656*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1077.31,-142.89C1086.6,-144.64 1095.67,-146.35 1103.86,-147.89\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1103.25,-151.34 1113.73,-149.74 1104.55,-144.46 1103.25,-151.34\"/>\n</g>\n<!-- 139966976154912+&#45;&gt;139966976154912 -->\n<g id=\"edge7\" class=\"edge\">\n<title>139966976154912+&#45;&gt;139966976154912</title>\n<path fill=\"none\" stroke=\"black\" d=\"M876.32,-127.5C883.87,-127.5 892.55,-127.5 901.71,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"901.8,-131 911.8,-127.5 901.8,-124 901.8,-131\"/>\n</g>\n<!-- 139966976153376 -->\n<g id=\"node20\" class=\"node\">\n<title>139966976153376</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"2556,-135.5 2556,-171.5 2714,-171.5 2714,-135.5 2556,-135.5\"/>\n<text text-anchor=\"middle\" x=\"2567.5\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2579,-135.5 2579,-171.5 \"/>\n<text text-anchor=\"middle\" x=\"2619\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"2659,-135.5 2659,-171.5 \"/>\n<text text-anchor=\"middle\" x=\"2686.5\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.</text>\n</g>\n<!-- 139966976153376*&#45;&gt;139966976153376 -->\n<g id=\"edge8\" class=\"edge\">\n<title>139966976153376*&#45;&gt;139966976153376</title>\n<path fill=\"none\" stroke=\"black\" d=\"M2520,-153.5C2527.61,-153.5 2536.39,-153.5 2545.62,-153.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"2545.78,-157 2555.78,-153.5 2545.78,-150 2545.78,-157\"/>\n</g>\n<!-- 139966976158512 -->\n<g id=\"node22\" class=\"node\">\n<title>139966976158512</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"277.5,-55.5 277.5,-91.5 445.5,-91.5 445.5,-55.5 277.5,-55.5\"/>\n<text text-anchor=\"middle\" x=\"304\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"330.5,-55.5 330.5,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"357\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"383.5,-55.5 383.5,-91.5 \"/>\n<text text-anchor=\"middle\" x=\"414.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5</text>\n</g>\n<!-- 139966976158512&#45;&gt;139966976156160+ -->\n<g id=\"edge22\" class=\"edge\">\n<title>139966976158512&#45;&gt;139966976156160+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M445.73,-88.74C455.97,-90.61 465.96,-92.44 474.9,-94.07\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"474.39,-97.54 484.86,-95.9 475.65,-90.65 474.39,-97.54\"/>\n</g>\n<!-- 139966976158512*&#45;&gt;139966976158512 -->\n<g id=\"edge9\" class=\"edge\">\n<title>139966976158512*&#45;&gt;139966976158512</title>\n<path fill=\"none\" stroke=\"black\" d=\"M239.03,-73.5C247.37,-73.5 257.13,-73.5 267.43,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.44,-77 277.44,-73.5 267.44,-70 267.44,-77\"/>\n</g>\n<!-- 139966976162112 -->\n<g id=\"node24\" class=\"node\">\n<title>139966976162112</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"597.5,-137.5 597.5,-173.5 762.5,-173.5 762.5,-137.5 597.5,-137.5\"/>\n<text text-anchor=\"middle\" x=\"609\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"620.5,-137.5 620.5,-173.5 \"/>\n<text text-anchor=\"middle\" x=\"660.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"700.5,-137.5 700.5,-173.5 \"/>\n<text text-anchor=\"middle\" x=\"731.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5</text>\n</g>\n<!-- 139966976162112&#45;&gt;139966976154912+ -->\n<g id=\"edge16\" class=\"edge\">\n<title>139966976162112&#45;&gt;139966976154912+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M762.54,-141.83C780.13,-138.88 797.84,-135.91 812.53,-133.45\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"813.38,-136.85 822.66,-131.75 812.22,-129.95 813.38,-136.85\"/>\n</g>\n<!-- 139966991608720 -->\n<g id=\"node25\" class=\"node\">\n<title>139966991608720</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1.5,-110.5 1.5,-146.5 147.5,-146.5 147.5,-110.5 1.5,-110.5\"/>\n<text text-anchor=\"middle\" x=\"18\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"34.5,-110.5 34.5,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"63.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"92.5,-110.5 92.5,-146.5 \"/>\n<text text-anchor=\"middle\" x=\"120\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.</text>\n</g>\n<!-- 139966991608720&#45;&gt;139966976162304* -->\n<g id=\"edge14\" class=\"edge\">\n<title>139966991608720&#45;&gt;139966976162304*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M147.61,-128.5C156.95,-128.5 166.17,-128.5 174.57,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"174.71,-132 184.71,-128.5 174.71,-125 174.71,-132\"/>\n</g>\n<!-- 139966976152512 -->\n<g id=\"node26\" class=\"node\">\n<title>139966976152512</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1497,-136.5 1497,-172.5 1682,-172.5 1682,-136.5 1497,-136.5\"/>\n<text text-anchor=\"middle\" x=\"1508.5\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">e</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1520,-136.5 1520,-172.5 \"/>\n<text text-anchor=\"middle\" x=\"1560\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 5.8284</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1600,-136.5 1600,-172.5 \"/>\n<text text-anchor=\"middle\" x=\"1641\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0429</text>\n</g>\n<!-- 139966976152512&#45;&gt;139966976158848+ -->\n<g id=\"edge24\" class=\"edge\">\n<title>139966976152512&#45;&gt;139966976158848+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1682.01,-137.81C1691.49,-136.08 1700.66,-134.41 1708.9,-132.9\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1709.6,-136.33 1718.81,-131.09 1708.35,-129.45 1709.6,-136.33\"/>\n</g>\n<!-- 139966976152512&#45;&gt;139966976154240+ -->\n<g id=\"edge17\" class=\"edge\">\n<title>139966976152512&#45;&gt;139966976154240+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1682.01,-170.6C1691.49,-172.26 1700.66,-173.88 1708.9,-175.33\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1708.36,-178.78 1718.81,-177.07 1709.57,-171.89 1708.36,-178.78\"/>\n</g>\n<!-- 139966976152512exp&#45;&gt;139966976152512 -->\n<g id=\"edge10\" class=\"edge\">\n<title>139966976152512exp&#45;&gt;139966976152512</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1461.04,-154.5C1468.58,-154.5 1477.3,-154.5 1486.57,-154.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1486.81,-158 1496.81,-154.5 1486.81,-151 1486.81,-158\"/>\n</g>\n<!-- 139966976153040 -->\n<g id=\"node28\" class=\"node\">\n<title>139966976153040</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"4,-0.5 4,-36.5 145,-36.5 145,-0.5 4,-0.5\"/>\n<text text-anchor=\"middle\" x=\"20.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"37,-0.5 37,-36.5 \"/>\n<text text-anchor=\"middle\" x=\"63.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"90,-0.5 90,-36.5 \"/>\n<text text-anchor=\"middle\" x=\"117.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.</text>\n</g>\n<!-- 139966976153040&#45;&gt;139966976158512* -->\n<g id=\"edge12\" class=\"edge\">\n<title>139966976153040&#45;&gt;139966976158512*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M126.29,-36.62C133.94,-39.5 141.7,-42.52 149,-45.5 159.36,-49.73 170.54,-54.67 180.51,-59.21\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"179.1,-62.41 189.65,-63.4 182.02,-56.04 179.1,-62.41\"/>\n</g>\n</g>\n</svg>\n",
            "text/plain": [
              "<graphviz.graphs.Digraph at 0x7f4c99e4ed10>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "# create tensors and enable gradient tracking\n",
        "\n",
        "# inputs x1,x2\n",
        "x1 = torch.Tensor([2.0]); x1.requires_grad = True \n",
        "x2 = torch.Tensor([-.0]); x2.requires_grad = True\n",
        "\n",
        "# weights w1, w2\n",
        "w1 = torch.Tensor([-3.0]); w1.requires_grad = True  \n",
        "w2 = torch.Tensor([1.0]); w2.requires_grad = True  \n",
        "\n",
        "# bias of the neurons\n",
        "b = torch.Tensor([6.8813735870195432]); b.requires_grad = True\n",
        "\n",
        "# perform the computation: n = x1*w1 + x2*w2 + b\n",
        "n = x1*w1 + x2*w2 + b\n",
        "\n",
        "# apply the hyperbolic tangent function to n\n",
        "o = torch.tanh(n)\n",
        "\n",
        "print(o.item())\n",
        "\n",
        "# perform backward propagation to compute gradients\n",
        "o.backward()\n",
        "\n",
        "print(\"------------\")\n",
        "print(\"x1\", x1.grad.item())\n",
        "print(\"w1\", w1.grad.item())\n",
        "print(\"x2\", x2.grad.item())\n",
        "print(\"w2\", w2.grad.item())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "REFIqSGtSqGk",
        "outputId": "d88d7dfd-4fe1-4575-e6e1-46ab2397ebe6"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.7071067094802856\n",
            "------------\n",
            "x1 -1.5000003576278687\n",
            "w1 1.000000238418579\n",
            "x2 0.5000001192092896\n",
            "w2 -0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "class Neuron:\n",
        "\n",
        "  def __init__(self, nin):\n",
        "    # Initialize the neuron with random weights and bias between -1 and 1\n",
        "    self.w = [Value(random.uniform(-1, 1)) for _ in range(nin)]\n",
        "    self.b = Value(random.uniform(-1,1))\n",
        "  \n",
        "  def __call__(self, x):\n",
        "    # Calculate the weighted sum of inputs multiplied by weights and add the bias\n",
        "    act = sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
        "    # Apply the hyperbolic tangent function to the activation\n",
        "    out = act.tanh()\n",
        "    # Return the output of the neuron\n",
        "    return out\n",
        "  \n",
        "  def parameters(self):\n",
        "    # Return the weights and bias of the neuron as parameters\n",
        "    return self.w + [self.b]\n",
        "\n",
        "  \n",
        "class Layer:\n",
        "\n",
        "  def __init__(self, nin, nout):\n",
        "    # Create a layer with a specified number of input and output neurons\n",
        "    self.neurons = [Neuron(nin) for _ in range(nout)]\n",
        "  \n",
        "  def __call__(self, x):\n",
        "    # Compute the output of each neuron in the layer given an input\n",
        "    outs = [n(x) for n in self.neurons]\n",
        "    # If there is only one output neuron, return it directly; otherwise, return a list of outputs\n",
        "    return outs[0] if len(outs) == 1 else outs\n",
        "  \n",
        "  def parameters(self):\n",
        "    # Return the parameters of all neurons in the layer\n",
        "    return [p for neuron in self.neurons for p in neuron.parameters()]\n",
        "\n",
        "  \n",
        "class MLP:\n",
        "\n",
        "  def __init__(self, nin, nouts):\n",
        "    # Create a multi-layer perceptron (MLP) with the specified number of input and output neurons for each layer\n",
        "    sz = [nin] + nouts\n",
        "    self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
        "\n",
        "  def __call__(self, x):\n",
        "    # Forward propagate the input through all layers of the MLP\n",
        "    for layer in self.layers:\n",
        "      x = layer(x)\n",
        "    # Return the final output of the MLP\n",
        "    return x\n",
        "\n",
        "  def parameters(self):\n",
        "    # Return the parameters of all layers in the MLP\n",
        "    return [p for layer in self.layers for p in layer.parameters()]"
      ],
      "metadata": {
        "id": "G8HuWjA8jmah"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set the input values\n",
        "x = [2.0, 3.0, -1.0]\n",
        "\n",
        "# Create an MLP object with 3 input neurons, 3 hidden layers (each with 4 neurons), and 1 output neuron\n",
        "n = MLP(3, [4, 4, 1])\n",
        "\n",
        "# Call the MLP object with the input values\n",
        "# This triggers the __call__ method of the MLP class\n",
        "n(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GNkdRcm6jpOq",
        "outputId": "2b71b8b9-7cf8-4838-f3bc-f8fb3d44b77f"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Value(data=0.13582530558482833)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set the input values for the training examples\n",
        "xs = [\n",
        "    [2.0, 3.0, -1.0],\n",
        "    [3.0, -1.0, 0.5],\n",
        "    [0.5, 1.0, 1.0],\n",
        "    [1.0, 1.0, -1.0],\n",
        "]\n",
        "\n",
        "# Set the desired target values for the training examples\n",
        "ys = [1.0, -1.0, -1.0, 1.0]\n",
        "\n",
        "# Training data:\n",
        "# xs[0] corresponds to ys[0]\n",
        "# xs[1] corresponds to ys[1]\n",
        "# xs[2] corresponds to ys[2]\n",
        "# xs[3] corresponds to ys[3]"
      ],
      "metadata": {
        "id": "IIlO_dCAlskf"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for k in range(20):\n",
        "  \n",
        "    # Forward pass\n",
        "    ypred = [n(x) for x in xs]\n",
        "    loss = sum((yout - ygt) ** 2 for ygt, yout in zip(ys, ypred))\n",
        "\n",
        "    # Backward pass\n",
        "    for p in n.parameters():\n",
        "        p.grad = 0.0\n",
        "    \n",
        "    # Compute gradients using the backward method\n",
        "    loss.backward()\n",
        "\n",
        "    # Update weights\n",
        "    for p in n.parameters():\n",
        "        p.data += -0.5 * p.grad\n",
        "\n",
        "    print(k, loss.data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GTTbuq2MmIKt",
        "outputId": "a26c7ab7-ba99-46e6-92d9-a8b1f646a176"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 1.7775435867450773\n",
            "1 0.06611265824278505\n",
            "2 0.006901234745973038\n",
            "3 0.005322222223457995\n",
            "4 0.00435559052011085\n",
            "5 0.0036985923357483346\n",
            "6 0.0032213463630148048\n",
            "7 0.0028582056604797364\n",
            "8 0.0025722214875243394\n",
            "9 0.0023409353789170177\n",
            "10 0.0021498805294869095\n",
            "11 0.001989303823876685\n",
            "12 0.0018523837621004615\n",
            "13 0.0017342022705594778\n",
            "14 0.0016311220038520088\n",
            "15 0.0015403936871487\n",
            "16 0.001459899907529591\n",
            "17 0.0013879829606015203\n",
            "18 0.0013233261908806024\n",
            "19 0.0012648703553402552\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# ypred = [n(x) for x in xs]\n",
        "ypred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e5VyzOp_rJPi",
        "outputId": "618337cb-5974-462a-98bb-8216a5ba672e"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Value(data=0.9954731228138854),\n",
              " Value(data=-0.9704885589563487),\n",
              " Value(data=-0.9826181423902708),\n",
              " Value(data=0.9915546692282492)]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    }
  ]
}