Lesson5_sgd.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Base code (functions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class Node(object):\n",
    "    \"\"\"\n",
    "    Base class for nodes in the network.\n",
    "\n",
    "    Arguments:\n",
    "\n",
    "        `inbound_nodes`: A list of nodes with edges into this node.\n",
    "    \"\"\"\n",
    "    def __init__(self, inbound_nodes=[]):\n",
    "        \"\"\"\n",
    "        Node's constructor (runs when the object is instantiated). Sets\n",
    "        properties that all nodes need.\n",
    "        \"\"\"\n",
    "        # A list of nodes with edges into this node.\n",
    "        self.inbound_nodes = inbound_nodes\n",
    "        # The eventual value of this node. Set by running\n",
    "        # the forward() method.\n",
    "        self.value = None\n",
    "        # A list of nodes that this node outputs to.\n",
    "        self.outbound_nodes = []\n",
    "        # New property! Keys are the inputs to this node and\n",
    "        # their values are the partials of this node with\n",
    "        # respect to that input.\n",
    "        self.gradients = {}\n",
    "        # Sets this node as an outbound node for all of\n",
    "        # this node's inputs.\n",
    "        for node in inbound_nodes:\n",
    "            node.outbound_nodes.append(self)\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Every node that uses this class as a base class will\n",
    "        need to define its own `forward` method.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Every node that uses this class as a base class will\n",
    "        need to define its own `backward` method.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "\n",
    "class Input(Node):\n",
    "    \"\"\"\n",
    "    A generic input into the network.\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        # The base class constructor has to run to set all\n",
    "        # the properties here.\n",
    "        #\n",
    "        # The most important property on an Input is value.\n",
    "        # self.value is set during `topological_sort` later.\n",
    "        Node.__init__(self)\n",
    "\n",
    "    def forward(self):\n",
    "        # Do nothing because nothing is calculated.\n",
    "        pass\n",
    "\n",
    "    def backward(self):\n",
    "        # An Input node has no inputs so the gradient (derivative)\n",
    "        # is zero.\n",
    "        # The key, `self`, is reference to this object.\n",
    "        self.gradients = {self: 0}\n",
    "        # Weights and bias may be inputs, so you need to sum\n",
    "        # the gradient from output gradients.\n",
    "        for n in self.outbound_nodes:\n",
    "            self.gradients[self] += n.gradients[self]\n",
    "\n",
    "class Linear(Node):\n",
    "    \"\"\"\n",
    "    Represents a node that performs a linear transform.\n",
    "    \"\"\"\n",
    "    def __init__(self, X, W, b):\n",
    "        # The base class (Node) constructor. Weights and bias\n",
    "        # are treated like inbound nodes.\n",
    "        Node.__init__(self, [X, W, b])\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Performs the math behind a linear transform.\n",
    "        \"\"\"\n",
    "        X = self.inbound_nodes[0].value\n",
    "        W = self.inbound_nodes[1].value\n",
    "        b = self.inbound_nodes[2].value\n",
    "        self.value = np.dot(X, W) + b\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Calculates the gradient based on the output values.\n",
    "        \"\"\"\n",
    "        # Initialize a partial for each of the inbound_nodes.\n",
    "        self.gradients = {n: np.zeros_like(n.value) for n in self.inbound_nodes}\n",
    "        # Cycle through the outputs. The gradient will change depending\n",
    "        # on each output, so the gradients are summed over all outputs.\n",
    "        for n in self.outbound_nodes:\n",
    "            # Get the partial of the cost with respect to this node.\n",
    "            grad_cost = n.gradients[self]\n",
    "            # Set the partial of the loss with respect to this node's inputs.\n",
    "            self.gradients[self.inbound_nodes[0]] += np.dot(grad_cost, self.inbound_nodes[1].value.T)\n",
    "            # Set the partial of the loss with respect to this node's weights.\n",
    "            self.gradients[self.inbound_nodes[1]] += np.dot(self.inbound_nodes[0].value.T, grad_cost)\n",
    "            # Set the partial of the loss with respect to this node's bias.\n",
    "            self.gradients[self.inbound_nodes[2]] += np.sum(grad_cost, axis=0, keepdims=False)\n",
    "\n",
    "\n",
    "class Sigmoid(Node):\n",
    "    \"\"\"\n",
    "    Represents a node that performs the sigmoid activation function.\n",
    "    \"\"\"\n",
    "    def __init__(self, node):\n",
    "        # The base class constructor.\n",
    "        Node.__init__(self, [node])\n",
    "\n",
    "    def _sigmoid(self, x):\n",
    "        \"\"\"\n",
    "        This method is separate from `forward` because it\n",
    "        will be used with `backward` as well.\n",
    "\n",
    "        `x`: A numpy array-like object.\n",
    "        \"\"\"\n",
    "        return 1. / (1. + np.exp(-x))\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Perform the sigmoid function and set the value.\n",
    "        \"\"\"\n",
    "        input_value = self.inbound_nodes[0].value\n",
    "        self.value = self._sigmoid(input_value)\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Calculates the gradient using the derivative of\n",
    "        the sigmoid function.\n",
    "        \"\"\"\n",
    "        # Initialize the gradients to 0.\n",
    "        self.gradients = {n: np.zeros_like(n.value) for n in self.inbound_nodes}\n",
    "        # Sum the partial with respect to the input over all the outputs.\n",
    "        for n in self.outbound_nodes:\n",
    "            grad_cost = n.gradients[self]\n",
    "            sigmoid = self.value\n",
    "            self.gradients[self.inbound_nodes[0]] += sigmoid * (1 - sigmoid) * grad_cost\n",
    "\n",
    "\n",
    "class MSE(Node):\n",
    "    def __init__(self, y, a):\n",
    "        \"\"\"\n",
    "        The mean squared error cost function.\n",
    "        Should be used as the last node for a network.\n",
    "        \"\"\"\n",
    "        # Call the base class' constructor.\n",
    "        Node.__init__(self, [y, a])\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Calculates the mean squared error.\n",
    "        \"\"\"\n",
    "        # NOTE: We reshape these to avoid possible matrix/vector broadcast\n",
    "        # errors.\n",
    "        #\n",
    "        # For example, if we subtract an array of shape (3,) from an array of shape\n",
    "        # (3,1) we get an array of shape(3,3) as the result when we want\n",
    "        # an array of shape (3,1) instead.\n",
    "        #\n",
    "        # Making both arrays (3,1) insures the result is (3,1) and does\n",
    "        # an elementwise subtraction as expected.\n",
    "        y = self.inbound_nodes[0].value.reshape(-1, 1)\n",
    "        a = self.inbound_nodes[1].value.reshape(-1, 1)\n",
    "\n",
    "        self.m = self.inbound_nodes[0].value.shape[0]\n",
    "        # Save the computed output for backward.\n",
    "        self.diff = y - a\n",
    "        self.value = np.mean(self.diff**2)\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Calculates the gradient of the cost.\n",
    "        \"\"\"\n",
    "        self.gradients[self.inbound_nodes[0]] = (2 / self.m) * self.diff\n",
    "        self.gradients[self.inbound_nodes[1]] = (-2 / self.m) * self.diff\n",
    "\n",
    "\n",
    "def topological_sort(feed_dict):\n",
    "    \"\"\"\n",
    "    Sort the nodes in topological order using Kahn's Algorithm.\n",
    "\n",
    "    `feed_dict`: A dictionary where the key is a `Input` Node and the value is the respective value feed to that Node.\n",
    "\n",
    "    Returns a list of sorted nodes.\n",
    "    \"\"\"\n",
    "\n",
    "    input_nodes = [n for n in feed_dict.keys()]\n",
    "\n",
    "    G = {}\n",
    "    nodes = [n for n in input_nodes]\n",
    "    while len(nodes) > 0:\n",
    "        n = nodes.pop(0)\n",
    "        if n not in G:\n",
    "            G[n] = {'in': set(), 'out': set()}\n",
    "        for m in n.outbound_nodes:\n",
    "            if m not in G:\n",
    "                G[m] = {'in': set(), 'out': set()}\n",
    "            G[n]['out'].add(m)\n",
    "            G[m]['in'].add(n)\n",
    "            nodes.append(m)\n",
    "\n",
    "    L = []\n",
    "    S = set(input_nodes)\n",
    "    while len(S) > 0:\n",
    "        n = S.pop()\n",
    "\n",
    "        if isinstance(n, Input):\n",
    "            n.value = feed_dict[n]\n",
    "\n",
    "        L.append(n)\n",
    "        for m in n.outbound_nodes:\n",
    "            G[n]['out'].remove(m)\n",
    "            G[m]['in'].remove(n)\n",
    "            # if no other incoming edges add to S\n",
    "            if len(G[m]['in']) == 0:\n",
    "                S.add(m)\n",
    "    return L\n",
    "\n",
    "\n",
    "def forward_and_backward(graph):\n",
    "    \"\"\"\n",
    "    Performs a forward pass and a backward pass through a list of sorted Nodes.\n",
    "\n",
    "    Arguments:\n",
    "\n",
    "        `graph`: The result of calling `topological_sort`.\n",
    "    \"\"\"\n",
    "    # Forward pass\n",
    "    for n in graph:\n",
    "        n.forward()\n",
    "\n",
    "    # Backward pass\n",
    "    # see: https://docs.python.org/2.3/whatsnew/section-slices.html\n",
    "    for n in graph[::-1]:\n",
    "        n.backward()\n",
    "\n",
    "\n",
    "def sgd_update(trainables, learning_rate=1e-2):\n",
    "    \"\"\"\n",
    "    Updates the value of each trainable with SGD.\n",
    "\n",
    "    Arguments:\n",
    "\n",
    "        `trainables`: A list of `Input` Nodes representing weights/biases.\n",
    "        `learning_rate`: The learning rate.\n",
    "    \"\"\"\n",
    "    # TODO: update all the `trainables` with SGD\n",
    "    # You can access and assign the value of a trainable with `value` attribute.\n",
    "    # Example:\n",
    "    # for t in trainables:\n",
    "    #   t.value = your implementation here\n",
    "    \n",
    "    for t in trainables:\n",
    "        t.value -= learning_rate * t.gradients[t] \n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Main function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[url](https://d17h27t6h515a5.cloudfront.net/topher/2016/October/581021c4_12/12.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of examples = 506\n",
      "Epoch: 1, Loss: 57.303\n",
      "Epoch: 2, Loss: 27.540\n",
      "Epoch: 3, Loss: 18.537\n",
      "Epoch: 4, Loss: 15.205\n",
      "Epoch: 5, Loss: 14.008\n",
      "Epoch: 6, Loss: 13.941\n",
      "Epoch: 7, Loss: 10.554\n",
      "Epoch: 8, Loss: 15.031\n",
      "Epoch: 9, Loss: 11.416\n",
      "Epoch: 10, Loss: 10.992\n",
      "Epoch: 11, Loss: 9.227\n",
      "Epoch: 12, Loss: 14.299\n",
      "Epoch: 13, Loss: 8.343\n",
      "Epoch: 14, Loss: 10.570\n",
      "Epoch: 15, Loss: 12.354\n",
      "Epoch: 16, Loss: 11.154\n",
      "Epoch: 17, Loss: 10.775\n",
      "Epoch: 18, Loss: 8.929\n",
      "Epoch: 19, Loss: 7.429\n",
      "Epoch: 20, Loss: 7.868\n",
      "Epoch: 21, Loss: 7.636\n",
      "Epoch: 22, Loss: 8.316\n",
      "Epoch: 23, Loss: 9.113\n",
      "Epoch: 24, Loss: 7.674\n",
      "Epoch: 25, Loss: 8.829\n",
      "Epoch: 26, Loss: 9.391\n",
      "Epoch: 27, Loss: 6.971\n",
      "Epoch: 28, Loss: 9.129\n",
      "Epoch: 29, Loss: 7.090\n",
      "Epoch: 30, Loss: 8.044\n",
      "Epoch: 31, Loss: 8.353\n",
      "Epoch: 32, Loss: 7.048\n",
      "Epoch: 33, Loss: 5.898\n",
      "Epoch: 34, Loss: 7.066\n",
      "Epoch: 35, Loss: 7.196\n",
      "Epoch: 36, Loss: 5.622\n",
      "Epoch: 37, Loss: 9.452\n",
      "Epoch: 38, Loss: 8.574\n",
      "Epoch: 39, Loss: 8.877\n",
      "Epoch: 40, Loss: 7.579\n",
      "Epoch: 41, Loss: 8.198\n",
      "Epoch: 42, Loss: 7.045\n",
      "Epoch: 43, Loss: 6.584\n",
      "Epoch: 44, Loss: 6.701\n",
      "Epoch: 45, Loss: 7.856\n",
      "Epoch: 46, Loss: 6.200\n",
      "Epoch: 47, Loss: 7.521\n",
      "Epoch: 48, Loss: 6.922\n",
      "Epoch: 49, Loss: 7.155\n",
      "Epoch: 50, Loss: 6.945\n",
      "Epoch: 51, Loss: 6.524\n",
      "Epoch: 52, Loss: 6.762\n",
      "Epoch: 53, Loss: 6.618\n",
      "Epoch: 54, Loss: 6.899\n",
      "Epoch: 55, Loss: 5.648\n",
      "Epoch: 56, Loss: 5.438\n",
      "Epoch: 57, Loss: 5.653\n",
      "Epoch: 58, Loss: 4.998\n",
      "Epoch: 59, Loss: 6.277\n",
      "Epoch: 60, Loss: 6.107\n",
      "Epoch: 61, Loss: 5.344\n",
      "Epoch: 62, Loss: 5.173\n",
      "Epoch: 63, Loss: 5.729\n",
      "Epoch: 64, Loss: 6.396\n",
      "Epoch: 65, Loss: 6.597\n",
      "Epoch: 66, Loss: 6.521\n",
      "Epoch: 67, Loss: 5.467\n",
      "Epoch: 68, Loss: 6.478\n",
      "Epoch: 69, Loss: 4.850\n",
      "Epoch: 70, Loss: 5.813\n",
      "Epoch: 71, Loss: 7.057\n",
      "Epoch: 72, Loss: 7.481\n",
      "Epoch: 73, Loss: 5.407\n",
      "Epoch: 74, Loss: 6.070\n",
      "Epoch: 75, Loss: 5.285\n",
      "Epoch: 76, Loss: 5.513\n",
      "Epoch: 77, Loss: 5.710\n",
      "Epoch: 78, Loss: 6.420\n",
      "Epoch: 79, Loss: 5.075\n",
      "Epoch: 80, Loss: 5.609\n",
      "Epoch: 81, Loss: 6.462\n",
      "Epoch: 82, Loss: 6.389\n",
      "Epoch: 83, Loss: 6.565\n",
      "Epoch: 84, Loss: 6.154\n",
      "Epoch: 85, Loss: 6.676\n",
      "Epoch: 86, Loss: 5.797\n",
      "Epoch: 87, Loss: 7.610\n",
      "Epoch: 88, Loss: 6.687\n",
      "Epoch: 89, Loss: 7.239\n",
      "Epoch: 90, Loss: 6.010\n",
      "Epoch: 91, Loss: 6.440\n",
      "Epoch: 92, Loss: 6.280\n",
      "Epoch: 93, Loss: 6.109\n",
      "Epoch: 94, Loss: 5.791\n",
      "Epoch: 95, Loss: 5.180\n",
      "Epoch: 96, Loss: 5.845\n",
      "Epoch: 97, Loss: 6.053\n",
      "Epoch: 98, Loss: 5.415\n",
      "Epoch: 99, Loss: 5.553\n",
      "Epoch: 100, Loss: 7.267\n",
      "Epoch: 101, Loss: 5.444\n",
      "Epoch: 102, Loss: 5.231\n",
      "Epoch: 103, Loss: 5.592\n",
      "Epoch: 104, Loss: 5.303\n",
      "Epoch: 105, Loss: 5.036\n",
      "Epoch: 106, Loss: 4.696\n",
      "Epoch: 107, Loss: 5.585\n",
      "Epoch: 108, Loss: 4.833\n",
      "Epoch: 109, Loss: 5.492\n",
      "Epoch: 110, Loss: 5.528\n",
      "Epoch: 111, Loss: 6.129\n",
      "Epoch: 112, Loss: 4.947\n",
      "Epoch: 113, Loss: 6.466\n",
      "Epoch: 114, Loss: 4.546\n",
      "Epoch: 115, Loss: 5.547\n",
      "Epoch: 116, Loss: 5.431\n",
      "Epoch: 117, Loss: 6.331\n",
      "Epoch: 118, Loss: 5.174\n",
      "Epoch: 119, Loss: 5.391\n",
      "Epoch: 120, Loss: 5.828\n",
      "Epoch: 121, Loss: 5.540\n",
      "Epoch: 122, Loss: 6.020\n",
      "Epoch: 123, Loss: 5.867\n",
      "Epoch: 124, Loss: 5.754\n",
      "Epoch: 125, Loss: 5.454\n",
      "Epoch: 126, Loss: 6.477\n",
      "Epoch: 127, Loss: 5.842\n",
      "Epoch: 128, Loss: 6.167\n",
      "Epoch: 129, Loss: 5.065\n",
      "Epoch: 130, Loss: 5.650\n",
      "Epoch: 131, Loss: 4.848\n",
      "Epoch: 132, Loss: 5.342\n",
      "Epoch: 133, Loss: 5.708\n",
      "Epoch: 134, Loss: 5.430\n",
      "Epoch: 135, Loss: 4.362\n",
      "Epoch: 136, Loss: 5.142\n",
      "Epoch: 137, Loss: 5.780\n",
      "Epoch: 138, Loss: 5.528\n",
      "Epoch: 139, Loss: 6.091\n",
      "Epoch: 140, Loss: 5.254\n",
      "Epoch: 141, Loss: 5.647\n",
      "Epoch: 142, Loss: 5.648\n",
      "Epoch: 143, Loss: 5.180\n",
      "Epoch: 144, Loss: 5.572\n",
      "Epoch: 145, Loss: 5.344\n",
      "Epoch: 146, Loss: 4.499\n",
      "Epoch: 147, Loss: 4.934\n",
      "Epoch: 148, Loss: 5.039\n",
      "Epoch: 149, Loss: 5.164\n",
      "Epoch: 150, Loss: 5.494\n",
      "Epoch: 151, Loss: 5.610\n",
      "Epoch: 152, Loss: 4.381\n",
      "Epoch: 153, Loss: 6.018\n",
      "Epoch: 154, Loss: 5.347\n",
      "Epoch: 155, Loss: 5.097\n",
      "Epoch: 156, Loss: 5.127\n",
      "Epoch: 157, Loss: 5.166\n",
      "Epoch: 158, Loss: 4.727\n",
      "Epoch: 159, Loss: 4.411\n",
      "Epoch: 160, Loss: 6.175\n",
      "Epoch: 161, Loss: 4.536\n",
      "Epoch: 162, Loss: 5.377\n",
      "Epoch: 163, Loss: 5.184\n",
      "Epoch: 164, Loss: 5.433\n",
      "Epoch: 165, Loss: 4.548\n",
      "Epoch: 166, Loss: 5.204\n",
      "Epoch: 167, Loss: 5.655\n",
      "Epoch: 168, Loss: 4.446\n",
      "Epoch: 169, Loss: 4.952\n",
      "Epoch: 170, Loss: 6.334\n",
      "Epoch: 171, Loss: 5.138\n",
      "Epoch: 172, Loss: 4.626\n",
      "Epoch: 173, Loss: 5.389\n",
      "Epoch: 174, Loss: 5.841\n",
      "Epoch: 175, Loss: 5.287\n",
      "Epoch: 176, Loss: 4.955\n",
      "Epoch: 177, Loss: 4.740\n",
      "Epoch: 178, Loss: 4.503\n",
      "Epoch: 179, Loss: 4.546\n",
      "Epoch: 180, Loss: 5.116\n",
      "Epoch: 181, Loss: 4.154\n",
      "Epoch: 182, Loss: 4.831\n",
      "Epoch: 183, Loss: 4.842\n",
      "Epoch: 184, Loss: 5.016\n",
      "Epoch: 185, Loss: 4.722\n",
      "Epoch: 186, Loss: 4.869\n",
      "Epoch: 187, Loss: 6.065\n",
      "Epoch: 188, Loss: 4.369\n",
      "Epoch: 189, Loss: 5.539\n",
      "Epoch: 190, Loss: 4.658\n",
      "Epoch: 191, Loss: 5.157\n",
      "Epoch: 192, Loss: 4.843\n",
      "Epoch: 193, Loss: 4.655\n",
      "Epoch: 194, Loss: 4.640\n",
      "Epoch: 195, Loss: 5.766\n",
      "Epoch: 196, Loss: 5.492\n",
      "Epoch: 197, Loss: 5.432\n",
      "Epoch: 198, Loss: 5.551\n",
      "Epoch: 199, Loss: 4.807\n",
      "Epoch: 200, Loss: 5.410\n",
      "Epoch: 201, Loss: 4.751\n",
      "Epoch: 202, Loss: 5.389\n",
      "Epoch: 203, Loss: 5.391\n",
      "Epoch: 204, Loss: 4.771\n",
      "Epoch: 205, Loss: 4.549\n",
      "Epoch: 206, Loss: 5.040\n",
      "Epoch: 207, Loss: 4.491\n",
      "Epoch: 208, Loss: 5.021\n",
      "Epoch: 209, Loss: 4.835\n",
      "Epoch: 210, Loss: 4.267\n",
      "Epoch: 211, Loss: 4.580\n",
      "Epoch: 212, Loss: 5.828\n",
      "Epoch: 213, Loss: 4.943\n",
      "Epoch: 214, Loss: 5.759\n",
      "Epoch: 215, Loss: 5.296\n",
      "Epoch: 216, Loss: 4.897\n",
      "Epoch: 217, Loss: 4.955\n",
      "Epoch: 218, Loss: 4.726\n",
      "Epoch: 219, Loss: 5.241\n",
      "Epoch: 220, Loss: 4.464\n",
      "Epoch: 221, Loss: 4.968\n",
      "Epoch: 222, Loss: 4.567\n",
      "Epoch: 223, Loss: 6.235\n",
      "Epoch: 224, Loss: 4.297\n",
      "Epoch: 225, Loss: 4.705\n",
      "Epoch: 226, Loss: 5.640\n",
      "Epoch: 227, Loss: 5.006\n",
      "Epoch: 228, Loss: 4.250\n",
      "Epoch: 229, Loss: 4.495\n",
      "Epoch: 230, Loss: 5.488\n",
      "Epoch: 231, Loss: 5.035\n",
      "Epoch: 232, Loss: 4.258\n",
      "Epoch: 233, Loss: 4.858\n",
      "Epoch: 234, Loss: 4.312\n",
      "Epoch: 235, Loss: 5.219\n",
      "Epoch: 236, Loss: 5.186\n",
      "Epoch: 237, Loss: 4.977\n",
      "Epoch: 238, Loss: 5.170\n",
      "Epoch: 239, Loss: 4.793\n",
      "Epoch: 240, Loss: 5.621\n",
      "Epoch: 241, Loss: 5.031\n",
      "Epoch: 242, Loss: 5.016\n",
      "Epoch: 243, Loss: 4.361\n",
      "Epoch: 244, Loss: 4.654\n",
      "Epoch: 245, Loss: 5.276\n",
      "Epoch: 246, Loss: 5.316\n",
      "Epoch: 247, Loss: 4.731\n",
      "Epoch: 248, Loss: 4.658\n",
      "Epoch: 249, Loss: 3.881\n",
      "Epoch: 250, Loss: 5.027\n",
      "Epoch: 251, Loss: 4.156\n",
      "Epoch: 252, Loss: 4.266\n",
      "Epoch: 253, Loss: 4.895\n",
      "Epoch: 254, Loss: 5.268\n",
      "Epoch: 255, Loss: 5.303\n",
      "Epoch: 256, Loss: 5.423\n",
      "Epoch: 257, Loss: 4.869\n",
      "Epoch: 258, Loss: 4.300\n",
      "Epoch: 259, Loss: 5.928\n",
      "Epoch: 260, Loss: 4.759\n",
      "Epoch: 261, Loss: 4.635\n",
      "Epoch: 262, Loss: 4.310\n",
      "Epoch: 263, Loss: 4.621\n",
      "Epoch: 264, Loss: 4.977\n",
      "Epoch: 265, Loss: 5.166\n",
      "Epoch: 266, Loss: 4.880\n",
      "Epoch: 267, Loss: 5.277\n",
      "Epoch: 268, Loss: 4.535\n",
      "Epoch: 269, Loss: 4.302\n",
      "Epoch: 270, Loss: 4.508\n",
      "Epoch: 271, Loss: 4.578\n",
      "Epoch: 272, Loss: 5.039\n",
      "Epoch: 273, Loss: 4.224\n",
      "Epoch: 274, Loss: 4.819\n",
      "Epoch: 275, Loss: 4.903\n",
      "Epoch: 276, Loss: 5.799\n",
      "Epoch: 277, Loss: 4.871\n",
      "Epoch: 278, Loss: 4.681\n",
      "Epoch: 279, Loss: 4.587\n",
      "Epoch: 280, Loss: 5.048\n",
      "Epoch: 281, Loss: 4.826\n",
      "Epoch: 282, Loss: 5.200\n",
      "Epoch: 283, Loss: 4.019\n",
      "Epoch: 284, Loss: 4.252\n",
      "Epoch: 285, Loss: 4.870\n",
      "Epoch: 286, Loss: 4.679\n",
      "Epoch: 287, Loss: 4.890\n",
      "Epoch: 288, Loss: 4.238\n",
      "Epoch: 289, Loss: 4.262\n",
      "Epoch: 290, Loss: 4.522\n",
      "Epoch: 291, Loss: 4.921\n",
      "Epoch: 292, Loss: 4.580\n",
      "Epoch: 293, Loss: 5.605\n",
      "Epoch: 294, Loss: 4.168\n",
      "Epoch: 295, Loss: 5.030\n",
      "Epoch: 296, Loss: 4.695\n",
      "Epoch: 297, Loss: 4.512\n",
      "Epoch: 298, Loss: 4.729\n",
      "Epoch: 299, Loss: 4.963\n",
      "Epoch: 300, Loss: 4.829\n",
      "Epoch: 301, Loss: 5.445\n",
      "Epoch: 302, Loss: 4.959\n",
      "Epoch: 303, Loss: 5.194\n",
      "Epoch: 304, Loss: 5.175\n",
      "Epoch: 305, Loss: 4.647\n",
      "Epoch: 306, Loss: 4.803\n",
      "Epoch: 307, Loss: 3.994\n",
      "Epoch: 308, Loss: 4.585\n",
      "Epoch: 309, Loss: 5.106\n",
      "Epoch: 310, Loss: 4.868\n",
      "Epoch: 311, Loss: 4.841\n",
      "Epoch: 312, Loss: 4.245\n",
      "Epoch: 313, Loss: 4.927\n",
      "Epoch: 314, Loss: 4.381\n",
      "Epoch: 315, Loss: 4.565\n",
      "Epoch: 316, Loss: 4.096\n",
      "Epoch: 317, Loss: 4.643\n",
      "Epoch: 318, Loss: 5.082\n",
      "Epoch: 319, Loss: 5.225\n",
      "Epoch: 320, Loss: 4.481\n",
      "Epoch: 321, Loss: 4.708\n",
      "Epoch: 322, Loss: 3.682\n",
      "Epoch: 323, Loss: 4.373\n",
      "Epoch: 324, Loss: 4.865\n",
      "Epoch: 325, Loss: 4.557\n",
      "Epoch: 326, Loss: 4.421\n",
      "Epoch: 327, Loss: 4.354\n",
      "Epoch: 328, Loss: 4.636\n",
      "Epoch: 329, Loss: 4.466\n",
      "Epoch: 330, Loss: 4.462\n",
      "Epoch: 331, Loss: 4.760\n",
      "Epoch: 332, Loss: 3.718\n",
      "Epoch: 333, Loss: 4.996\n",
      "Epoch: 334, Loss: 4.944\n",
      "Epoch: 335, Loss: 3.921\n",
      "Epoch: 336, Loss: 4.691\n",
      "Epoch: 337, Loss: 4.236\n",
      "Epoch: 338, Loss: 4.257\n",
      "Epoch: 339, Loss: 4.535\n",
      "Epoch: 340, Loss: 4.470\n",
      "Epoch: 341, Loss: 5.095\n",
      "Epoch: 342, Loss: 4.380\n",
      "Epoch: 343, Loss: 5.129\n",
      "Epoch: 344, Loss: 5.128\n",
      "Epoch: 345, Loss: 4.180\n",
      "Epoch: 346, Loss: 4.885\n",
      "Epoch: 347, Loss: 4.953\n",
      "Epoch: 348, Loss: 4.390\n",
      "Epoch: 349, Loss: 4.606\n",
      "Epoch: 350, Loss: 4.335\n",
      "Epoch: 351, Loss: 4.872\n",
      "Epoch: 352, Loss: 4.476\n",
      "Epoch: 353, Loss: 4.771\n",
      "Epoch: 354, Loss: 5.244\n",
      "Epoch: 355, Loss: 4.900\n",
      "Epoch: 356, Loss: 4.646\n",
      "Epoch: 357, Loss: 4.938\n",
      "Epoch: 358, Loss: 5.108\n",
      "Epoch: 359, Loss: 4.802\n",
      "Epoch: 360, Loss: 4.114\n",
      "Epoch: 361, Loss: 4.832\n",
      "Epoch: 362, Loss: 4.471\n",
      "Epoch: 363, Loss: 4.605\n",
      "Epoch: 364, Loss: 4.659\n",
      "Epoch: 365, Loss: 4.952\n",
      "Epoch: 366, Loss: 4.401\n",
      "Epoch: 367, Loss: 4.753\n",
      "Epoch: 368, Loss: 4.383\n",
      "Epoch: 369, Loss: 4.606\n",
      "Epoch: 370, Loss: 5.182\n",
      "Epoch: 371, Loss: 4.442\n",
      "Epoch: 372, Loss: 4.663\n",
      "Epoch: 373, Loss: 4.730\n",
      "Epoch: 374, Loss: 3.970\n",
      "Epoch: 375, Loss: 4.227\n",
      "Epoch: 376, Loss: 3.581\n",
      "Epoch: 377, Loss: 4.627\n",
      "Epoch: 378, Loss: 4.755\n",
      "Epoch: 379, Loss: 4.200\n",
      "Epoch: 380, Loss: 3.587\n",
      "Epoch: 381, Loss: 3.761\n",
      "Epoch: 382, Loss: 4.310\n",
      "Epoch: 383, Loss: 4.520\n",
      "Epoch: 384, Loss: 5.272\n",
      "Epoch: 385, Loss: 4.889\n",
      "Epoch: 386, Loss: 4.394\n",
      "Epoch: 387, Loss: 4.099\n",
      "Epoch: 388, Loss: 4.764\n",
      "Epoch: 389, Loss: 3.550\n",
      "Epoch: 390, Loss: 4.282\n",
      "Epoch: 391, Loss: 4.473\n",
      "Epoch: 392, Loss: 4.123\n",
      "Epoch: 393, Loss: 4.549\n",
      "Epoch: 394, Loss: 3.992\n",
      "Epoch: 395, Loss: 4.784\n",
      "Epoch: 396, Loss: 4.659\n",
      "Epoch: 397, Loss: 4.308\n",
      "Epoch: 398, Loss: 4.547\n",
      "Epoch: 399, Loss: 4.560\n",
      "Epoch: 400, Loss: 4.748\n",
      "Epoch: 401, Loss: 4.204\n",
      "Epoch: 402, Loss: 4.751\n",
      "Epoch: 403, Loss: 4.580\n",
      "Epoch: 404, Loss: 4.689\n",
      "Epoch: 405, Loss: 4.837\n",
      "Epoch: 406, Loss: 4.104\n",
      "Epoch: 407, Loss: 4.385\n",
      "Epoch: 408, Loss: 5.154\n",
      "Epoch: 409, Loss: 4.401\n",
      "Epoch: 410, Loss: 4.993\n",
      "Epoch: 411, Loss: 3.737\n",
      "Epoch: 412, Loss: 3.654\n",
      "Epoch: 413, Loss: 4.205\n",
      "Epoch: 414, Loss: 4.761\n",
      "Epoch: 415, Loss: 3.900\n",
      "Epoch: 416, Loss: 4.123\n",
      "Epoch: 417, Loss: 4.280\n",
      "Epoch: 418, Loss: 4.234\n",
      "Epoch: 419, Loss: 3.885\n",
      "Epoch: 420, Loss: 4.211\n",
      "Epoch: 421, Loss: 4.470\n",
      "Epoch: 422, Loss: 5.183\n",
      "Epoch: 423, Loss: 4.533\n",
      "Epoch: 424, Loss: 5.061\n",
      "Epoch: 425, Loss: 4.519\n",
      "Epoch: 426, Loss: 4.542\n",
      "Epoch: 427, Loss: 4.796\n",
      "Epoch: 428, Loss: 4.284\n",
      "Epoch: 429, Loss: 4.529\n",
      "Epoch: 430, Loss: 4.625\n",
      "Epoch: 431, Loss: 4.775\n",
      "Epoch: 432, Loss: 4.363\n",
      "Epoch: 433, Loss: 4.560\n",
      "Epoch: 434, Loss: 4.272\n",
      "Epoch: 435, Loss: 3.900\n",
      "Epoch: 436, Loss: 4.179\n",
      "Epoch: 437, Loss: 4.775\n",
      "Epoch: 438, Loss: 5.320\n",
      "Epoch: 439, Loss: 4.702\n",
      "Epoch: 440, Loss: 4.931\n",
      "Epoch: 441, Loss: 3.662\n",
      "Epoch: 442, Loss: 4.058\n",
      "Epoch: 443, Loss: 4.259\n",
      "Epoch: 444, Loss: 4.322\n",
      "Epoch: 445, Loss: 4.572\n",
      "Epoch: 446, Loss: 4.559\n",
      "Epoch: 447, Loss: 4.232\n",
      "Epoch: 448, Loss: 5.282\n",
      "Epoch: 449, Loss: 3.784\n",
      "Epoch: 450, Loss: 4.970\n",
      "Epoch: 451, Loss: 4.341\n",
      "Epoch: 452, Loss: 4.382\n",
      "Epoch: 453, Loss: 3.928\n",
      "Epoch: 454, Loss: 4.836\n",
      "Epoch: 455, Loss: 4.527\n",
      "Epoch: 456, Loss: 4.553\n",
      "Epoch: 457, Loss: 4.594\n",
      "Epoch: 458, Loss: 3.864\n",
      "Epoch: 459, Loss: 3.718\n",
      "Epoch: 460, Loss: 4.954\n",
      "Epoch: 461, Loss: 4.505\n",
      "Epoch: 462, Loss: 4.322\n",
      "Epoch: 463, Loss: 4.712\n",
      "Epoch: 464, Loss: 4.313\n",
      "Epoch: 465, Loss: 4.879\n",
      "Epoch: 466, Loss: 4.308\n",
      "Epoch: 467, Loss: 3.939\n",
      "Epoch: 468, Loss: 4.494\n",
      "Epoch: 469, Loss: 4.945\n",
      "Epoch: 470, Loss: 4.000\n",
      "Epoch: 471, Loss: 4.100\n",
      "Epoch: 472, Loss: 4.153\n",
      "Epoch: 473, Loss: 4.663\n",
      "Epoch: 474, Loss: 3.734\n",
      "Epoch: 475, Loss: 4.157\n",
      "Epoch: 476, Loss: 4.418\n",
      "Epoch: 477, Loss: 4.285\n",
      "Epoch: 478, Loss: 5.014\n",
      "Epoch: 479, Loss: 3.838\n",
      "Epoch: 480, Loss: 4.731\n",
      "Epoch: 481, Loss: 3.960\n",
      "Epoch: 482, Loss: 4.878\n",
      "Epoch: 483, Loss: 4.806\n",
      "Epoch: 484, Loss: 4.467\n",
      "Epoch: 485, Loss: 4.963\n",
      "Epoch: 486, Loss: 5.223\n",
      "Epoch: 487, Loss: 4.841\n",
      "Epoch: 488, Loss: 4.600\n",
      "Epoch: 489, Loss: 4.053\n",
      "Epoch: 490, Loss: 4.553\n",
      "Epoch: 491, Loss: 4.064\n",
      "Epoch: 492, Loss: 4.315\n",
      "Epoch: 493, Loss: 4.600\n",
      "Epoch: 494, Loss: 3.587\n",
      "Epoch: 495, Loss: 4.424\n",
      "Epoch: 496, Loss: 4.584\n",
      "Epoch: 497, Loss: 4.066\n",
      "Epoch: 498, Loss: 4.174\n",
      "Epoch: 499, Loss: 3.910\n",
      "Epoch: 500, Loss: 4.083\n",
      "Epoch: 501, Loss: 4.108\n",
      "Epoch: 502, Loss: 5.787\n",
      "Epoch: 503, Loss: 4.383\n",
      "Epoch: 504, Loss: 4.139\n",
      "Epoch: 505, Loss: 4.362\n",
      "Epoch: 506, Loss: 4.370\n",
      "Epoch: 507, Loss: 4.117\n",
      "Epoch: 508, Loss: 5.335\n",
      "Epoch: 509, Loss: 4.703\n",
      "Epoch: 510, Loss: 4.757\n",
      "Epoch: 511, Loss: 4.990\n",
      "Epoch: 512, Loss: 4.055\n",
      "Epoch: 513, Loss: 4.231\n",
      "Epoch: 514, Loss: 4.172\n",
      "Epoch: 515, Loss: 4.855\n",
      "Epoch: 516, Loss: 4.115\n",
      "Epoch: 517, Loss: 4.420\n",
      "Epoch: 518, Loss: 4.275\n",
      "Epoch: 519, Loss: 4.200\n",
      "Epoch: 520, Loss: 4.092\n",
      "Epoch: 521, Loss: 4.310\n",
      "Epoch: 522, Loss: 4.425\n",
      "Epoch: 523, Loss: 4.231\n",
      "Epoch: 524, Loss: 3.857\n",
      "Epoch: 525, Loss: 4.672\n",
      "Epoch: 526, Loss: 4.751\n",
      "Epoch: 527, Loss: 4.384\n",
      "Epoch: 528, Loss: 3.695\n",
      "Epoch: 529, Loss: 4.372\n",
      "Epoch: 530, Loss: 3.743\n",
      "Epoch: 531, Loss: 4.051\n",
      "Epoch: 532, Loss: 4.825\n",
      "Epoch: 533, Loss: 3.877\n",
      "Epoch: 534, Loss: 4.527\n",
      "Epoch: 535, Loss: 4.566\n",
      "Epoch: 536, Loss: 3.825\n",
      "Epoch: 537, Loss: 5.029\n",
      "Epoch: 538, Loss: 4.223\n",
      "Epoch: 539, Loss: 4.267\n",
      "Epoch: 540, Loss: 4.537\n",
      "Epoch: 541, Loss: 3.857\n",
      "Epoch: 542, Loss: 4.413\n",
      "Epoch: 543, Loss: 4.760\n",
      "Epoch: 544, Loss: 4.182\n",
      "Epoch: 545, Loss: 4.606\n",
      "Epoch: 546, Loss: 4.502\n",
      "Epoch: 547, Loss: 4.393\n",
      "Epoch: 548, Loss: 4.499\n",
      "Epoch: 549, Loss: 3.930\n",
      "Epoch: 550, Loss: 4.316\n",
      "Epoch: 551, Loss: 4.584\n",
      "Epoch: 552, Loss: 4.346\n",
      "Epoch: 553, Loss: 4.374\n",
      "Epoch: 554, Loss: 4.651\n",
      "Epoch: 555, Loss: 5.120\n",
      "Epoch: 556, Loss: 4.665\n",
      "Epoch: 557, Loss: 4.419\n",
      "Epoch: 558, Loss: 5.353\n",
      "Epoch: 559, Loss: 4.054\n",
      "Epoch: 560, Loss: 4.811\n",
      "Epoch: 561, Loss: 4.864\n",
      "Epoch: 562, Loss: 4.415\n",
      "Epoch: 563, Loss: 4.598\n",
      "Epoch: 564, Loss: 4.825\n",
      "Epoch: 565, Loss: 4.288\n",
      "Epoch: 566, Loss: 4.191\n",
      "Epoch: 567, Loss: 4.102\n",
      "Epoch: 568, Loss: 4.532\n",
      "Epoch: 569, Loss: 5.002\n",
      "Epoch: 570, Loss: 3.624\n",
      "Epoch: 571, Loss: 4.305\n",
      "Epoch: 572, Loss: 5.068\n",
      "Epoch: 573, Loss: 4.065\n",
      "Epoch: 574, Loss: 4.984\n",
      "Epoch: 575, Loss: 4.317\n",
      "Epoch: 576, Loss: 4.647\n",
      "Epoch: 577, Loss: 4.456\n",
      "Epoch: 578, Loss: 4.277\n",
      "Epoch: 579, Loss: 4.569\n",
      "Epoch: 580, Loss: 4.527\n",
      "Epoch: 581, Loss: 3.864\n",
      "Epoch: 582, Loss: 4.584\n",
      "Epoch: 583, Loss: 5.128\n",
      "Epoch: 584, Loss: 4.493\n",
      "Epoch: 585, Loss: 4.325\n",
      "Epoch: 586, Loss: 4.316\n",
      "Epoch: 587, Loss: 4.428\n",
      "Epoch: 588, Loss: 3.991\n",
      "Epoch: 589, Loss: 4.382\n",
      "Epoch: 590, Loss: 3.812\n",
      "Epoch: 591, Loss: 3.978\n",
      "Epoch: 592, Loss: 4.577\n",
      "Epoch: 593, Loss: 4.289\n",
      "Epoch: 594, Loss: 4.029\n",
      "Epoch: 595, Loss: 3.919\n",
      "Epoch: 596, Loss: 4.391\n",
      "Epoch: 597, Loss: 4.460\n",
      "Epoch: 598, Loss: 4.598\n",
      "Epoch: 599, Loss: 4.061\n",
      "Epoch: 600, Loss: 3.769\n",
      "Epoch: 601, Loss: 4.568\n",
      "Epoch: 602, Loss: 3.896\n",
      "Epoch: 603, Loss: 3.955\n",
      "Epoch: 604, Loss: 4.403\n",
      "Epoch: 605, Loss: 4.319\n",
      "Epoch: 606, Loss: 3.972\n",
      "Epoch: 607, Loss: 3.679\n",
      "Epoch: 608, Loss: 4.286\n",
      "Epoch: 609, Loss: 4.031\n",
      "Epoch: 610, Loss: 4.468\n",
      "Epoch: 611, Loss: 3.828\n",
      "Epoch: 612, Loss: 4.448\n",
      "Epoch: 613, Loss: 4.272\n",
      "Epoch: 614, Loss: 3.921\n",
      "Epoch: 615, Loss: 4.576\n",
      "Epoch: 616, Loss: 3.742\n",
      "Epoch: 617, Loss: 4.052\n",
      "Epoch: 618, Loss: 4.198\n",
      "Epoch: 619, Loss: 4.070\n",
      "Epoch: 620, Loss: 4.537\n",
      "Epoch: 621, Loss: 4.484\n",
      "Epoch: 622, Loss: 4.423\n",
      "Epoch: 623, Loss: 4.653\n",
      "Epoch: 624, Loss: 3.897\n",
      "Epoch: 625, Loss: 4.550\n",
      "Epoch: 626, Loss: 4.072\n",
      "Epoch: 627, Loss: 3.775\n",
      "Epoch: 628, Loss: 4.039\n",
      "Epoch: 629, Loss: 4.235\n",
      "Epoch: 630, Loss: 4.329\n",
      "Epoch: 631, Loss: 3.915\n",
      "Epoch: 632, Loss: 4.469\n",
      "Epoch: 633, Loss: 3.466\n",
      "Epoch: 634, Loss: 4.054\n",
      "Epoch: 635, Loss: 3.782\n",
      "Epoch: 636, Loss: 3.938\n",
      "Epoch: 637, Loss: 3.894\n",
      "Epoch: 638, Loss: 4.261\n",
      "Epoch: 639, Loss: 4.531\n",
      "Epoch: 640, Loss: 5.086\n",
      "Epoch: 641, Loss: 4.450\n",
      "Epoch: 642, Loss: 3.889\n",
      "Epoch: 643, Loss: 3.846\n",
      "Epoch: 644, Loss: 4.334\n",
      "Epoch: 645, Loss: 4.067\n",
      "Epoch: 646, Loss: 4.592\n",
      "Epoch: 647, Loss: 4.052\n",
      "Epoch: 648, Loss: 4.773\n",
      "Epoch: 649, Loss: 4.377\n",
      "Epoch: 650, Loss: 3.991\n",
      "Epoch: 651, Loss: 4.304\n",
      "Epoch: 652, Loss: 4.461\n",
      "Epoch: 653, Loss: 4.096\n",
      "Epoch: 654, Loss: 4.750\n",
      "Epoch: 655, Loss: 4.332\n",
      "Epoch: 656, Loss: 3.774\n",
      "Epoch: 657, Loss: 4.040\n",
      "Epoch: 658, Loss: 4.138\n",
      "Epoch: 659, Loss: 3.765\n",
      "Epoch: 660, Loss: 4.474\n",
      "Epoch: 661, Loss: 4.287\n",
      "Epoch: 662, Loss: 4.058\n",
      "Epoch: 663, Loss: 5.167\n",
      "Epoch: 664, Loss: 4.290\n",
      "Epoch: 665, Loss: 4.818\n",
      "Epoch: 666, Loss: 3.776\n",
      "Epoch: 667, Loss: 4.398\n",
      "Epoch: 668, Loss: 4.288\n",
      "Epoch: 669, Loss: 3.970\n",
      "Epoch: 670, Loss: 4.197\n",
      "Epoch: 671, Loss: 4.005\n",
      "Epoch: 672, Loss: 4.137\n",
      "Epoch: 673, Loss: 4.531\n",
      "Epoch: 674, Loss: 3.774\n",
      "Epoch: 675, Loss: 3.766\n",
      "Epoch: 676, Loss: 4.137\n",
      "Epoch: 677, Loss: 3.694\n",
      "Epoch: 678, Loss: 4.213\n",
      "Epoch: 679, Loss: 4.367\n",
      "Epoch: 680, Loss: 4.239\n",
      "Epoch: 681, Loss: 4.232\n",
      "Epoch: 682, Loss: 4.772\n",
      "Epoch: 683, Loss: 4.340\n",
      "Epoch: 684, Loss: 4.240\n",
      "Epoch: 685, Loss: 4.761\n",
      "Epoch: 686, Loss: 4.074\n",
      "Epoch: 687, Loss: 3.935\n",
      "Epoch: 688, Loss: 4.188\n",
      "Epoch: 689, Loss: 4.622\n",
      "Epoch: 690, Loss: 3.873\n",
      "Epoch: 691, Loss: 4.031\n",
      "Epoch: 692, Loss: 4.102\n",
      "Epoch: 693, Loss: 3.873\n",
      "Epoch: 694, Loss: 4.162\n",
      "Epoch: 695, Loss: 4.977\n",
      "Epoch: 696, Loss: 3.821\n",
      "Epoch: 697, Loss: 4.309\n",
      "Epoch: 698, Loss: 4.322\n",
      "Epoch: 699, Loss: 4.889\n",
      "Epoch: 700, Loss: 3.534\n",
      "Epoch: 701, Loss: 3.904\n",
      "Epoch: 702, Loss: 4.330\n",
      "Epoch: 703, Loss: 4.307\n",
      "Epoch: 704, Loss: 4.923\n",
      "Epoch: 705, Loss: 4.259\n",
      "Epoch: 706, Loss: 4.007\n",
      "Epoch: 707, Loss: 4.255\n",
      "Epoch: 708, Loss: 4.146\n",
      "Epoch: 709, Loss: 3.919\n",
      "Epoch: 710, Loss: 4.412\n",
      "Epoch: 711, Loss: 4.248\n",
      "Epoch: 712, Loss: 4.378\n",
      "Epoch: 713, Loss: 4.374\n",
      "Epoch: 714, Loss: 4.088\n",
      "Epoch: 715, Loss: 3.726\n",
      "Epoch: 716, Loss: 4.259\n",
      "Epoch: 717, Loss: 4.085\n",
      "Epoch: 718, Loss: 3.325\n",
      "Epoch: 719, Loss: 3.764\n",
      "Epoch: 720, Loss: 4.261\n",
      "Epoch: 721, Loss: 4.983\n",
      "Epoch: 722, Loss: 3.988\n",
      "Epoch: 723, Loss: 4.077\n",
      "Epoch: 724, Loss: 4.396\n",
      "Epoch: 725, Loss: 4.041\n",
      "Epoch: 726, Loss: 4.001\n",
      "Epoch: 727, Loss: 4.197\n",
      "Epoch: 728, Loss: 4.432\n",
      "Epoch: 729, Loss: 4.327\n",
      "Epoch: 730, Loss: 4.426\n",
      "Epoch: 731, Loss: 3.519\n",
      "Epoch: 732, Loss: 4.789\n",
      "Epoch: 733, Loss: 4.087\n",
      "Epoch: 734, Loss: 3.696\n",
      "Epoch: 735, Loss: 4.300\n",
      "Epoch: 736, Loss: 4.248\n",
      "Epoch: 737, Loss: 4.390\n",
      "Epoch: 738, Loss: 3.956\n",
      "Epoch: 739, Loss: 4.202\n",
      "Epoch: 740, Loss: 4.300\n",
      "Epoch: 741, Loss: 4.224\n",
      "Epoch: 742, Loss: 4.345\n",
      "Epoch: 743, Loss: 3.928\n",
      "Epoch: 744, Loss: 3.938\n",
      "Epoch: 745, Loss: 5.114\n",
      "Epoch: 746, Loss: 4.406\n",
      "Epoch: 747, Loss: 4.075\n",
      "Epoch: 748, Loss: 4.001\n",
      "Epoch: 749, Loss: 3.798\n",
      "Epoch: 750, Loss: 4.617\n",
      "Epoch: 751, Loss: 4.133\n",
      "Epoch: 752, Loss: 3.605\n",
      "Epoch: 753, Loss: 4.646\n",
      "Epoch: 754, Loss: 4.585\n",
      "Epoch: 755, Loss: 3.743\n",
      "Epoch: 756, Loss: 3.987\n",
      "Epoch: 757, Loss: 3.860\n",
      "Epoch: 758, Loss: 4.642\n",
      "Epoch: 759, Loss: 4.464\n",
      "Epoch: 760, Loss: 3.860\n",
      "Epoch: 761, Loss: 3.918\n",
      "Epoch: 762, Loss: 4.228\n",
      "Epoch: 763, Loss: 3.893\n",
      "Epoch: 764, Loss: 4.138\n",
      "Epoch: 765, Loss: 3.419\n",
      "Epoch: 766, Loss: 4.913\n",
      "Epoch: 767, Loss: 5.052\n",
      "Epoch: 768, Loss: 4.935\n",
      "Epoch: 769, Loss: 3.574\n",
      "Epoch: 770, Loss: 4.049\n",
      "Epoch: 771, Loss: 4.025\n",
      "Epoch: 772, Loss: 4.267\n",
      "Epoch: 773, Loss: 4.418\n",
      "Epoch: 774, Loss: 4.426\n",
      "Epoch: 775, Loss: 4.107\n",
      "Epoch: 776, Loss: 4.949\n",
      "Epoch: 777, Loss: 4.281\n",
      "Epoch: 778, Loss: 4.520\n",
      "Epoch: 779, Loss: 4.694\n",
      "Epoch: 780, Loss: 4.277\n",
      "Epoch: 781, Loss: 4.294\n",
      "Epoch: 782, Loss: 4.006\n",
      "Epoch: 783, Loss: 3.662\n",
      "Epoch: 784, Loss: 3.933\n",
      "Epoch: 785, Loss: 4.339\n",
      "Epoch: 786, Loss: 4.039\n",
      "Epoch: 787, Loss: 3.730\n",
      "Epoch: 788, Loss: 3.795\n",
      "Epoch: 789, Loss: 4.099\n",
      "Epoch: 790, Loss: 3.739\n",
      "Epoch: 791, Loss: 4.260\n",
      "Epoch: 792, Loss: 4.360\n",
      "Epoch: 793, Loss: 4.074\n",
      "Epoch: 794, Loss: 4.311\n",
      "Epoch: 795, Loss: 4.552\n",
      "Epoch: 796, Loss: 4.888\n",
      "Epoch: 797, Loss: 3.519\n",
      "Epoch: 798, Loss: 4.090\n",
      "Epoch: 799, Loss: 3.690\n",
      "Epoch: 800, Loss: 3.684\n",
      "Epoch: 801, Loss: 3.471\n",
      "Epoch: 802, Loss: 4.787\n",
      "Epoch: 803, Loss: 4.294\n",
      "Epoch: 804, Loss: 3.838\n",
      "Epoch: 805, Loss: 3.835\n",
      "Epoch: 806, Loss: 3.769\n",
      "Epoch: 807, Loss: 4.215\n",
      "Epoch: 808, Loss: 3.808\n",
      "Epoch: 809, Loss: 3.459\n",
      "Epoch: 810, Loss: 4.458\n",
      "Epoch: 811, Loss: 4.496\n",
      "Epoch: 812, Loss: 4.380\n",
      "Epoch: 813, Loss: 4.446\n",
      "Epoch: 814, Loss: 4.356\n",
      "Epoch: 815, Loss: 4.653\n",
      "Epoch: 816, Loss: 5.123\n",
      "Epoch: 817, Loss: 4.312\n",
      "Epoch: 818, Loss: 4.497\n",
      "Epoch: 819, Loss: 3.685\n",
      "Epoch: 820, Loss: 3.391\n",
      "Epoch: 821, Loss: 4.534\n",
      "Epoch: 822, Loss: 4.027\n",
      "Epoch: 823, Loss: 4.052\n",
      "Epoch: 824, Loss: 3.740\n",
      "Epoch: 825, Loss: 4.015\n",
      "Epoch: 826, Loss: 5.106\n",
      "Epoch: 827, Loss: 3.997\n",
      "Epoch: 828, Loss: 3.980\n",
      "Epoch: 829, Loss: 4.190\n",
      "Epoch: 830, Loss: 4.210\n",
      "Epoch: 831, Loss: 3.358\n",
      "Epoch: 832, Loss: 3.805\n",
      "Epoch: 833, Loss: 3.810\n",
      "Epoch: 834, Loss: 4.473\n",
      "Epoch: 835, Loss: 3.999\n",
      "Epoch: 836, Loss: 3.697\n",
      "Epoch: 837, Loss: 3.617\n",
      "Epoch: 838, Loss: 3.571\n",
      "Epoch: 839, Loss: 3.523\n",
      "Epoch: 840, Loss: 3.758\n",
      "Epoch: 841, Loss: 3.827\n",
      "Epoch: 842, Loss: 4.404\n",
      "Epoch: 843, Loss: 4.331\n",
      "Epoch: 844, Loss: 4.129\n",
      "Epoch: 845, Loss: 4.258\n",
      "Epoch: 846, Loss: 4.363\n",
      "Epoch: 847, Loss: 3.633\n",
      "Epoch: 848, Loss: 3.732\n",
      "Epoch: 849, Loss: 4.042\n",
      "Epoch: 850, Loss: 4.761\n",
      "Epoch: 851, Loss: 4.782\n",
      "Epoch: 852, Loss: 4.046\n",
      "Epoch: 853, Loss: 4.912\n",
      "Epoch: 854, Loss: 4.281\n",
      "Epoch: 855, Loss: 3.852\n",
      "Epoch: 856, Loss: 4.352\n",
      "Epoch: 857, Loss: 4.203\n",
      "Epoch: 858, Loss: 4.002\n",
      "Epoch: 859, Loss: 4.303\n",
      "Epoch: 860, Loss: 4.019\n",
      "Epoch: 861, Loss: 4.060\n",
      "Epoch: 862, Loss: 3.997\n",
      "Epoch: 863, Loss: 3.818\n",
      "Epoch: 864, Loss: 4.329\n",
      "Epoch: 865, Loss: 3.767\n",
      "Epoch: 866, Loss: 3.721\n",
      "Epoch: 867, Loss: 4.053\n",
      "Epoch: 868, Loss: 3.872\n",
      "Epoch: 869, Loss: 4.286\n",
      "Epoch: 870, Loss: 3.883\n",
      "Epoch: 871, Loss: 4.065\n",
      "Epoch: 872, Loss: 3.931\n",
      "Epoch: 873, Loss: 4.042\n",
      "Epoch: 874, Loss: 4.631\n",
      "Epoch: 875, Loss: 4.696\n",
      "Epoch: 876, Loss: 4.949\n",
      "Epoch: 877, Loss: 3.706\n",
      "Epoch: 878, Loss: 4.694\n",
      "Epoch: 879, Loss: 4.325\n",
      "Epoch: 880, Loss: 3.784\n",
      "Epoch: 881, Loss: 4.451\n",
      "Epoch: 882, Loss: 4.342\n",
      "Epoch: 883, Loss: 4.146\n",
      "Epoch: 884, Loss: 3.980\n",
      "Epoch: 885, Loss: 4.113\n",
      "Epoch: 886, Loss: 3.859\n",
      "Epoch: 887, Loss: 4.082\n",
      "Epoch: 888, Loss: 4.838\n",
      "Epoch: 889, Loss: 3.732\n",
      "Epoch: 890, Loss: 4.211\n",
      "Epoch: 891, Loss: 4.393\n",
      "Epoch: 892, Loss: 4.228\n",
      "Epoch: 893, Loss: 4.260\n",
      "Epoch: 894, Loss: 4.505\n",
      "Epoch: 895, Loss: 4.444\n",
      "Epoch: 896, Loss: 3.820\n",
      "Epoch: 897, Loss: 3.600\n",
      "Epoch: 898, Loss: 4.532\n",
      "Epoch: 899, Loss: 4.523\n",
      "Epoch: 900, Loss: 4.004\n",
      "Epoch: 901, Loss: 4.578\n",
      "Epoch: 902, Loss: 3.724\n",
      "Epoch: 903, Loss: 3.948\n",
      "Epoch: 904, Loss: 4.265\n",
      "Epoch: 905, Loss: 4.237\n",
      "Epoch: 906, Loss: 4.562\n",
      "Epoch: 907, Loss: 4.174\n",
      "Epoch: 908, Loss: 3.831\n",
      "Epoch: 909, Loss: 3.545\n",
      "Epoch: 910, Loss: 4.322\n",
      "Epoch: 911, Loss: 4.158\n",
      "Epoch: 912, Loss: 3.905\n",
      "Epoch: 913, Loss: 3.691\n",
      "Epoch: 914, Loss: 3.908\n",
      "Epoch: 915, Loss: 3.934\n",
      "Epoch: 916, Loss: 4.286\n",
      "Epoch: 917, Loss: 3.969\n",
      "Epoch: 918, Loss: 4.198\n",
      "Epoch: 919, Loss: 4.070\n",
      "Epoch: 920, Loss: 3.448\n",
      "Epoch: 921, Loss: 4.137\n",
      "Epoch: 922, Loss: 3.705\n",
      "Epoch: 923, Loss: 4.515\n",
      "Epoch: 924, Loss: 4.636\n",
      "Epoch: 925, Loss: 4.108\n",
      "Epoch: 926, Loss: 3.765\n",
      "Epoch: 927, Loss: 4.514\n",
      "Epoch: 928, Loss: 3.949\n",
      "Epoch: 929, Loss: 3.852\n",
      "Epoch: 930, Loss: 4.172\n",
      "Epoch: 931, Loss: 3.826\n",
      "Epoch: 932, Loss: 3.710\n",
      "Epoch: 933, Loss: 4.356\n",
      "Epoch: 934, Loss: 4.126\n",
      "Epoch: 935, Loss: 3.692\n",
      "Epoch: 936, Loss: 3.634\n",
      "Epoch: 937, Loss: 4.264\n",
      "Epoch: 938, Loss: 3.630\n",
      "Epoch: 939, Loss: 3.105\n",
      "Epoch: 940, Loss: 4.368\n",
      "Epoch: 941, Loss: 4.123\n",
      "Epoch: 942, Loss: 4.073\n",
      "Epoch: 943, Loss: 4.121\n",
      "Epoch: 944, Loss: 4.645\n",
      "Epoch: 945, Loss: 4.246\n",
      "Epoch: 946, Loss: 4.678\n",
      "Epoch: 947, Loss: 4.451\n",
      "Epoch: 948, Loss: 4.147\n",
      "Epoch: 949, Loss: 3.768\n",
      "Epoch: 950, Loss: 4.713\n",
      "Epoch: 951, Loss: 4.134\n",
      "Epoch: 952, Loss: 3.777\n",
      "Epoch: 953, Loss: 4.210\n",
      "Epoch: 954, Loss: 4.417\n",
      "Epoch: 955, Loss: 3.726\n",
      "Epoch: 956, Loss: 3.709\n",
      "Epoch: 957, Loss: 4.218\n",
      "Epoch: 958, Loss: 4.747\n",
      "Epoch: 959, Loss: 3.942\n",
      "Epoch: 960, Loss: 3.938\n",
      "Epoch: 961, Loss: 4.274\n",
      "Epoch: 962, Loss: 3.931\n",
      "Epoch: 963, Loss: 3.866\n",
      "Epoch: 964, Loss: 4.309\n",
      "Epoch: 965, Loss: 3.978\n",
      "Epoch: 966, Loss: 4.175\n",
      "Epoch: 967, Loss: 4.336\n",
      "Epoch: 968, Loss: 4.142\n",
      "Epoch: 969, Loss: 4.494\n",
      "Epoch: 970, Loss: 3.629\n",
      "Epoch: 971, Loss: 3.634\n",
      "Epoch: 972, Loss: 4.200\n",
      "Epoch: 973, Loss: 3.790\n",
      "Epoch: 974, Loss: 4.413\n",
      "Epoch: 975, Loss: 3.963\n",
      "Epoch: 976, Loss: 4.609\n",
      "Epoch: 977, Loss: 3.986\n",
      "Epoch: 978, Loss: 4.224\n",
      "Epoch: 979, Loss: 4.783\n",
      "Epoch: 980, Loss: 4.937\n",
      "Epoch: 981, Loss: 3.633\n",
      "Epoch: 982, Loss: 4.646\n",
      "Epoch: 983, Loss: 4.299\n",
      "Epoch: 984, Loss: 4.400\n",
      "Epoch: 985, Loss: 3.972\n",
      "Epoch: 986, Loss: 4.535\n",
      "Epoch: 987, Loss: 4.408\n",
      "Epoch: 988, Loss: 3.778\n",
      "Epoch: 989, Loss: 4.146\n",
      "Epoch: 990, Loss: 3.710\n",
      "Epoch: 991, Loss: 3.522\n",
      "Epoch: 992, Loss: 4.412\n",
      "Epoch: 993, Loss: 3.746\n",
      "Epoch: 994, Loss: 4.120\n",
      "Epoch: 995, Loss: 4.512\n",
      "Epoch: 996, Loss: 3.438\n",
      "Epoch: 997, Loss: 4.038\n",
      "Epoch: 998, Loss: 4.053\n",
      "Epoch: 999, Loss: 4.420\n",
      "Epoch: 1000, Loss: 4.244\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Check out the new network architecture and dataset!\n",
    "\n",
    "Notice that the weights and biases are\n",
    "generated randomly.\n",
    "\n",
    "No need to change anything, but feel free to tweak\n",
    "to test your network, play around with the epochs, batch size, etc!\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.utils import shuffle, resample\n",
    "#from miniflow import *\n",
    "\n",
    "# Load data\n",
    "data = load_boston()\n",
    "X_ = data['data']\n",
    "y_ = data['target']\n",
    "\n",
    "# Normalize data\n",
    "X_ = (X_ - np.mean(X_, axis=0)) / np.std(X_, axis=0)\n",
    "\n",
    "n_features = X_.shape[1]\n",
    "n_hidden = 10\n",
    "W1_ = np.random.randn(n_features, n_hidden)\n",
    "b1_ = np.zeros(n_hidden)\n",
    "W2_ = np.random.randn(n_hidden, 1)\n",
    "b2_ = np.zeros(1)\n",
    "\n",
    "# Neural network\n",
    "X, y = Input(), Input()\n",
    "W1, b1 = Input(), Input()\n",
    "W2, b2 = Input(), Input()\n",
    "\n",
    "l1 = Linear(X, W1, b1)\n",
    "s1 = Sigmoid(l1)\n",
    "l2 = Linear(s1, W2, b2)\n",
    "cost = MSE(y, l2)\n",
    "\n",
    "feed_dict = {\n",
    "    X: X_,\n",
    "    y: y_,\n",
    "    W1: W1_,\n",
    "    b1: b1_,\n",
    "    W2: W2_,\n",
    "    b2: b2_\n",
    "}\n",
    "\n",
    "epochs = 1000\n",
    "# Total number of examples\n",
    "m = X_.shape[0]\n",
    "batch_size = 11\n",
    "steps_per_epoch = m // batch_size\n",
    "\n",
    "graph = topological_sort(feed_dict)\n",
    "trainables = [W1, b1, W2, b2]\n",
    "\n",
    "print(\"Total number of examples = {}\".format(m))\n",
    "\n",
    "# Step 4\n",
    "for i in range(epochs):\n",
    "    loss = 0\n",
    "    for j in range(steps_per_epoch):\n",
    "        # Step 1\n",
    "        # Randomly sample a batch of examples\n",
    "        X_batch, y_batch = resample(X_, y_, n_samples=batch_size)\n",
    "\n",
    "        # Reset value of X and y Inputs\n",
    "        X.value = X_batch\n",
    "        y.value = y_batch\n",
    "\n",
    "        # Step 2\n",
    "        forward_and_backward(graph)\n",
    "\n",
    "        # Step 3\n",
    "        sgd_update(trainables)\n",
    "\n",
    "        loss += graph[-1].value\n",
    "\n",
    "    print(\"Epoch: {}, Loss: {:.3f}\".format(i+1, loss/steps_per_epoch))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}