SMAML_for_Experiments_c1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SMAML_for_Experiments_c1",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOnY7b24RAPb6oGtUpMQorM",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/John-zf/SMAML_for_Tool_Wear_Monitoring-Prediction/blob/master/SMAML_for_Experiments_c1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrqRNqI6RrhI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181
        },
        "outputId": "04b3ce28-f835-400d-dabd-749778ff80b1"
      },
      "source": [
        "from google.colab import drive\n",
        "import torch\n",
        "\n",
        "print(torch.cuda.get_device_name(0))\n",
        "drive.mount('/content/drive/', force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tesla T4\n",
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive/\n",
            "Tesla T4\n",
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0B6gSMg5nTQZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZXAKVZKS4qv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "e0fb2ba6-1667-4b36-916e-ca926ab4feef"
      },
      "source": [
        "# Load data and transform them into 2d images\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "\n",
        "def get_data(data_folder):\n",
        "  # Read the sensor data\n",
        "  data = []\n",
        "  for file in os.listdir(data_folder):\n",
        "    file_path = data_folder + file\n",
        "    ins = pd.read_csv(file_path, header=None)\n",
        "    ins = np.array(ins).T[:, :224*224]\n",
        "    # Centralization\n",
        "    ins = np.array([x-np.mean(x) for x in ins])\n",
        "    data.append(ins)\n",
        "  data = np.array(data)\n",
        "  print('{} reading is done.'.format(data_folder))\n",
        "  return data\n",
        "\n",
        "data_folder_c1 = '/content/drive/My Drive/mhm_data/c1/c1/'\n",
        "data_folder_c4 = '/content/drive/My Drive/mhm_data/c4/c4/'\n",
        "data_folder_c6 = '/content/drive/My Drive/mhm_data/c6/c6/'\n",
        "\n",
        "data_c1 = get_data(data_folder_c1)\n",
        "data_c4 = get_data(data_folder_c4)\n",
        "data_c6 = get_data(data_folder_c6) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/mhm_data/c1/c1/ reading is done.\n",
            "/content/drive/My Drive/mhm_data/c4/c4/ reading is done.\n",
            "/content/drive/My Drive/mhm_data/c6/c6/ reading is done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y6agwVP8Xc8q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "99abdeae-eae1-48b6-8ec6-5f611534376e"
      },
      "source": [
        "# Load target\n",
        "def get_target(target_file, step=10):\n",
        "    # Read target, \n",
        "    target = pd.read_csv(target_file)\n",
        "    target = np.array(target[['flute_1', 'flute_2', 'flute_3']])\n",
        "    target = [x[0] for x in target]\n",
        "    print(target)\n",
        "    if step == 1:\n",
        "      return target\n",
        "    else:\n",
        "      targets_list = []\n",
        "      for i, value in enumerate(target):\n",
        "        targets = []\n",
        "        for j in range(i, (i+step)):\n",
        "          if j > len(target)-1:\n",
        "            targets.append(target[-1])\n",
        "          else:\n",
        "            targets.append(target[j])\n",
        "        targets_list.append(targets)\n",
        "      return np.array(targets_list)\n",
        "\n",
        "target_file_c1 = '/content/drive/My Drive/mhm_data/c1/c1_wear.csv'\n",
        "target_file_c4 = '/content/drive/My Drive/mhm_data/c4/c4_wear.csv'\n",
        "target_file_c6 = '/content/drive/My Drive/mhm_data/c6/c6_wear.csv'\n",
        "\n",
        "target_c1 = get_target(target_file_c1)\n",
        "target_c4 = get_target(target_file_c4)\n",
        "target_c6 = get_target(target_file_c6)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[32.31711361, 37.914879, 43.08790971, 47.8590723, 52.25032922, 56.28276562, 59.97661561, 63.351287899999996, 66.4253909, 69.21675719, 71.74246752, 74.01887416, 76.06162381, 77.88567991, 79.50534445, 80.93427925, 82.18552678, 83.27153045, 84.20415441, 84.99470288, 85.65393900000001, 86.19210327, 86.61893145, 86.94367205, 87.17510345, 87.32155046, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.39090058, 87.7061647, 88.0993478, 88.49130564, 88.88155409, 89.26963001, 89.65509154, 90.03751838, 90.41651198, 90.79169571, 91.16271501, 91.52923739, 91.89095253, 92.24757218, 92.59883018, 92.94448231, 93.2843062, 93.61810111, 93.94568777, 94.26690815, 94.58162515, 94.88972237, 95.19110376, 95.48569329, 95.77343457, 96.05429047, 96.32824273, 96.59529149, 96.85545491, 97.10876863, 97.35528535, 97.59507432, 97.82822082, 98.05482567, 98.27500469, 98.48888816, 98.69662026, 98.89835857, 99.09427345, 99.2845475, 99.469375, 99.64896132, 99.82352237, 99.99328397, 100.15848129999999, 100.3193584, 100.4761675, 100.62916829999999, 100.7786279, 100.9248194, 101.0680222, 101.20852099999999, 101.3466052, 101.4825683, 101.6167078, 101.7493242, 101.8807204, 102.0112017, 102.1410748, 102.2706475, 102.400228, 102.5301247, 102.6606455, 102.7920973, 102.9247859, 103.05901490000001, 103.1950858, 103.3332975, 103.4739456, 103.6173222, 103.7637155, 103.9134094, 104.0666829, 104.2238103, 104.38506009999999, 104.55069549999999, 104.7209733, 104.89614399999999, 105.07645149999999, 105.26213290000001, 105.45341780000001, 105.6505285, 105.85367959999999, 106.06307759999999, 106.2789212, 106.5014004, 106.73069679999999, 106.96698329999998, 107.210424, 107.46117379999998, 107.71937859999998, 107.985175, 108.2586902, 108.5400419, 108.8293382, 109.1266779, 109.4321497, 109.7458328, 110.06779660000001, 110.3981009, 110.73679540000002, 111.0839203, 111.4395061, 111.8035733, 112.17613300000001, 112.5571865, 112.9467257, 113.344733, 113.75118119999999, 114.1660341, 114.58924609999998, 115.0207626, 115.46052009999998, 115.9084463, 116.36446000000001, 116.82847199999999, 117.3003841, 117.7800905, 118.2674771, 118.7624221, 119.26479609999998, 119.77446210000001, 120.2912763, 120.8150876, 121.3457382, 121.8830639, 122.42689409999998, 122.97705230000001, 123.53335609999999, 124.0956175, 124.66364329999999, 125.2372354, 125.8161908, 126.40030209999999, 126.9893575, 127.58314159999999, 128.18143510000002, 128.78401540000002, 129.3906569, 130.0011312, 130.6152074, 131.2326523, 131.8532309, 132.4767066, 133.1028415, 133.73139640000002, 134.3621319, 134.9948077, 135.62918369999997, 136.2650196, 136.902076, 137.540114, 138.1788957, 138.8181847, 139.45774619999997, 140.09734709999998, 140.73675690000002, 141.375747, 142.01409219999996, 142.6515699, 143.2879608, 143.92304950000002, 144.556624, 145.1884767, 145.8184043, 146.44620780000002, 147.0716935, 147.69467240000003, 148.31496090000002, 148.9323811, 149.5467607, 150.1579333, 150.765739, 151.3700239, 151.970641, 152.5674499, 153.16031719999998, 153.7491167, 154.3337295, 154.914044, 155.4899564, 156.0613707, 156.62819869999998, 157.1903603, 157.7477835, 158.30040459999998, 158.84816840000002, 159.3910281, 159.9289454, 160.4618907, 160.98984319999997, 161.5127908, 162.03073030000002, 162.5436673, 163.0516165, 163.5546014, 164.0526545, 164.5458173, 165.03414030000002, 165.51768280000002, 165.9965134, 166.4707092, 166.9403565, 167.40555030000002, 167.8663945, 168.3230015, 168.7754925, 169.22399719999999, 169.66865380000002, 170.109609, 170.54701730000002, 170.9810419, 171.41185349999998, 171.839631, 172.26456090000002, 172.68683719999999]\n",
            "[31.41635516, 34.892770399999996, 38.10284657, 41.06102301, 43.78119133, 46.27671037, 48.56042098, 50.64466037, 52.54127629, 54.26164088, 55.81666418, 57.21680745, 58.47209617, 59.59213275, 60.58610902, 61.46281844, 62.23066801, 62.89769001, 63.47155339, 63.959575, 64.36873051, 64.70566514, 64.97670409999999, 65.1878629, 65.34485725, 65.45311297, 65.51777545, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.54371906, 65.58464803, 65.89045864, 66.21371695, 66.55393858, 66.9106077, 67.28317974, 67.67108395, 68.07372593, 68.49049008, 68.92074185, 69.36383003, 69.81908890000001, 70.28584028, 70.76339554, 71.25105747, 71.74812216, 72.25388071, 72.76762093, 73.28862893, 73.81619066, 74.3495934, 74.88812711, 75.4310858, 75.97776878, 76.52748184, 77.07953845, 77.6332608, 78.18798081, 78.74304115, 79.29779609, 79.85161241, 80.40387017, 80.95396348, 81.50130122, 82.04530765, 82.58542306, 83.12110433, 83.65182542, 84.17707788, 84.69637128, 85.20923356, 85.71521147, 86.2138708, 86.7047967, 87.18759393, 87.66188705, 88.12732061, 88.58355929, 89.030288, 89.46721198, 89.89405683, 90.31056855, 90.71651354, 91.11167857, 91.49587070000001, 91.86891723, 92.23066559, 92.5809832, 92.91975736, 93.24689504, 93.56232270000001, 93.86598613, 94.15785016, 94.43789847, 94.70613329, 94.96257518, 95.20726268, 95.44025205, 95.66161694, 95.87144806, 96.06985284, 96.25695507, 96.43289455, 96.59782671, 96.75192223, 96.89536664, 97.02835992, 97.15111612, 97.26386291, 97.36684116, 97.46030457, 97.54451915, 97.61976286, 97.68632512, 97.7445064, 97.79461776, 97.83698037, 97.87192512, 97.89979211, 97.92093022, 97.93569664, 97.94445643, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 97.94758207, 98.04939711, 98.16385952, 98.28908463, 98.42538882, 98.57308034, 98.73245902, 98.90381609, 99.08743402, 99.28358632, 99.49253735, 99.71454217, 99.94984642, 100.19868609999999, 100.4612875, 100.73786700000001, 101.0286311, 101.3337761, 101.65348809999999, 101.98794290000001, 102.33730600000001, 102.7017323, 103.0813662, 103.4763417, 103.8867818, 104.31279909999999, 104.7544953, 105.21196129999998, 105.6852774, 106.1745129, 106.67972639999999, 107.20096559999999, 107.7382675, 108.29165829999998, 108.86115330000001, 109.4467572, 110.048464, 110.6662571, 111.3001091, 111.94998229999999, 112.6158284, 113.2975887, 113.9951942, 114.7085656, 115.43761370000001, 116.1822389, 116.94233200000001, 117.7177737, 118.5084351, 119.3141778, 120.13485370000001, 120.97030559999999, 121.8203669, 122.6848622, 123.56360690000001, 124.45640790000002, 125.3630635, 126.28336329999999, 127.2170889, 128.1640137, 129.1239033, 130.09651540000002, 131.0816002, 132.0789007, 133.08815249999998, 134.1090844, 135.14141830000003, 136.1848696, 137.2391473, 138.3039542, 139.37898719999998, 140.46393740000002, 141.5584906, 142.6623269, 143.7751217, 144.8965453, 146.0262635, 147.1639376, 148.3092247, 149.461778, 150.621247, 151.78727769999998, 152.95951269999998, 154.1375919, 155.321152, 156.5098275, 157.7032503, 158.9010505, 160.1028562, 161.3082938, 162.5169886, 163.7285644, 164.9426445, 166.1588513, 167.37680669999997, 168.59613280000002, 169.8164512, 171.0373842, 172.2585545, 173.47958540000002, 174.7001014, 175.9197281, 177.1380923, 178.35482290000002, 179.5695503, 180.7819071, 181.99152819999998, 183.1980513, 184.4011164, 185.6003667, 186.79544860000001, 187.9860118, 189.1717095, 190.352199, 191.5271412, 192.6962013, 193.8590491, 195.01535859999998, 196.16480890000003]\n",
            "[62.78864802, 64.13604296, 65.36527295, 66.4868801, 67.51080822, 68.44642494, 69.30254323, 70.08744241, 70.8088887, 71.47415519, 72.09004135, 72.66289207, 73.19861618, 73.70270453, 74.18024756, 74.63595243, 75.07415974, 75.49885970000001, 75.91370798, 76.32204104, 76.72689103, 77.13100039, 77.53683587, 77.94660226, 78.36225568, 78.78551648, 79.21788179, 79.66063762, 80.11487062, 80.58147952, 81.06118616, 81.55454615, 82.06195924, 82.58367929, 83.11982397, 83.67038407, 84.23523251, 84.81413304, 85.40674866, 86.01264963, 86.63132132, 87.26217167, 87.90453839, 88.55769588, 89.22086191, 89.89320394, 90.57384526, 91.26187086, 91.956333, 92.65625657, 93.36064421, 94.06848117, 94.77873996, 95.49038474, 96.20237554, 96.91367223, 97.62323828, 98.33004431, 99.03307145, 99.73131453, 100.423785, 101.10951379999999, 101.7875539, 102.45698259999999, 103.1169041, 103.7664512, 104.4047877, 105.0311097, 105.64464740000001, 106.24466650000001, 106.8304699, 107.4013983, 107.95683149999999, 108.4961893, 109.0189324, 109.52456299999999, 110.0126251, 110.48270520000001, 110.9344325, 111.3674791, 111.7815604, 112.17643429999998, 112.55190209999999, 112.9078077, 113.24403729999999, 113.56051940000002, 113.8572241, 114.1341627, 114.39138700000001, 114.62898870000001, 114.84709869999999, 115.0458861, 115.2255578, 115.386357, 115.52856270000001, 115.6524886, 115.7584818, 115.8469217, 115.91821940000001, 115.9728158, 116.0111807, 116.03381159999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.04123229999999, 116.2270765, 116.6029478, 116.9916592, 117.39283370000001, 117.8060712, 118.23094990000001, 118.66702679999999, 119.1138385, 119.5709023, 120.03771709999998, 120.5137642, 120.9985086, 121.4914, 121.99187340000002, 122.4993512, 123.01324340000001, 123.5329492, 124.05785829999999, 124.58735190000002, 125.1208041, 125.65758319999999, 126.1970529, 126.73857379999998, 127.2815045, 127.8252031, 128.36902890000002, 128.912343, 129.4545105, 129.9949016, 130.532893, 131.0678691, 131.5992242, 132.1263628, 132.6487023, 133.16567320000001, 133.6767215, 134.1813097, 134.67891799999998, 135.1690461, 135.65121440000001, 136.1249653, 136.5898649, 137.0455035, 137.491498, 137.9274922, 138.3531587, 138.7681996, 139.1723481, 139.56536930000001, 139.9470615, 140.3172571, 140.6758236, 141.0226648, 141.3577211, 141.680971, 141.99243130000002, 142.2921584, 142.58024809999998, 142.856837, 143.1221024, 143.3762632, 143.61957969999997, 143.85235409999999, 144.07493069999998, 144.28769590000002, 144.4910781, 144.6855475, 144.87161630000003, 145.0498375, 145.2208054, 145.3851541, 145.5435575, 145.69672830000002, 145.8454166, 145.9904093, 146.1325289, 146.27263159999998, 146.4116064, 146.550373, 146.6898802, 146.83110380000002, 146.9750444, 147.1227254, 147.27519009999997, 147.43349919999997, 147.5987281, 147.7719636, 147.95430080000003, 148.1468398, 148.35068169999997, 148.5669251, 148.7966619, 149.0409732, 149.30092439999999, 149.5775609, 149.871903, 150.1849406, 150.517628, 150.870878, 151.245556, 151.64247369999998, 152.0623831, 152.5059689, 152.9738426, 153.4665341, 153.98448480000002, 154.5280396, 155.09743840000002, 155.6928081, 156.31415330000002, 156.9613477, 157.63412399999999, 158.3320649, 159.05459219999997, 159.800957, 160.57022849999998, 161.36128300000001, 162.17279230000003, 163.0032119, 163.85076850000002, 164.7134478, 165.5889811, 166.47483180000003, 167.36818190000002, 168.2659177, 169.1646147, 170.06052340000002, 170.94955269999997, 171.827255, 172.68880900000002, 173.52900359999998, 174.3422199, 175.1224141, 175.86309880000002, 176.5573244, 177.19766040000002, 177.776175, 178.2844153, 178.7133868, 179.05353190000002, 179.2947081, 179.4261663, 179.43652749999998, 179.43652749999998, 179.43652749999998]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UyLPcz9wXJV8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# split data\n",
        "\n",
        "######## Training and Test Data ########\n",
        "train_fea = [data_c4, data_c6]\n",
        "train_tar = [target_c4, target_c6]\n",
        "test_fea = data_c1\n",
        "test_tar = target_c1\n",
        "#############################################\n",
        "\n",
        "train_data = np.vstack(train_fea)\n",
        "train_label = np.vstack(train_tar)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vFYzvIESga_5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Traditional feature extraction\n",
        "import numpy as np\n",
        "import scipy.stats as sts\n",
        "from pywt import WaveletPacket\n",
        "\n",
        "def rms_fea(a):\n",
        "\treturn np.sqrt(np.mean(np.square(a)))\n",
        "\n",
        "def var_fea(a):\n",
        "\treturn np.var(a)\n",
        "\n",
        "def max_fea(a):\n",
        "\treturn np.max(a)\n",
        "\n",
        "def pp_fea(a):\n",
        "\treturn np.max(a)-np.min(a)\n",
        "\n",
        "def skew_fea(a):\n",
        "\treturn sts.skew(a)\n",
        "\n",
        "def kurt_fea(a):\n",
        "\treturn sts.kurtosis(a)\n",
        "\n",
        "def wave_fea(a):\n",
        "\twp = WaveletPacket(a,'db1', maxlevel=8)\n",
        "\tnodes = wp.get_level(8, \"freq\")\n",
        "\treturn np.linalg.norm(np.array([n.data for n in nodes]), 2)\n",
        "\n",
        "def spectral_kurt(a):\n",
        "\tN = len(a)\n",
        "\tmag = np.abs(np.fft.fft(a))\n",
        "\tmag\t= mag[1: int(N/2)] * 2.00/N\n",
        "\treturn sts.kurtosis(mag)\n",
        "\n",
        "def spectral_skw(a):\n",
        "\tN = len(a)\n",
        "\tmag = np.abs(np.fft.fft(a))\n",
        "\tmag\t= mag[1:int(N/2)]*2.00/N\n",
        "\treturn sts.skew(mag)\n",
        "\n",
        "def spectral_pow(a):\n",
        "\tN = len(a)\n",
        "\tmag = np.abs(np.fft.fft(a))\n",
        "\tmag\t= mag[1:int(N/2)]*2.00/N\n",
        "\treturn np.mean(np.power(mag, 3))\n",
        " \n",
        "def extract_fea(data):\n",
        "\tdata_fea = []\n",
        "\tfor i in range(7):\n",
        "\t\tdata_fea.append(rms_fea(data[i]))\n",
        "\t\tdata_fea.append(var_fea(data[i]))\n",
        "\t\tdata_fea.append(max_fea(data[i]))\n",
        "\t\tdata_fea.append(pp_fea(data[i]))\n",
        "\t\tdata_fea.append(skew_fea(data[i]))\n",
        "\t\tdata_fea.append(kurt_fea(data[i]))\n",
        "\t\tdata_fea.append(wave_fea(data[i]))\n",
        "\t\tdata_fea.append(spectral_kurt(data[i]))\n",
        "\t\tdata_fea.append(spectral_skw(data[i]))\n",
        "\t\tdata_fea.append(spectral_pow(data[i]))\n",
        "\treturn np.array(data_fea)\n",
        " \n",
        "trian_data_ml = np.array([extract_fea(x) for x in train_data])\n",
        "test_data_ml = np.array([extract_fea(x) for x in test_fea])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "veya2xFfgd6q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 215
        },
        "outputId": "f81a39b9-ef87-409e-e8d6-ecbcaa179322"
      },
      "source": [
        "from sklearn.svm import SVR\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from xgboost import XGBRegressor\n",
        "from lightgbm import LGBMRegressor\n",
        "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "setp = 9\n",
        "train_label_ml = train_label[:, setp]\n",
        "test_label_ml = test_tar[:, setp]\n",
        "\n",
        "scaler = StandardScaler()\n",
        "scaler.fit(trian_data_ml)\n",
        "train_data_sc = scaler.transform(trian_data_ml)\n",
        "test_data_sc = scaler.transform(test_data_ml)\n",
        "\n",
        "# SVR\n",
        "regr = SVR()\n",
        "regr.fit(train_data_sc, train_label_ml)\n",
        "preds = regr.predict(test_data_sc)\n",
        "print('SVR -> MSE:{:.2f}, MAE:{:.2f}'.format(mean_squared_error(test_label_ml, preds), mean_absolute_error(test_label_ml, preds)))\n",
        "\n",
        "# XGBoost\n",
        "seed = 2020\n",
        "for i in range(10):\n",
        "  regr = XGBRegressor(\n",
        "      learning_rate=0.1,\n",
        "      n_estimators=200,\n",
        "      max_depth=1,\n",
        "      subsample=0.8,\n",
        "      colsample_bytree=0.8,\n",
        "      objective='reg:squarederror',\n",
        "      random_state=seed+i)\n",
        "  regr.fit(train_data_sc, train_label_ml)\n",
        "  preds = regr.predict(test_data_sc)\n",
        "  print('{:.2f},{:.2f}'.format(mean_squared_error(test_label_ml, preds), mean_absolute_error(test_label_ml, preds)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SVR -> MSE:157.26, MAE:8.22\n",
            "104.73,8.21\n",
            "96.34,7.99\n",
            "95.17,7.99\n",
            "108.86,8.21\n",
            "122.76,8.85\n",
            "96.33,7.87\n",
            "100.51,7.94\n",
            "100.91,7.87\n",
            "113.57,8.47\n",
            "104.14,8.06\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KN6sGwhTeFM4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "ddfac65e-5ea0-4e40-8143-a7aa6f9e0382"
      },
      "source": [
        "# For Pytorch\n",
        "import random\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, random_split\n",
        "\n",
        "\n",
        "def seed_torch(seed=2020):\n",
        "  torch.manual_seed(seed)\n",
        "  torch.cuda.manual_seed_all(seed)\n",
        "  np.random.seed(seed)\n",
        "  random.seed(seed)\n",
        "  torch.backends.cudnn.deterministic = True\n",
        "seed_torch()\n",
        "\n",
        "train_data_dl = torch.from_numpy(train_data).float()\n",
        "train_label_dl = torch.from_numpy(train_label).float()\n",
        "deal_dataset = TensorDataset(train_data_dl, train_label_dl)\n",
        "\n",
        "# Split training and validation data\n",
        "ratio = 0.9\n",
        "num_train = int(ratio * len(deal_dataset))\n",
        "num_val = len(deal_dataset) - num_train\n",
        "train_data_dl, val_data_dl = random_split(deal_dataset, [num_train, num_val])\n",
        "print('train:', len(train_data_dl), 'validation:', len(val_data_dl))\n",
        "train_loader = DataLoader(dataset=train_data_dl, batch_size=16, shuffle=True)\n",
        "val_loader = DataLoader(dataset=val_data_dl, batch_size=16, shuffle=True)\n",
        "\n",
        "# Test data\n",
        "test_data_dl = torch.from_numpy(test_fea).float()\n",
        "test_label_dl = torch.from_numpy(test_tar).float()\n",
        "\n",
        "deal_test = TensorDataset(test_data_dl, test_label_dl)\n",
        "test_loader = DataLoader(dataset=deal_test, batch_size=16, shuffle=False)\n",
        "print('test:', len(deal_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train: 567 validation: 63\n",
            "test: 315\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z8EAMjtIXp8y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Define Mono Loss Function\n",
        "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F \n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "class MonoLossFunc(nn.Module):\n",
        "  def __init__(self, lamb=1):\n",
        "    super(MonoLossFunc, self).__init__()\n",
        "    self.lamb = lamb\n",
        "\n",
        "  def forward(self, prediction, truth):\n",
        "    # calculate mse\n",
        "    mse_func = torch.nn.MSELoss(reduction='sum')\n",
        "    loss1 = mse_func(prediction, truth)\n",
        "\n",
        "    # calculate monotonicity\n",
        "    data1 = prediction[:, : -1]\n",
        "    data2 = prediction[:, 1 :]\n",
        "    loss2 = torch.sum(torch.clamp(data1 - data2, min=0))\n",
        "    return loss1 + self.lamb * loss2\n",
        "\n",
        "\n",
        "def evaluate(model, hidden, criteria, val_loader, test_loader):\n",
        "  model.eval()\n",
        "  val_loss = 0.0\n",
        "  # hidden = model.hidden\n",
        "  hidden = model.initHidden()\n",
        "  for i, (inputs, labels) in enumerate(val_loader):\n",
        "    inputs = inputs.to(device)\n",
        "    labels = labels.to(device)\n",
        "    output, _ = model(inputs, hidden)\n",
        "    loss = criteria(output, labels)\n",
        "    val_loss += loss.item()\n",
        "  val_loss = val_loss/63\n",
        "\n",
        "  mse_0 = 0.0\n",
        "  mae_0 = 0.0\n",
        "  mse_1 = 0.0\n",
        "  mae_1 = 0.0\n",
        "  mse_5 = 0.0\n",
        "  mae_5 = 0.0\n",
        "  mse_9 = 0.0\n",
        "  mae_9 = 0.0\n",
        "\n",
        "  for i, (inputs, labels) in enumerate(test_loader):\n",
        "    inputs = inputs.to(device)\n",
        "    labels = labels.to(device)\n",
        "    output, weights = model(inputs, hidden)\n",
        "    mse_0 += torch.sum(torch.pow((output[:, 0] - labels[:, 0]), 2)).item()\n",
        "    mae_0 += torch.sum(torch.abs(output[:, 0] - labels[:, 0])).item()\n",
        "\n",
        "    mse_1 += torch.sum(torch.pow((output[:, 1] - labels[:, 1]), 2)).item()\n",
        "    mae_1 += torch.sum(torch.abs(output[:, 1] - labels[:, 1])).item()\n",
        "\n",
        "    mse_5 += torch.sum(torch.pow((output[:, 5] - labels[:, 5]), 2)).item()\n",
        "    mae_5 += torch.sum(torch.abs(output[:, 5] - labels[:, 5])).item()\n",
        "\n",
        "    mse_9 += torch.sum(torch.pow((output[:, 9] - labels[:, 9]), 2)).item()\n",
        "    mae_9 += torch.sum(torch.abs(output[:, 9] - labels[:, 9])).item()\n",
        "\n",
        "  return val_loss, weights, mse_0/315, mae_0/315, mse_1/315, mae_1/315,\\\n",
        "      mse_5/315, mae_5/315, mse_9/315, mae_9/315\n",
        "\n",
        "\n",
        "def train(model, lamb=0, max_iter=200):\n",
        "  learning_rate = 0.001\n",
        "  model.to(device)\n",
        "  # hidden = model.hidden\n",
        "\n",
        "  optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.9)\n",
        "  criteria = MonoLossFunc(lamb=lamb)\n",
        "\n",
        "  # training the model\n",
        "  for epoch in range(max_iter):\n",
        "    model.train()\n",
        "\n",
        "    hidden = model.initHidden()\n",
        "    train_loss = 0.0\n",
        "    for i, (inputs, labels) in enumerate(train_loader):\n",
        "      inputs = inputs.to(device)\n",
        "      labels = labels.to(device)\n",
        "      optimizer.zero_grad()\n",
        "      output, _ = model(inputs, hidden)\n",
        "      loss = criteria(output, labels)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      train_loss += loss.item()\n",
        "    train_loss = train_loss/567\n",
        "    val_loss, weights, mse_0, mae_0, mse_1, mae_1, mse_5, mae_5, \\\n",
        "    mse_9, mae_9 = evaluate(model, hidden, criteria, val_loader, test_loader)\n",
        "    print('EPOCH={} {:.6f} {:.6f} {:.6f} {:.6f} {:.6f} {:.6f} {:.6f}\\\n",
        "    {:.6f} {:.6f} {:.6f}'.format(epoch+1, train_loss, val_loss, \\\n",
        "    mse_0, mae_0, mse_1, mae_1, mse_5, mae_5, mse_9, mae_9))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dsMC-MaZxXoa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c255f9d4-fcbd-4c3d-8196-d327adbe35f0"
      },
      "source": [
        "# CNN\n",
        "class Model(nn.Module):\n",
        "  def __init__(self, hidden_size=256, output_size=10):\n",
        "    super(Model, self).__init__()\n",
        "    # Encoder\n",
        "    self.hidden_size = hidden_size\n",
        "    self.output_size = output_size\n",
        "    self.encoder = nn.Sequential(\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7, 7*9, 3),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 112*112\n",
        "\n",
        "          nn.Conv2d(7*9, 7*18, 3),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 56*56\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*18, 7*36, 3),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 28*28\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*36, 7*72, 3),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 14*14\n",
        "\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7*72, 7*72, 3),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 7*7\n",
        "          nn.BatchNorm2d(7*72),\n",
        "\n",
        "          nn.Flatten(),\n",
        "          nn.Linear(7*7*7*72, 256*10),\n",
        "          nn.Linear(256*10, 10),\n",
        "        )\n",
        "\n",
        "  def forward(self, input, hidden):\n",
        "    input = input.view(-1, 7, 224, 224)\n",
        "    # Encoder\n",
        "    output = self.encoder(input)\n",
        "    return output, 1\n",
        "\n",
        "  def initHidden(self):\n",
        "    return torch.zeros(16, self.output_size, self.hidden_size*2, device=device)\n",
        "\n",
        "model = Model()\n",
        "train(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH=1 27683.128210 36688.550595 627.885124 21.993053 2091.266655 35.938150 1800.562021    34.497418 2406.853189 39.389100\n",
            "EPOCH=2 5340.267985 9899.740079 117.201517 8.187642 358.714604 14.982607 155.234696    10.207590 345.865407 14.627842\n",
            "EPOCH=3 3577.264736 4513.816716 1056.789618 26.609429 642.008478 22.275126 870.207278    25.326321 761.157626 24.945435\n",
            "EPOCH=4 3504.846712 1507.063957 777.813053 25.245856 648.743756 24.848861 580.305469    21.914392 497.630811 21.522438\n",
            "EPOCH=5 7960.360112 6094.723834 156.329126 10.041890 282.072269 14.069490 109.766783    8.553682 224.894491 12.614897\n",
            "EPOCH=6 4155.295180 8767.729167 347.000355 16.050343 995.784263 26.607640 264.313168    13.997479 442.170245 18.191059\n",
            "EPOCH=7 3740.208430 5448.924727 415.878051 18.461288 181.907499 11.656418 246.155722    13.166414 433.687635 17.042779\n",
            "EPOCH=8 4528.493080 67627.210317 3832.586361 50.377995 5695.053048 61.800994 2777.929237    46.996487 2483.623419 46.562377\n",
            "EPOCH=9 3651.211018 2288.940104 450.389682 19.464814 98.810270 7.045252 562.402539    21.270136 194.152254 11.270996\n",
            "EPOCH=10 2810.980461 12447.835317 381.339987 15.791400 404.278321 16.285113 285.469454    13.197691 359.398853 15.047892\n",
            "EPOCH=11 2353.149283 2926.340712 574.658729 21.816489 486.035824 20.218499 331.805591    16.868343 700.709910 24.205151\n",
            "EPOCH=12 3327.943768 4863.295449 892.277041 24.376013 810.842581 23.039888 757.650467    21.262826 608.520915 23.233620\n",
            "EPOCH=13 2397.714668 1693.292287 458.082954 19.683489 412.550488 18.984784 500.267004    19.149651 882.460403 22.583768\n",
            "EPOCH=14 1722.885672 3480.069072 1273.242536 34.304872 1158.414647 32.237049 1236.668066    34.008125 1381.380256 36.352741\n",
            "EPOCH=15 1864.431162 768.119234 520.465575 22.015371 665.858040 24.602928 491.115803    20.389656 457.772784 20.535263\n",
            "EPOCH=16 1932.495609 3627.602524 858.824224 26.562584 805.287187 25.732402 774.148246    25.617390 802.664327 26.405597\n",
            "EPOCH=17 1692.797471 3890.140253 823.667949 25.872634 995.557402 28.065957 813.903184    24.990268 567.451788 21.323046\n",
            "EPOCH=18 1448.704271 4020.717758 271.931118 13.632006 304.460835 14.797467 394.776880    16.474659 277.176206 13.731876\n",
            "EPOCH=19 1475.613587 3978.222966 1025.069345 30.470483 947.381910 29.050786 1015.234906    30.578426 901.851718 28.519326\n",
            "EPOCH=20 1892.527221 14344.382688 1820.701060 40.286248 1797.377849 39.758834 1997.693283    41.767749 1871.438918 40.142029\n",
            "EPOCH=21 1233.051351 776.156250 874.574154 28.389229 849.720558 27.991127 956.306576    29.869662 879.812008 28.441888\n",
            "EPOCH=22 1096.413346 2608.114955 192.297053 12.015428 175.265447 11.001278 244.198961    14.607571 126.048008 8.632995\n",
            "EPOCH=23 1204.362724 623.735491 960.624162 29.661727 926.268418 29.141318 825.634158    26.877830 924.731961 29.309206\n",
            "EPOCH=24 1790.541068 1446.004557 437.780735 19.065636 438.386449 19.381699 105.667941    8.663863 345.488127 16.435540\n",
            "EPOCH=25 1141.174052 3365.840588 727.116616 24.402546 764.141101 23.385544 829.839494    22.201081 677.732786 22.719680\n",
            "EPOCH=26 991.759324 2206.977586 1013.304631 30.707667 917.918682 28.847640 975.412622    30.122504 807.072143 27.017704\n",
            "EPOCH=27 642.845282 2322.340185 609.865030 23.022231 633.567530 23.614740 524.828676    20.892319 511.285292 20.511604\n",
            "EPOCH=28 777.098238 1110.553664 355.032373 17.676818 275.476416 15.256024 346.977186    17.485110 216.687586 13.249623\n",
            "EPOCH=29 1155.550441 4122.351252 943.888353 28.789521 964.449295 28.401294 924.567144    29.060601 845.702570 27.329827\n",
            "EPOCH=30 740.680589 4741.096757 1235.507352 32.560585 1232.279143 32.230439 1164.516429    31.117300 1092.764146 29.801181\n",
            "EPOCH=31 1306.082793 1588.208457 1078.415868 31.443713 1045.479907 30.936298 964.472443    29.551456 907.807617 28.641174\n",
            "EPOCH=32 532.607315 3285.135727 950.205308 28.794713 911.870492 28.265106 872.381159    27.789102 848.830334 27.441920\n",
            "EPOCH=33 1186.909416 3269.769097 1371.379012 35.972121 1340.360678 35.513607 1333.785674    35.428660 1107.557677 32.081487\n",
            "EPOCH=34 661.014792 1601.203962 290.407661 15.638321 318.854969 16.496067 252.907226    14.589340 267.587296 15.200774\n",
            "EPOCH=35 874.093346 254.871671 697.857096 25.231950 665.958511 24.624026 590.855115    23.264499 561.247669 22.763152\n",
            "EPOCH=36 713.652036 2946.672991 288.962895 14.665524 282.873651 14.618639 217.900168    12.800631 176.517391 11.474661\n",
            "EPOCH=37 695.753062 2992.696615 1022.103638 29.894113 1017.344607 29.858140 915.459157    27.733050 822.739102 25.742349\n",
            "EPOCH=38 673.802633 1087.373062 395.404229 18.631957 382.741283 18.319520 345.547535    17.354845 299.662951 16.061145\n",
            "EPOCH=39 538.734180 2822.465402 967.296355 28.158652 929.442902 27.580378 843.845876    25.506501 782.582058 24.089722\n",
            "EPOCH=40 937.344390 6679.826761 425.558100 18.647212 422.351383 18.511478 403.565849    18.114713 353.416453 16.890632\n",
            "EPOCH=41 1625.476105 3514.777034 801.629454 21.988908 857.118842 21.533991 727.973266    19.642577 710.341926 18.756577\n",
            "EPOCH=42 1069.933313 1364.190724 591.526581 22.258218 545.536293 21.257901 573.576009    21.878458 429.339859 18.909951\n",
            "EPOCH=43 744.756060 421.087038 757.744231 25.969817 795.087655 26.619054 719.596614    25.348907 686.885175 24.769165\n",
            "EPOCH=44 501.768038 266.037241 741.956454 25.380181 769.356009 25.833108 678.624838    24.273305 617.432372 23.174576\n",
            "EPOCH=45 728.484652 475.657273 730.899145 25.462063 768.292934 26.155563 623.474528    23.258205 534.547721 21.211583\n",
            "EPOCH=46 855.603781 859.157645 725.708181 25.545101 757.889252 25.682954 599.108424    22.554670 484.136446 20.022278\n",
            "EPOCH=47 580.309864 2026.744606 835.137493 26.082759 835.387964 25.594606 764.823666    24.393514 673.356867 22.197692\n",
            "EPOCH=48 603.926971 222.763649 792.558565 26.648151 731.846674 25.484139 734.707321    25.623514 695.095969 25.000774\n",
            "EPOCH=49 542.748829 567.205078 835.159673 27.475553 830.253596 27.358807 720.788992    25.505278 675.193471 24.546297\n",
            "EPOCH=50 279.743206 350.429858 595.470985 22.185936 599.124335 22.275369 522.958469    20.433686 484.064815 19.424537\n",
            "EPOCH=51 403.259258 1847.736824 415.547133 18.993670 401.452026 18.660730 392.649847    18.555127 372.258834 18.054551\n",
            "EPOCH=52 496.281724 289.335569 539.793272 21.736900 539.194001 21.602137 503.572883    20.821532 484.312041 20.447544\n",
            "EPOCH=53 246.869896 752.226020 561.549605 22.269951 551.557702 22.058708 505.626947    21.142138 456.501888 20.018477\n",
            "EPOCH=54 502.698948 1008.321708 425.590499 18.390226 377.098168 16.808370 321.470775    15.065561 281.808722 13.726150\n",
            "EPOCH=55 823.330920 969.669147 168.430213 11.378199 165.111772 11.217172 144.544460    10.009957 138.276945 9.347291\n",
            "EPOCH=56 864.755762 1566.997861 805.699006 26.763805 1005.157166 30.420028 801.977076    26.983277 824.135533 27.531264\n",
            "EPOCH=57 357.677386 320.593006 662.999916 24.406993 958.891168 29.510445 663.341717    24.593986 675.982572 24.871587\n",
            "EPOCH=58 432.575863 389.890082 659.601940 23.439286 637.204652 22.740514 551.648898    20.773518 486.937674 19.161794\n",
            "EPOCH=59 405.520405 553.611313 1180.412726 32.630763 1027.229060 30.175537 960.283201    29.142494 850.369757 27.189353\n",
            "EPOCH=60 339.466618 154.979198 537.739254 21.394208 532.768670 21.193484 501.127600    20.451293 479.605386 19.947785\n",
            "EPOCH=61 359.427498 729.119799 423.884238 19.200663 408.382145 18.769408 356.082042    17.353538 304.324054 15.862639\n",
            "EPOCH=62 399.760320 783.487584 852.419025 27.121259 845.301089 27.008888 724.939087    24.703066 640.001667 22.969749\n",
            "EPOCH=63 444.728185 1500.384828 729.924379 23.966850 728.368513 23.938489 667.807176    22.230526 632.347677 21.044684\n",
            "EPOCH=64 275.853100 345.302878 816.724522 26.815515 771.373633 25.917469 771.887267    26.041258 717.424875 25.082588\n",
            "EPOCH=65 322.505772 515.460759 732.238942 25.636415 710.430533 25.221146 633.476400    23.877955 596.304528 23.160682\n",
            "EPOCH=66 280.125987 452.588642 503.055470 20.586448 485.015682 20.125488 445.839030    19.099385 410.026575 18.241719\n",
            "EPOCH=67 319.273797 227.162171 418.931095 18.160851 403.186264 17.702247 360.866275    16.534359 323.704044 15.445369\n",
            "EPOCH=68 280.188986 319.474044 744.446805 25.824024 702.419719 25.044897 603.721228    23.228908 518.434651 21.467281\n",
            "EPOCH=69 231.638551 265.314783 698.430998 24.575259 676.918689 24.127009 640.842307    23.490624 598.630726 22.698905\n",
            "EPOCH=70 234.611895 1903.035435 543.922279 21.260495 512.271403 20.483889 413.580168    18.035308 355.303357 16.420440\n",
            "EPOCH=71 319.982082 380.628058 546.622519 21.674591 533.886187 21.342233 454.567186    19.388880 405.672282 18.148005\n",
            "EPOCH=72 484.129835 983.730732 1076.994427 30.385592 994.542492 29.032539 948.559601    28.109482 935.483932 27.931560\n",
            "EPOCH=73 388.157421 511.516191 632.389246 22.125803 644.353887 22.568460 616.377325    21.827705 576.730681 20.842529\n",
            "EPOCH=74 257.958872 314.705648 761.199245 26.011413 818.233766 27.044729 770.024643    26.246645 716.454769 25.259826\n",
            "EPOCH=75 199.689346 271.829260 527.397087 20.397330 513.544991 20.034987 452.123183    18.514472 408.370684 17.497815\n",
            "EPOCH=76 432.979332 231.609933 709.163291 25.021443 736.298335 25.461403 613.476169    22.953705 526.765023 20.935221\n",
            "EPOCH=77 192.650976 303.365847 601.085347 22.980799 608.116897 23.002969 566.385406    22.066630 538.690900 21.395901\n",
            "EPOCH=78 340.912312 145.243371 757.763988 26.102333 761.596192 26.169749 684.095173    24.808644 613.457129 23.370925\n",
            "EPOCH=79 481.313435 462.150739 715.600370 24.895115 630.949785 23.248333 551.434088    21.210281 547.141304 20.959722\n",
            "EPOCH=80 440.100694 958.921387 300.738931 14.710470 266.594385 13.735410 245.019153    12.899857 229.033225 12.231195\n",
            "EPOCH=81 369.439642 325.001790 550.870595 19.422025 665.895957 22.209969 599.835101    20.901359 595.317047 21.032223\n",
            "EPOCH=82 228.323573 300.141764 534.325306 21.111551 544.011072 21.224343 486.030357    19.870650 437.399363 18.790368\n",
            "EPOCH=83 310.699645 244.243858 492.851588 20.223317 455.128870 19.085061 391.795097    17.411118 323.065476 15.323194\n",
            "EPOCH=84 248.205152 253.426882 576.980149 22.177429 585.740914 22.324963 538.389312    21.417364 508.133170 20.738444\n",
            "EPOCH=85 190.712966 507.252217 890.884906 27.462001 872.198912 27.117709 795.525382    25.787791 757.787350 25.208668\n",
            "EPOCH=86 264.531644 222.651604 710.325068 24.206264 699.619312 24.015342 638.327213    22.765088 610.440091 22.272283\n",
            "EPOCH=87 145.481011 178.236627 616.615084 22.720259 613.587255 22.609074 648.265468    23.552753 665.857156 24.097957\n",
            "EPOCH=88 134.172744 222.823347 702.597832 24.857000 686.463058 24.552796 632.066295    23.599608 584.762799 22.634115\n",
            "EPOCH=89 201.474951 1096.941003 354.140523 16.827904 340.244694 16.398642 304.814383    15.337018 289.002400 14.966336\n",
            "EPOCH=90 339.792459 492.519330 707.395669 24.673915 679.968882 24.014818 607.831446    22.440522 543.067083 20.847193\n",
            "EPOCH=91 663.640966 1686.538814 1004.883720 28.717052 683.977276 23.839744 828.677623    26.546411 824.694865 25.485623\n",
            "EPOCH=92 505.676699 297.214782 552.111012 21.711766 452.558223 18.862755 502.718252    20.348557 515.318823 20.893862\n",
            "EPOCH=93 325.940959 754.431315 454.225435 19.414161 429.169872 18.726469 476.040072    19.979747 494.959372 20.686978\n",
            "EPOCH=94 678.217923 406.417554 571.862276 22.033434 581.149850 22.236641 571.157334    22.163311 555.389702 21.783149\n",
            "EPOCH=95 829.072385 703.502728 364.053764 16.118942 365.121510 16.950028 388.296579    17.590544 423.712600 18.641804\n",
            "EPOCH=96 277.548012 315.516175 461.815398 19.207841 461.144036 19.019411 451.087433    18.873967 424.629974 18.218043\n",
            "EPOCH=97 221.385843 988.513455 350.492507 17.315607 335.898515 16.905341 332.123771    16.861421 318.907648 16.679686\n",
            "EPOCH=98 300.812583 214.138486 627.457240 22.979208 629.391178 23.140553 618.528988    23.190986 583.559276 22.480072\n",
            "EPOCH=99 236.767596 346.565546 417.749492 18.625475 461.307021 19.649606 420.545172    18.660309 384.624725 17.820413\n",
            "EPOCH=100 209.005847 242.351020 432.215022 18.851077 403.346084 18.116856 394.143046    18.072897 385.603053 18.107733\n",
            "EPOCH=101 248.493691 187.657814 505.858445 20.223606 478.207735 19.337746 492.011987    20.083656 440.837558 18.790542\n",
            "EPOCH=102 345.104760 212.968461 346.223641 16.718133 366.327430 17.162031 335.883681    16.457753 309.401739 15.779472\n",
            "EPOCH=103 188.793133 405.950997 689.440160 23.635474 674.831591 23.329659 643.825458    22.871125 629.239190 22.717530\n",
            "EPOCH=104 216.958060 256.237657 537.280420 20.859463 509.465369 20.185603 525.626186    20.620312 509.159499 20.289929\n",
            "EPOCH=105 105.414187 424.952075 523.217236 20.964237 497.338427 20.362002 449.192148    19.321186 415.896070 18.719822\n",
            "EPOCH=106 189.638011 183.929087 513.116156 20.639876 533.534244 21.128034 466.305623    19.651242 440.099424 19.163674\n",
            "EPOCH=107 165.286585 316.553246 552.247934 20.687429 537.288740 20.582705 495.921025    19.481688 448.289092 18.534244\n",
            "EPOCH=108 170.998687 151.405177 470.663054 19.251943 469.887319 19.048716 435.008452    18.403028 396.754176 17.779051\n",
            "EPOCH=109 109.938496 164.613787 624.860381 22.730816 583.248172 21.911142 532.979332    20.986787 502.447751 20.342140\n",
            "EPOCH=110 129.328140 148.992308 596.093349 22.128829 573.711738 21.660603 558.809756    21.530562 575.823673 22.119632\n",
            "EPOCH=111 117.023283 193.461707 389.221933 16.762372 386.899815 16.779938 381.255746    16.769356 365.168733 16.550229\n",
            "EPOCH=112 102.964252 350.587139 580.773697 21.252131 582.814692 21.433718 562.851441    21.215265 536.471885 20.861604\n",
            "EPOCH=113 96.337959 353.444925 454.285970 19.511263 441.803768 19.187152 451.240708    19.563567 439.816398 19.369008\n",
            "EPOCH=114 199.011260 286.348028 510.922646 19.688151 496.280393 19.179674 443.292796    18.171254 376.648825 16.492964\n",
            "EPOCH=115 134.887710 229.933040 589.327359 22.097982 561.709655 21.482912 523.674725    20.898526 481.992078 20.050088\n",
            "EPOCH=116 85.309053 135.628499 481.072617 19.596971 466.402656 19.240111 483.820451    19.818395 447.924075 18.988313\n",
            "EPOCH=117 118.604586 277.659110 582.899014 20.956885 547.891237 20.328205 469.758541    18.655707 406.700742 16.990558\n",
            "EPOCH=118 122.164490 288.947541 748.132819 25.093694 727.277329 24.697320 717.818600    24.801396 663.372945 24.002524\n",
            "EPOCH=119 138.484402 177.968924 507.095731 20.662904 499.513125 20.468330 495.019899    20.397495 471.779876 19.944495\n",
            "EPOCH=120 124.680718 120.353795 581.436341 22.293384 574.898031 22.173198 537.154627    21.270035 583.071115 22.531920\n",
            "EPOCH=121 95.789354 254.382890 613.126910 22.821396 615.924059 22.956045 596.144770    22.580234 532.503650 21.383424\n",
            "EPOCH=122 139.758056 362.206256 290.294205 15.236887 310.738043 15.841510 332.005941    16.532072 297.590150 15.531823\n",
            "EPOCH=123 150.817970 119.982304 561.306140 21.083323 578.918185 21.457868 557.090951    21.131130 533.796653 20.710244\n",
            "EPOCH=124 95.434035 69.413026 533.655744 21.145724 550.139104 21.549473 516.172763    20.867981 475.744602 20.027903\n",
            "EPOCH=125 117.534340 520.868986 745.619780 25.262727 721.237235 24.732643 636.341303    23.133262 574.790184 22.211257\n",
            "EPOCH=126 177.117491 318.052262 478.343758 19.776216 477.630393 19.810398 456.861155    19.471045 413.077561 18.420485\n",
            "EPOCH=127 113.900324 94.919379 502.240815 20.342669 473.387524 19.676875 434.948899    18.865440 394.646157 17.878131\n",
            "EPOCH=128 94.166158 113.558497 548.404803 21.159323 523.334919 20.623770 458.545447    19.238930 425.811333 18.640921\n",
            "EPOCH=129 95.142288 195.858654 595.957752 22.264940 577.256074 21.941253 558.744865    21.729366 534.535626 21.304295\n",
            "EPOCH=130 162.754027 273.333198 663.320652 24.016206 636.645194 23.490027 638.796629    23.579186 647.874487 23.783886\n",
            "EPOCH=131 221.847539 512.257099 508.742667 20.554308 506.685244 20.550684 505.173872    20.540252 518.611120 20.841773\n",
            "EPOCH=132 556.133549 2975.815228 346.255210 16.286552 393.280255 17.471878 441.727966    18.728542 280.152332 14.659501\n",
            "EPOCH=133 640.789578 302.002689 684.620144 24.842492 545.005552 22.108643 714.921096    25.218600 546.583776 21.462187\n",
            "EPOCH=134 448.480839 522.978919 619.882079 23.507906 642.339796 24.155265 457.658432    20.226223 448.953352 20.093946\n",
            "EPOCH=135 314.737978 219.188911 462.986084 20.001691 411.051768 18.706198 476.481907    20.414924 454.591786 19.835400\n",
            "EPOCH=136 259.757473 288.606457 492.053739 19.987473 483.908737 19.937354 519.215934    20.891355 520.651691 20.965084\n",
            "EPOCH=137 256.500315 563.577001 643.753028 22.424641 622.561051 22.032199 600.589607    21.751712 595.552220 21.891520\n",
            "EPOCH=138 154.843223 203.450583 534.508605 20.393921 532.857773 20.445087 458.545735    19.055035 413.558331 18.110699\n",
            "EPOCH=139 119.955723 119.032847 535.901054 20.746772 524.866030 20.554670 508.946275    20.463738 474.268202 19.756952\n",
            "EPOCH=140 90.232145 228.592429 567.144495 21.726098 559.323287 21.593293 515.332688    20.616737 502.983405 20.355313\n",
            "EPOCH=141 215.198846 837.876411 282.449860 14.717865 272.099160 14.330102 272.449309    14.399358 253.433577 13.760657\n",
            "EPOCH=142 159.687279 273.209682 718.791710 24.497525 699.164860 24.031472 610.881599    22.724128 573.050470 21.905200\n",
            "EPOCH=143 143.585743 588.813911 677.857242 23.701274 676.556565 23.661510 569.963865    21.720763 540.978147 21.157167\n",
            "EPOCH=144 191.792097 232.200525 619.697736 22.881779 590.534198 22.299903 581.302049    22.203555 561.342990 21.846191\n",
            "EPOCH=145 141.664456 118.516450 379.990721 17.638365 383.488943 17.741576 370.874871    17.428119 344.931320 16.698695\n",
            "EPOCH=146 117.612842 681.764664 327.295070 16.146486 331.570447 16.257477 308.547635    15.651892 295.920558 15.329201\n",
            "EPOCH=147 126.276773 146.967826 437.001614 18.731382 432.686299 18.614638 452.991478    19.244140 430.249772 18.741573\n",
            "EPOCH=148 127.815379 135.680825 437.845084 18.508699 432.668899 18.380365 387.863765    17.233442 360.150141 16.508537\n",
            "EPOCH=149 80.064698 114.193241 512.434356 19.897109 513.994700 20.016784 472.600230    19.184601 451.085813 18.756007\n",
            "EPOCH=150 75.717044 130.270636 466.858149 19.237406 464.784719 19.187826 443.512035    18.780528 410.431021 17.996349\n",
            "EPOCH=151 90.368591 138.722821 525.215249 20.490916 508.747371 20.111628 462.647352    19.144226 413.715500 17.894846\n",
            "EPOCH=152 132.650703 689.295883 369.406896 17.534984 364.855958 17.401177 353.925034    17.106908 326.460337 16.241738\n",
            "EPOCH=153 130.750866 123.325073 334.729567 15.706313 325.384365 15.457842 335.661683    15.906741 304.246019 15.029960\n",
            "EPOCH=154 137.388754 338.752868 417.499405 17.914501 406.463924 17.634877 376.050260    17.010161 322.269666 15.522240\n",
            "EPOCH=155 153.368583 258.612991 564.105638 21.505727 538.631974 20.912152 501.801353    19.964258 418.053609 18.154771\n",
            "EPOCH=156 85.378957 178.081938 614.874408 22.821165 593.693868 22.394179 498.954492    20.349850 428.759440 18.646785\n",
            "EPOCH=157 93.739526 110.595957 492.516564 20.208681 467.758343 19.630796 408.036708    18.119093 367.000519 17.031060\n",
            "EPOCH=158 109.884058 149.352667 420.269247 18.193891 387.402961 17.326266 366.720306    16.908536 350.603673 16.596295\n",
            "EPOCH=159 102.580771 688.545170 690.815640 23.820115 683.638621 23.757545 615.201061    22.468967 557.366462 21.323292\n",
            "EPOCH=160 126.854729 417.679401 429.862387 18.436340 386.351896 17.302600 351.081715    16.545780 335.180971 16.190197\n",
            "EPOCH=161 240.561166 189.733871 573.959675 22.314130 564.736972 22.164750 558.771081    22.007595 505.148232 20.494217\n",
            "EPOCH=162 284.334145 760.686144 467.328263 19.039409 443.286201 18.517801 447.804044    18.453633 395.500933 16.537348\n",
            "EPOCH=163 467.729364 3527.871032 157.087851 9.714569 145.814489 9.288202 120.709726    8.515506 82.873346 7.294718\n",
            "EPOCH=164 1385.860423 2273.664993 804.040720 24.740573 767.532962 23.839887 780.157068    24.411904 754.691263 23.793344\n",
            "EPOCH=165 8418.829021 69158.828373 3209.487724 47.800390 3226.449947 48.637075 4271.565889    57.819211 4498.505680 60.278812\n",
            "EPOCH=166 30275.523403 43198.931548 3760.506966 51.665378 3100.453895 47.989785 2391.806458    42.879277 4914.163794 56.292740\n",
            "EPOCH=167 5820.687720 1788.373047 256.043200 13.807015 246.760123 13.863035 279.253963    14.792431 416.889571 19.092062\n",
            "EPOCH=168 2219.222069 2424.946522 138.454555 9.944894 159.287354 10.719587 170.227536    11.301940 178.761255 11.995561\n",
            "EPOCH=169 1867.069701 1159.223245 129.405442 9.327225 129.919259 9.429564 125.203634    9.385510 121.744042 9.351413\n",
            "EPOCH=170 1034.366958 1731.676401 201.594307 11.932991 199.553437 11.858238 191.984616    11.597375 217.930530 12.281849\n",
            "EPOCH=171 852.264010 1071.860429 142.569986 9.945874 135.842762 9.740069 116.821034    9.015798 116.358469 9.009392\n",
            "EPOCH=172 738.490161 402.276177 125.758816 8.474492 122.067442 8.442244 108.843530    8.302521 101.500233 8.259283\n",
            "EPOCH=173 734.385086 2538.180928 425.760127 19.160025 437.943118 19.520581 447.879426    19.892499 440.137918 19.711186\n",
            "EPOCH=174 856.083381 1498.904839 276.510862 13.824211 269.613287 13.619615 251.812867    13.054096 210.103235 11.772795\n",
            "EPOCH=175 659.765554 1680.515005 442.824545 19.234881 444.416228 19.373357 426.633300    19.269643 359.960457 17.710650\n",
            "EPOCH=176 657.237948 1633.394934 167.998978 10.206297 157.856166 9.834307 127.044134    8.653232 100.991721 7.638994\n",
            "EPOCH=177 426.141308 890.043418 476.092053 20.094435 473.315537 20.099801 451.844993    19.798588 399.466797 18.662129\n",
            "EPOCH=178 553.663373 954.923673 203.769013 11.969226 201.001888 11.925637 179.796815    11.238824 158.095268 10.447946\n",
            "EPOCH=179 475.341356 1213.497551 413.983784 19.043942 412.847832 19.097580 365.739392    18.075302 304.816382 16.532060\n",
            "EPOCH=180 532.260608 830.720858 435.668369 18.855382 435.394000 18.941984 430.662672    19.089172 400.434620 18.526285\n",
            "EPOCH=181 386.421978 736.969882 209.898294 12.087195 204.014067 11.957841 185.724299    11.583928 160.720618 10.659604\n",
            "EPOCH=182 377.901197 295.862254 272.320671 14.910534 267.803007 14.837716 247.949370    14.455180 224.131870 13.782818\n",
            "EPOCH=183 524.882777 371.779471 178.063808 11.823232 173.519331 11.740334 160.062408    11.359032 142.370552 10.658505\n",
            "EPOCH=184 408.825989 220.322510 280.450361 14.848390 274.196140 14.764616 255.127677    14.505100 223.256834 13.673081\n",
            "EPOCH=185 283.251667 902.127976 215.690158 11.605561 211.860319 11.510997 194.670775    11.104119 176.878216 10.472062\n",
            "EPOCH=186 302.876692 224.539363 304.581355 15.977848 304.155048 16.031685 294.961418    15.949593 264.388961 15.144950\n",
            "EPOCH=187 416.340463 1116.833891 321.646505 16.299136 314.514317 16.148957 284.284396    15.423437 246.775800 14.286723\n",
            "EPOCH=188 312.305939 405.531300 525.929566 21.474896 516.502714 21.316690 483.463404    20.745565 433.620080 19.729154\n",
            "EPOCH=189 238.128790 366.915791 267.790841 14.629644 260.619190 14.488984 243.466884    14.207118 211.328988 13.251531\n",
            "EPOCH=190 294.255197 898.593890 290.700151 14.954812 284.690637 14.843213 266.793038    14.636869 244.817898 14.156318\n",
            "EPOCH=191 314.584558 474.922270 483.744182 20.355007 475.556570 20.210742 432.548580    19.436672 384.256346 18.373452\n",
            "EPOCH=192 431.458199 609.638920 242.736057 14.106904 241.675156 14.114530 230.785086    13.833201 216.065754 13.419785\n",
            "EPOCH=193 281.210902 245.728738 376.774991 18.230291 372.366973 18.143036 359.175770    17.874276 331.791832 17.206415\n",
            "EPOCH=194 197.184287 1489.267361 220.461524 11.443069 216.313708 11.398764 203.716810    11.340245 188.147411 11.007777\n",
            "EPOCH=195 186.569083 183.190767 394.632833 18.502432 392.297631 18.483779 377.345686    18.285658 349.062108 17.687298\n",
            "EPOCH=196 253.052461 237.780620 307.694865 16.313629 302.188769 16.211263 287.659483    15.928080 258.800077 15.149126\n",
            "EPOCH=197 198.910440 332.337282 405.233013 18.381385 395.605434 18.193419 358.260690    17.473972 316.318217 16.424628\n",
            "EPOCH=198 186.400437 339.401685 299.141315 15.777289 294.693355 15.697007 273.059448    15.211575 246.763088 14.504820\n",
            "EPOCH=199 204.933174 256.702307 402.829465 18.818691 398.015937 18.730758 375.450019    18.253387 341.332931 17.449049\n",
            "EPOCH=200 169.176014 171.887970 485.044852 20.769588 476.245850 20.602544 449.379638    20.094838 400.388918 19.014892\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YpEUtVbHvdU-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "15830966-d088-41c3-db8f-f5c9164e1e74"
      },
      "source": [
        "# G-CNN\n",
        "class Model(nn.Module):\n",
        "  def __init__(self, hidden_size=256, output_size=10):\n",
        "    super(Model, self).__init__()\n",
        "    # Encoder\n",
        "    self.hidden_size = hidden_size\n",
        "    self.output_size = output_size\n",
        "    self.encoder = nn.Sequential(\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7, 7*9, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 112*112\n",
        "\n",
        "          nn.Conv2d(7*9, 7*18, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 56*56\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*18, 7*36, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 28*28\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*36, 7*72, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 14*14\n",
        "\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7*72, 7*72, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 7*7\n",
        "          nn.BatchNorm2d(7*72),\n",
        "\n",
        "          nn.Flatten(),\n",
        "          nn.Linear(7*7*7*72, 256*10),\n",
        "          nn.Linear(256*10, 10),\n",
        "        )\n",
        "\n",
        "  def forward(self, input, hidden):\n",
        "    input = input.view(-1, 7, 224, 224)\n",
        "    # Encoder\n",
        "    output = self.encoder(input)\n",
        "    return output, 1\n",
        "\n",
        "  def initHidden(self):\n",
        "    return torch.zeros(16, self.output_size, self.hidden_size*2, device=device)\n",
        "\n",
        "model = Model()\n",
        "train(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH=1 12743.737293 39590.204365 3935.367851 61.738008 4109.781138 63.079644 4280.498127    64.516863 4426.276500 65.818612\n",
            "EPOCH=2 1952.249801 40987.028770 4246.991412 63.009406 4198.339726 62.619859 4353.838281    64.058166 4483.885851 65.549625\n",
            "EPOCH=3 2209.218721 14683.310764 2744.568074 51.135515 2517.025918 49.110526 2699.581548    50.978034 2894.298772 51.945986\n",
            "EPOCH=4 2732.989978 2322.607267 534.482407 19.355042 665.848796 23.271873 370.637835    16.966094 235.291304 13.410157\n",
            "EPOCH=5 2126.484753 11674.061756 334.229836 15.278382 511.574611 20.206872 607.677755    23.377556 444.771816 17.733606\n",
            "EPOCH=6 1323.281538 10539.186260 2762.447445 49.550410 1097.452305 30.041689 68.300176    6.447385 60.719792 6.388438\n",
            "EPOCH=7 824.196668 2002.729570 598.418803 21.970505 86.259277 7.698534 97.556134    8.314916 63.065268 6.734463\n",
            "EPOCH=8 851.337028 1699.702474 159.729804 10.334446 339.489391 16.144755 591.832699    22.727270 309.596468 14.789944\n",
            "EPOCH=9 595.427631 3887.365823 203.927656 11.728085 179.878654 10.948324 250.004055    13.922471 295.610277 15.683408\n",
            "EPOCH=10 619.491805 3026.589410 809.724013 26.609183 762.997680 26.125724 703.750742    24.985742 855.730973 26.931512\n",
            "EPOCH=11 1413.757451 2375.270058 117.367553 8.665334 246.614614 12.355414 112.009171    8.717842 259.929820 14.312990\n",
            "EPOCH=12 1213.658271 23860.828373 1974.009552 43.470675 2452.496770 48.806767 2696.507130    51.329872 2943.155413 53.203455\n",
            "EPOCH=13 899.771623 1116.537280 78.904538 7.065846 108.379061 8.105586 83.362176    7.040518 141.820382 9.440005\n",
            "EPOCH=14 630.314761 5407.000124 1217.309611 33.584875 938.669514 29.332428 739.548272    26.080843 873.061562 28.595957\n",
            "EPOCH=15 708.895619 20321.774306 3000.033104 54.047076 2638.558054 50.648435 3254.914410    56.175123 3135.229067 55.542181\n",
            "EPOCH=16 445.784098 1533.405103 136.757685 9.299359 142.381946 9.694453 71.714953    6.612892 48.170655 5.964834\n",
            "EPOCH=17 417.583933 794.495055 359.328694 17.107351 230.259874 13.252645 313.408299    16.058402 172.911668 10.087529\n",
            "EPOCH=18 320.199053 804.077970 87.550162 7.758202 88.835790 7.678335 67.141043    6.899611 45.864130 5.734741\n",
            "EPOCH=19 326.231861 897.689624 94.650084 8.152224 125.650185 9.006018 91.927202    7.879948 95.285499 8.012405\n",
            "EPOCH=20 672.980005 1705.018136 280.744213 14.482178 328.002965 16.355190 613.785348    23.534138 482.512003 19.341208\n",
            "EPOCH=21 1183.004807 26548.088294 1091.673759 31.467729 1344.513137 35.405681 2427.864831    48.518831 3112.459183 54.458193\n",
            "EPOCH=22 499.367518 15144.196181 673.366853 24.025319 638.198603 23.539170 1113.302144    32.322906 157.513636 10.404145\n",
            "EPOCH=23 275.093612 1995.927486 416.529867 18.971243 412.491093 18.772969 598.800084    23.342044 525.040518 21.783721\n",
            "EPOCH=24 280.059976 977.091347 369.625391 17.667111 350.200520 17.203393 416.754712    19.161436 487.580910 21.093266\n",
            "EPOCH=25 242.717084 3969.897941 74.971025 6.687159 75.899227 6.845782 125.217080    9.396901 100.440388 8.448680\n",
            "EPOCH=26 256.098015 387.838464 164.601118 10.342129 144.857331 9.480521 171.356374    10.609866 120.702623 8.479008\n",
            "EPOCH=27 269.154502 1149.742265 86.681717 7.515163 91.585549 7.647592 78.617769    6.857163 80.488728 6.972107\n",
            "EPOCH=28 344.283590 1605.875341 591.059164 22.399830 483.540706 19.996929 466.986965    19.279852 393.900751 17.308895\n",
            "EPOCH=29 700.531396 3279.640315 682.085858 24.520802 801.515772 26.897318 977.449608    30.084296 1121.905111 32.498037\n",
            "EPOCH=30 360.022674 407.847788 209.684365 12.098516 187.028183 11.326409 236.629416    13.413558 192.890523 12.003095\n",
            "EPOCH=31 446.334233 4275.438740 866.572822 28.115970 985.849236 30.198253 1168.068223    33.194329 1256.392538 34.577640\n",
            "EPOCH=32 633.465037 1676.354198 217.338479 11.445824 174.979843 10.526470 122.955148    9.090239 94.884013 8.176544\n",
            "EPOCH=33 205.742437 3851.790551 106.267125 8.822505 90.836982 8.103782 102.172570    8.625777 83.010855 7.777332\n",
            "EPOCH=34 284.175382 245.493245 163.098767 10.514956 166.482797 10.659171 210.515561    12.623518 204.310970 12.487261\n",
            "EPOCH=35 136.779279 2732.689174 80.973395 7.094801 77.131140 7.030736 96.571880    8.050313 103.369519 8.537872\n",
            "EPOCH=36 174.378735 528.130080 320.205164 15.666326 290.569943 14.921363 283.917258    15.227354 230.562182 13.603878\n",
            "EPOCH=37 371.840928 6371.355283 204.771259 12.179254 204.759595 12.158603 203.843209    12.141514 185.269046 11.458476\n",
            "EPOCH=38 551.073080 5835.591890 192.761868 11.502654 170.916686 10.725947 290.746657    15.413860 322.107632 16.529994\n",
            "EPOCH=39 225.863185 2339.848772 179.260801 11.131617 121.113532 9.103613 84.425731    7.443183 199.052028 12.036566\n",
            "EPOCH=40 390.216572 36644.030754 3132.072197 55.354156 3689.560900 60.206218 6015.769916    77.210882 6727.992460 81.721740\n",
            "EPOCH=41 221.581696 4412.440848 903.420539 28.857733 797.652094 27.022052 942.804574    29.817163 904.035665 29.277057\n",
            "EPOCH=42 180.248811 1120.675859 365.643875 17.296677 382.756468 17.874288 436.948776    19.594805 422.294609 19.400432\n",
            "EPOCH=43 231.612965 415.634773 75.509411 7.263350 77.926162 7.381493 115.130286    8.968790 197.561400 12.597323\n",
            "EPOCH=44 285.284609 12370.719246 368.067882 17.194940 395.309525 18.164551 725.645240    25.919894 1017.326161 31.153339\n",
            "EPOCH=45 202.070074 22699.288194 2430.375992 48.602342 2359.069258 47.917976 3625.846416    59.757576 4007.301612 62.931409\n",
            "EPOCH=46 195.867892 1202.064949 355.334677 17.017139 356.739292 17.082040 378.457990    17.898971 402.650018 18.801668\n",
            "EPOCH=47 297.070618 1309.853516 76.040144 7.257159 63.012707 6.653015 47.954002    5.628016 45.723511 5.553698\n",
            "EPOCH=48 311.300869 423.625465 131.820380 8.885532 125.802979 8.729906 167.845828    10.495032 193.087614 11.627669\n",
            "EPOCH=49 269.106239 2836.690724 828.825964 27.520683 790.949995 26.885192 926.400899    29.509173 887.106879 28.952842\n",
            "EPOCH=50 315.169786 1233.038101 474.751324 19.890969 480.739551 20.112143 486.821746    20.623563 473.526353 20.530719\n",
            "EPOCH=51 251.800313 3857.912512 65.651165 6.764713 67.082328 6.491268 70.053151    6.825813 128.241363 9.620606\n",
            "EPOCH=52 297.786730 428.697754 131.718193 9.503793 177.880871 11.136469 165.470063    10.790209 235.990228 13.607176\n",
            "EPOCH=53 249.449966 1242.942925 87.675309 7.442544 104.073285 7.828510 72.990549    6.632606 80.166569 6.909382\n",
            "EPOCH=54 323.622277 2150.441158 501.376397 20.730065 471.054886 20.220407 641.484842    24.074820 767.446074 26.652537\n",
            "EPOCH=55 217.948424 304.680703 129.832258 9.546341 138.120231 9.809703 179.418873    11.343650 223.119030 13.268005\n",
            "EPOCH=56 153.232452 702.312415 97.004535 8.391943 99.411680 8.503458 63.656967    6.874051 52.289747 6.194447\n",
            "EPOCH=57 168.932164 1618.478082 373.697657 17.162104 402.936187 18.161583 644.741498    24.136394 785.634403 27.044077\n",
            "EPOCH=58 140.021416 257.752534 123.296192 9.254947 136.808378 9.868936 101.484407    8.353070 95.673948 8.060463\n",
            "EPOCH=59 582.642724 1551.654638 420.552535 17.990804 387.310016 17.495775 487.811296    20.482940 501.772630 21.056555\n",
            "EPOCH=60 687.243397 4112.770399 76.524232 7.270954 66.827320 6.519965 45.337841    5.571541 70.709816 6.817030\n",
            "EPOCH=61 277.096629 3277.325087 789.418967 26.658485 960.792422 29.655948 1083.660798    31.894584 1325.409167 35.620314\n",
            "EPOCH=62 249.072747 4389.475198 94.207450 8.211426 85.246462 7.822426 143.079748    10.269647 154.891923 10.604164\n",
            "EPOCH=63 250.037686 2048.768508 444.616256 19.685673 474.863985 20.476100 535.926493    22.170874 602.435862 23.805021\n",
            "EPOCH=64 340.316678 455.362390 178.325471 10.529925 197.102771 11.391651 226.473391    12.298665 254.631826 13.447070\n",
            "EPOCH=65 158.651408 409.593479 249.957330 13.765854 253.882583 14.033771 324.718537    16.491857 395.895366 18.741946\n",
            "EPOCH=66 174.990545 237.228120 137.112893 9.920794 141.610128 10.079369 148.123089    10.394372 182.274298 11.870140\n",
            "EPOCH=67 185.040908 1008.192166 283.273190 14.534174 302.085549 15.208763 469.494533    20.194480 559.584413 22.121910\n",
            "EPOCH=68 196.820428 467.670457 303.996496 15.635504 313.313220 15.988492 369.172589    17.844056 290.455641 15.747365\n",
            "EPOCH=69 303.456235 2150.272507 404.286427 18.057294 467.275675 19.888175 697.274265    25.268321 838.824067 27.997513\n",
            "EPOCH=70 192.067647 3738.346850 786.757090 26.199994 857.277592 27.702849 1053.900874    31.425564 1218.442798 34.162629\n",
            "EPOCH=71 193.572304 1417.122458 542.353072 21.648354 393.549582 18.183378 489.901407    20.962921 703.021261 25.844737\n",
            "EPOCH=72 158.872146 373.232538 178.628341 11.101953 215.048525 12.633076 279.666141    15.040943 346.938056 17.239449\n",
            "EPOCH=73 146.534295 807.765904 292.048429 15.142702 333.505043 16.529802 387.912990    18.266995 551.605133 22.520798\n",
            "EPOCH=74 140.193222 373.734786 237.662054 13.564165 240.696663 13.717334 215.420236    12.859596 171.663213 11.414279\n",
            "EPOCH=75 178.180220 353.086945 101.770237 8.185561 98.776234 8.083385 88.505827    7.652046 98.653353 8.150651\n",
            "EPOCH=76 102.453995 235.158052 182.434460 11.630105 168.258761 11.110628 128.756678    9.567851 111.101642 8.783616\n",
            "EPOCH=77 116.128313 795.465619 97.773757 7.887667 91.302387 7.558407 73.681538    6.763528 68.242160 6.500411\n",
            "EPOCH=78 122.060073 241.839813 236.346994 13.399277 238.052962 13.500460 230.610294    13.426863 241.332474 13.980167\n",
            "EPOCH=79 102.084577 1925.209604 81.404222 7.435976 81.597866 7.586923 67.197575    6.867661 60.098818 6.468403\n",
            "EPOCH=80 143.778064 1849.456318 63.551214 6.791898 55.596568 6.211166 48.917419    5.813495 51.039816 5.839259\n",
            "EPOCH=81 103.427051 1379.500403 424.372937 18.928288 439.020341 19.353489 495.887183    21.022888 489.359503 21.086216\n",
            "EPOCH=82 160.125128 251.361534 242.476685 13.541046 225.162855 13.173214 284.103127    15.162962 297.930010 15.712089\n",
            "EPOCH=83 316.757804 2286.284350 121.974534 8.582328 132.530918 8.804655 92.484584    8.000885 119.059933 9.371950\n",
            "EPOCH=84 175.218279 1756.080047 479.746780 20.514921 515.574368 21.416533 735.252882    26.211018 853.771522 28.496894\n",
            "EPOCH=85 133.432234 247.229980 164.941391 10.766816 172.491728 11.070135 191.176458    11.878336 193.520833 12.279709\n",
            "EPOCH=86 223.957308 2164.553695 354.893137 15.958955 403.718113 17.416770 683.927320    24.247604 972.486641 29.796926\n",
            "EPOCH=87 350.821177 2802.385975 720.318804 25.773653 663.869151 24.704836 781.121997    26.929996 545.960038 22.471586\n",
            "EPOCH=88 181.298394 395.717150 284.495736 14.985502 228.469443 13.414744 279.834782    15.434080 306.204989 16.304078\n",
            "EPOCH=89 122.270077 1031.655025 260.394326 14.123916 277.858915 14.783364 393.147070    18.450335 443.760153 19.987008\n",
            "EPOCH=90 114.892280 443.090619 188.041851 11.525857 190.373147 11.714281 174.310779    11.229831 146.934735 10.367323\n",
            "EPOCH=91 240.331177 2866.656312 113.119838 8.440614 143.817164 10.000113 102.035735    8.190135 117.158374 9.098648\n",
            "EPOCH=92 469.590500 1254.423069 285.257982 14.468472 294.951245 15.034291 278.569616    14.260498 241.738240 12.971099\n",
            "EPOCH=93 244.973241 894.052006 323.638937 16.007706 264.493199 14.191955 290.403502    14.973474 233.619197 13.691043\n",
            "EPOCH=94 262.093923 943.108336 206.354322 11.013575 246.855308 12.572903 378.666590    16.902997 644.507265 24.055534\n",
            "EPOCH=95 305.505882 13534.099206 1223.562168 33.991526 1452.970784 37.268354 2590.979154    50.366768 3461.285156 58.451433\n",
            "EPOCH=96 227.204238 366.726562 107.841007 8.570319 142.868740 9.873823 92.435883    7.954030 81.978845 7.468338\n",
            "EPOCH=97 142.461919 642.029204 342.315142 16.796011 314.959991 16.042326 367.225999    17.784559 411.817116 19.120805\n",
            "EPOCH=98 138.774282 1226.702706 119.649618 9.175790 83.562063 7.781594 61.387571    6.706100 74.911348 6.835804\n",
            "EPOCH=99 142.997351 264.105895 142.015213 9.860331 125.334181 9.177906 103.567088    8.345248 102.714971 8.313089\n",
            "EPOCH=100 124.202997 4234.136905 115.179517 8.254101 113.856301 8.270687 149.049261    10.035038 181.554459 11.606089\n",
            "EPOCH=101 94.668155 800.861948 78.963879 7.231780 72.266254 6.918992 57.210959    6.242893 45.916611 5.612374\n",
            "EPOCH=102 112.860851 258.460678 106.988623 8.565223 106.028151 8.660953 97.285845    8.265846 91.437570 7.997435\n",
            "EPOCH=103 114.184693 302.772806 91.650536 7.780881 90.941061 7.669941 87.149678    7.431447 102.063743 8.053041\n",
            "EPOCH=104 99.032384 203.928017 156.846816 10.440138 162.724898 10.769528 178.818471    11.518173 192.623648 12.120215\n",
            "EPOCH=105 125.792853 679.588666 293.014720 15.078204 330.454338 16.379665 376.947628    17.902635 416.946288 19.162232\n",
            "EPOCH=106 95.641729 294.813217 140.925243 10.055985 144.244841 10.184734 126.864958    9.494341 106.397850 8.688992\n",
            "EPOCH=107 103.128112 1010.677238 90.101131 7.503140 90.052049 7.496338 70.818970    6.740716 58.183914 6.185086\n",
            "EPOCH=108 239.807550 3284.147042 178.290222 11.234902 212.608392 12.410449 239.813155    12.879179 265.882878 13.100600\n",
            "EPOCH=109 233.566064 2024.265687 100.607888 8.398608 99.803197 8.407422 77.281587    7.260089 78.702958 7.188031\n",
            "EPOCH=110 295.967820 1308.995598 393.219570 16.948914 428.500517 18.100726 530.161195    20.885744 649.826732 23.627984\n",
            "EPOCH=111 820.115470 26201.167659 501.381816 20.566236 1054.971666 31.308474 1762.706507    41.247866 1971.109108 43.660274\n",
            "EPOCH=112 813.825666 3911.831907 1504.024306 35.098878 1010.351457 29.098985 686.412536    23.901714 357.067193 16.563594\n",
            "EPOCH=113 729.031662 3569.829613 144.717761 10.328504 108.843380 9.126845 134.113677    9.965138 94.906195 8.435930\n",
            "EPOCH=114 524.714415 742.545030 188.011964 11.246065 182.402874 11.087917 204.620107    11.779549 250.054491 13.236575\n",
            "EPOCH=115 233.026087 794.367699 110.793174 8.476625 97.105073 8.028962 85.017065    7.494137 66.006527 6.727316\n",
            "EPOCH=116 223.922979 379.034292 335.699593 16.492739 317.420329 15.997638 325.879729    16.518946 259.032749 14.582755\n",
            "EPOCH=117 187.166726 3705.908606 557.477068 22.086033 587.719186 22.831836 510.129680    21.335019 659.859206 24.714573\n",
            "EPOCH=118 163.646941 216.142466 101.504365 8.439853 104.774073 8.546952 117.429485    8.924959 153.313735 10.421556\n",
            "EPOCH=119 176.718581 186.699769 132.495620 9.534748 130.450376 9.385401 133.299232    9.536157 149.460654 10.280571\n",
            "EPOCH=120 181.541197 337.415396 112.400762 9.007653 104.077063 8.559742 114.551440    8.932740 149.135361 10.063519\n",
            "EPOCH=121 173.430659 380.058462 125.487229 9.718839 118.302331 9.410306 109.521751    8.861987 79.673196 7.612906\n",
            "EPOCH=122 264.976946 877.579768 89.241946 8.200406 89.333509 8.215322 72.072971    7.356611 63.859512 6.762976\n",
            "EPOCH=123 186.410162 307.729647 142.713535 10.068900 119.966730 9.243017 92.352883    8.067197 66.668336 6.877344\n",
            "EPOCH=124 139.012302 361.257719 224.764103 12.482316 232.751953 12.845319 289.871219    15.047439 311.523690 16.060678\n",
            "EPOCH=125 202.757457 3580.460069 125.265171 9.364524 108.162097 8.595845 104.399766    8.438473 92.266493 7.828917\n",
            "EPOCH=126 143.644887 2789.304253 553.245432 21.687852 580.993989 22.429206 735.180323    25.913975 843.527334 28.070394\n",
            "EPOCH=127 106.004297 569.334124 306.694621 15.489928 312.276746 15.751102 317.532641    16.269086 321.647468 16.643651\n",
            "EPOCH=128 136.012117 349.015741 87.039812 7.941189 87.817068 8.000513 70.270025    7.110399 65.066472 6.754069\n",
            "EPOCH=129 125.219475 178.144297 183.242838 11.417916 184.157625 11.443085 186.360382    11.728595 180.706260 11.635400\n",
            "EPOCH=130 99.899424 227.967173 214.873790 12.386685 220.601749 12.602264 246.543249    13.773289 262.145108 14.480548\n",
            "EPOCH=131 97.731767 246.409769 91.625648 8.093091 86.760910 7.867605 93.290090    7.985892 95.521991 8.059853\n",
            "EPOCH=132 81.258319 445.946545 290.277078 15.128074 294.013346 15.295125 266.512594    14.602137 229.249465 13.444476\n",
            "EPOCH=133 132.852741 264.615773 149.180839 10.060765 155.338555 10.239968 203.029488    12.177794 235.290583 13.460481\n",
            "EPOCH=134 123.213383 2245.769500 78.494197 7.247266 81.319539 7.415788 76.214959    7.141154 86.371758 7.616409\n",
            "EPOCH=135 103.967428 172.693069 186.420230 11.504900 203.677731 12.118900 192.192433    11.946889 182.501438 11.699122\n",
            "EPOCH=136 92.570422 218.800153 121.247170 8.739294 124.700113 8.814873 146.094604    9.808697 159.872204 10.486691\n",
            "EPOCH=137 99.932713 2069.524771 101.682931 8.267185 103.051581 8.362531 79.724234    7.226559 72.536638 6.862785\n",
            "EPOCH=138 77.272505 648.080326 290.126642 14.951466 295.300452 15.141598 341.046660    16.888220 366.556934 17.836005\n",
            "EPOCH=139 68.059780 140.192398 139.466903 10.042243 139.513191 10.009393 155.925656    10.543069 167.298977 11.135960\n",
            "EPOCH=140 90.156694 157.359687 177.583912 11.320791 171.144815 11.090580 173.341803    11.291403 161.193759 10.937602\n",
            "EPOCH=141 109.433285 196.680978 208.022531 12.449396 197.874198 12.125332 209.366402    12.745575 203.302934 12.759206\n",
            "EPOCH=142 85.267950 262.175979 134.239136 9.765521 130.557222 9.636793 90.919279    8.036667 73.933244 7.203422\n",
            "EPOCH=143 77.504960 172.608610 138.982371 9.714885 137.132089 9.666185 161.102089    10.720259 163.595913 10.995009\n",
            "EPOCH=144 71.418325 277.903313 183.111784 11.317442 182.077505 11.262038 245.692800    13.739375 269.774944 14.748358\n",
            "EPOCH=145 86.777753 134.015121 117.021448 9.082118 115.825846 9.049851 119.721876    9.152511 115.427694 9.026598\n",
            "EPOCH=146 117.128748 898.050270 235.715152 13.148557 245.343458 13.456050 279.197849    14.660550 316.648835 16.090284\n",
            "EPOCH=147 102.037439 240.395229 140.234352 10.019832 142.828605 10.110675 111.791597    8.871454 107.514269 8.607637\n",
            "EPOCH=148 91.213434 821.372845 85.011104 7.552730 78.110419 7.247458 67.370781    6.733239 68.259319 6.787459\n",
            "EPOCH=149 94.720129 213.929703 228.666266 13.088854 218.450707 12.769930 188.892414    11.838596 162.437492 10.883593\n",
            "EPOCH=150 100.588772 596.202675 78.901934 7.332033 76.282429 7.183656 57.527945    6.204591 48.723595 5.710157\n",
            "EPOCH=151 146.645121 225.369451 197.483848 11.915198 183.366885 11.419574 136.354749    9.687918 97.549138 8.216509\n",
            "EPOCH=152 97.904255 701.306408 289.741844 15.094354 293.403356 15.229219 375.645946    17.885766 405.591188 18.847227\n",
            "EPOCH=153 142.643178 1218.942181 383.510006 18.039763 399.880860 18.508456 468.835956    20.483328 517.404541 21.772568\n",
            "EPOCH=154 182.878761 683.370133 89.107275 7.941542 80.804097 7.589027 67.043231    6.981155 55.448335 6.405171\n",
            "EPOCH=155 146.282283 267.339708 166.653004 10.665416 176.589254 11.047263 218.809166    12.724437 258.479141 14.373219\n",
            "EPOCH=156 86.203431 234.531324 126.508882 9.078214 137.540027 9.557353 162.437142    10.625298 199.973286 12.301011\n",
            "EPOCH=157 111.962816 308.124845 93.463689 7.830433 94.638086 7.827635 106.574211    8.363785 120.264205 8.940855\n",
            "EPOCH=158 121.258290 668.412466 82.711539 7.371398 77.371996 7.134072 64.184802    6.472107 54.960674 6.008280\n",
            "EPOCH=159 128.868740 529.905211 226.695566 12.904000 241.250042 13.335682 292.679798    15.194244 381.215874 18.025720\n",
            "EPOCH=160 110.705203 192.108561 141.327845 10.068652 130.972539 9.680661 119.073341    9.149699 107.679789 8.672628\n",
            "EPOCH=161 75.881796 158.990494 100.528802 8.351963 99.671023 8.291732 123.313546    9.189758 152.749777 10.468279\n",
            "EPOCH=162 84.186115 1076.987956 93.486647 7.898271 87.598400 7.633416 71.758690    6.875468 62.679542 6.431252\n",
            "EPOCH=163 89.290954 1854.564794 479.167666 20.210966 494.994311 20.610295 521.556806    21.387933 548.243604 22.151532\n",
            "EPOCH=164 94.994578 509.762029 90.523054 7.808346 83.449513 7.479711 68.118077    6.719660 60.697200 6.303371\n",
            "EPOCH=165 57.626263 502.866691 83.452529 7.996659 79.875690 7.844561 69.748545    7.302286 59.015497 6.644756\n",
            "EPOCH=166 86.208601 342.716429 214.652019 12.570847 216.556695 12.677518 251.161950    14.074213 276.435795 15.091357\n",
            "EPOCH=167 73.176907 309.889842 157.024008 10.377063 171.524623 10.985081 232.688005    13.372640 293.735249 15.640457\n",
            "EPOCH=168 97.764729 187.552153 169.731172 10.933051 164.305516 10.807584 141.528060    9.989527 137.878145 9.932534\n",
            "EPOCH=169 94.661613 639.351299 274.574014 14.224163 291.623108 14.889565 309.560794    15.821440 293.423977 15.511570\n",
            "EPOCH=170 141.418012 294.199661 82.184109 7.644529 76.481527 7.377166 73.914516    7.332388 94.040475 8.188409\n",
            "EPOCH=171 154.561389 962.154545 438.600469 18.736765 423.588719 18.374129 363.573078    16.885207 356.824832 16.590994\n",
            "EPOCH=172 243.053745 726.569816 81.057097 7.800859 76.741004 7.566307 51.300164    6.142200 46.614898 5.579876\n",
            "EPOCH=173 137.954678 816.097424 313.855992 15.746829 318.565002 15.963712 379.054491    17.953058 418.207499 19.151897\n",
            "EPOCH=174 92.959156 1620.429315 90.018488 7.624784 79.851504 7.204579 63.469693    6.523368 62.504424 6.442497\n",
            "EPOCH=175 117.583589 659.046596 259.602676 14.180818 263.916676 14.381060 331.983517    16.802303 370.972407 18.098180\n",
            "EPOCH=176 147.760799 198.949963 119.530329 9.242163 129.800791 9.563086 135.014292    9.704228 141.362694 9.977888\n",
            "EPOCH=177 98.109168 180.591014 98.543412 8.532126 97.612448 8.481887 110.189902    8.789951 138.084436 9.910772\n",
            "EPOCH=178 98.309180 793.623636 328.472940 16.274358 338.043962 16.624496 374.347080    17.909819 395.133381 18.608625\n",
            "EPOCH=179 73.593278 255.356104 98.880099 8.516888 100.630085 8.561473 110.881697    8.788433 123.617814 9.228969\n",
            "EPOCH=180 101.111221 2521.738095 426.190802 18.856610 462.820507 19.869251 647.229908    24.296453 767.686875 26.744650\n",
            "EPOCH=181 113.082983 395.790760 125.007630 9.472371 121.752180 9.330871 86.683759    7.868899 67.025439 6.949514\n",
            "EPOCH=182 88.921452 1535.483212 331.582581 16.145828 347.917823 16.667684 413.952048    18.728641 488.317803 20.770048\n",
            "EPOCH=183 124.827692 1314.588573 80.079221 7.341840 77.408108 7.219162 73.809523    6.947278 76.925602 7.154077\n",
            "EPOCH=184 198.849053 322.360142 216.729975 12.788635 209.750238 12.545649 195.892113    12.119801 208.503119 12.748058\n",
            "EPOCH=185 143.613580 855.325459 103.390540 8.213409 90.951152 7.789739 81.146321    7.540653 73.639919 7.092870\n",
            "EPOCH=186 146.361023 690.245164 193.187618 11.170703 211.369636 11.695134 342.842070    16.275916 498.106091 20.614374\n",
            "EPOCH=187 121.558395 224.829900 211.207988 12.389890 208.087913 12.240686 190.487121    11.684484 190.762171 11.844368\n",
            "EPOCH=188 72.740699 165.536555 100.161426 8.723567 105.175796 8.874519 135.724915    9.835296 162.000256 10.805520\n",
            "EPOCH=189 71.202417 458.035862 105.334774 8.837349 95.655246 8.441529 73.241560    7.385913 63.575254 6.834778\n",
            "EPOCH=190 106.452324 1767.507999 308.463414 15.233535 321.418837 15.642257 418.006398    18.591732 503.944250 20.968197\n",
            "EPOCH=191 105.434863 525.938252 99.343231 8.126359 88.910385 7.743487 66.157110    6.696153 48.598936 5.718335\n",
            "EPOCH=192 135.985847 258.153065 86.025639 7.963147 79.853071 7.667899 85.747370    7.811709 92.490115 8.050031\n",
            "EPOCH=193 97.252684 166.230570 181.183065 11.242952 168.370068 10.804359 137.916835    9.785122 110.483195 8.788551\n",
            "EPOCH=194 80.552564 253.685462 102.294803 8.631002 100.662848 8.571797 94.515303    8.247101 93.479908 8.121321\n",
            "EPOCH=195 70.568705 342.489676 224.367602 12.766391 222.946792 12.751471 254.576963    13.966322 288.261503 15.228876\n",
            "EPOCH=196 83.780244 410.362692 80.346339 7.468694 69.953020 6.989486 54.595379    6.251805 46.548281 5.803668\n",
            "EPOCH=197 102.033809 1568.761688 453.675948 19.311682 420.252181 18.523217 508.346767    20.970268 562.665747 22.422212\n",
            "EPOCH=198 140.891387 125.000231 142.967341 10.116805 126.359780 9.522570 144.678190    10.193245 154.099900 10.628558\n",
            "EPOCH=199 82.774718 720.095099 76.911672 7.509367 73.297305 7.337712 59.840645    6.605510 55.094482 6.256480\n",
            "EPOCH=200 116.378543 750.706086 73.904453 7.209437 70.624459 6.905523 63.302848    6.485291 68.914968 6.664651\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TJPM0V9n7LcE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "734461a2-b861-4c56-9c7a-bc91e9fde629"
      },
      "source": [
        "# GRU\n",
        "class Model(nn.Module):\n",
        "  def __init__(self, hidden_size=256, output_size=10):\n",
        "    super(Model, self).__init__()\n",
        "    self.hidden_size = hidden_size\n",
        "    self.output_size = output_size\n",
        "    self.avg_pool = nn.AvgPool1d(kernel_size=7)\n",
        "    self.batch_norm = nn.BatchNorm1d(224*224)\n",
        "    self.gru = nn.GRU(1568, 256, batch_first=True, bidirectional=False, num_layers=1, dropout=0.6)\n",
        "    self.out = nn.Linear(256, 10)\n",
        "\n",
        "  def forward(self, input, hidden):\n",
        "    input = input.permute(0, 2, 1)\n",
        "    input = self.avg_pool(input)\n",
        "    input = input.squeeze(2)\n",
        "    input = self.batch_norm(input)\n",
        "\n",
        "    input = input.view(-1, 32, 1568)\n",
        "    gru_out, _ = self.gru(input)\n",
        "    output = gru_out[:, -1, :]\n",
        "    output = self.out(output)\n",
        "    return output, 1\n",
        "  def initHidden(self):\n",
        "    return torch.zeros(16, self.output_size, self.hidden_size*2, device=device)\n",
        "\n",
        "model = Model()\n",
        "train(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:60: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.6 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EPOCH=1 127047.513999 122668.986111 10477.915427 98.324017 10773.074950 99.847867 10900.845350    100.522431 11381.326190 102.825484\n",
            "EPOCH=2 106924.680776 103948.414683 8738.102703 89.030127 9002.901166 90.543148 9117.666815    91.216805 9552.349479 93.504575\n",
            "EPOCH=3 90138.232474 88128.505952 7280.927734 80.430349 7517.003100 81.926807 7618.846664    82.592669 8009.733755 84.855125\n",
            "EPOCH=4 75982.083003 74474.885913 6034.379967 72.300732 6242.994271 73.742457 6332.687004    74.399959 6681.835231 76.632003\n",
            "EPOCH=5 63847.017857 63106.929563 5008.162023 64.920283 5190.726606 66.271799 5268.944869    66.870462 5578.276345 69.057558\n",
            "EPOCH=6 53719.895282 53281.650794 4131.963399 57.926974 4289.625744 59.191021 4356.222281    59.656684 4627.141391 61.788299\n",
            "EPOCH=7 45176.754960 45086.001984 3410.534858 51.512183 3545.166474 52.682902 3601.726279    52.974444 3837.247914 55.026281\n",
            "EPOCH=8 38085.868965 38328.414683 2825.964102 45.689858 2939.215313 46.761759 2985.880018    46.884620 3188.101335 48.772394\n",
            "EPOCH=9 32262.130153 32777.609623 2354.264019 40.405926 2448.121012 41.377746 2486.025315    41.344807 2658.149061 43.005575\n",
            "EPOCH=10 27550.286927 28194.220486 1972.416669 35.568781 2048.889064 36.441134 2079.070799    36.264551 2224.066875 37.692297\n",
            "EPOCH=11 23738.435557 24531.415179 1675.002168 31.291771 1736.075709 32.058367 1759.257638    31.751801 1880.044874 32.950553\n",
            "EPOCH=12 20729.177207 21564.275050 1441.495097 27.501181 1489.258376 28.161045 1506.366123    27.743297 1606.040804 28.725163\n",
            "EPOCH=13 18307.329572 18947.520585 1252.774274 25.451002 1294.442604 26.056081 1310.278157    25.650511 1398.048120 26.616343\n",
            "EPOCH=14 16182.601948 16734.750744 1096.749940 23.064114 1131.325555 23.565362 1145.016421    23.191191 1220.606855 24.027032\n",
            "EPOCH=15 14428.228154 14818.921131 959.645719 20.738855 985.639297 21.008353 998.380750    20.891012 1059.620879 21.442059\n",
            "EPOCH=16 12863.052070 13069.964286 849.376046 19.817116 874.216273 20.198845 884.560957    19.885559 940.236593 20.545681\n",
            "EPOCH=17 11602.933518 11690.938740 764.158610 18.461197 784.975454 18.774356 798.108546    18.559387 847.855803 19.212054\n",
            "EPOCH=18 10610.139185 10443.624256 678.483871 16.890697 692.741896 17.037072 701.179598    17.001352 742.053723 17.447719\n",
            "EPOCH=19 9439.151600 9355.906498 606.416105 16.353429 618.383787 16.413502 631.103500    16.560760 667.240712 17.093095\n",
            "EPOCH=20 8576.911479 8274.675595 553.137637 16.088364 565.187371 16.231444 576.283560    16.442790 609.431631 16.941438\n",
            "EPOCH=21 7959.285666 7483.871032 508.397012 15.795772 515.206412 15.818554 523.813096    15.933024 550.710709 16.354384\n",
            "EPOCH=22 7163.920910 6949.452846 460.492158 14.842548 469.304784 14.972610 473.943847    14.993771 497.113683 15.399899\n",
            "EPOCH=23 6673.697710 6272.292907 423.611110 14.696774 429.770613 14.774722 435.306812    14.829453 456.696152 15.268102\n",
            "EPOCH=24 6224.851848 5663.693576 392.187948 14.258795 397.142861 14.342645 399.718487    14.356184 416.621256 14.667822\n",
            "EPOCH=25 5867.809393 5428.901228 385.048019 15.175419 390.346846 15.270043 393.240390    15.283384 408.394960 15.584351\n",
            "EPOCH=26 5303.427662 4631.897135 367.254651 15.234019 368.014372 15.207066 373.225128    15.346423 386.569396 15.620350\n",
            "EPOCH=27 5089.926501 4340.588046 338.216403 14.548306 342.109832 14.644307 344.346437    14.648620 356.247220 14.948553\n",
            "EPOCH=28 4897.481543 4314.866319 306.538577 13.365194 308.736110 13.419965 308.674161    13.383163 317.294459 13.601773\n",
            "EPOCH=29 4696.172405 3589.570157 302.687017 13.937406 300.521032 13.891687 305.032375    13.994131 314.439967 14.251154\n",
            "EPOCH=30 4370.532945 3526.693700 283.572558 13.609717 282.810157 13.585202 283.845134    13.571341 290.665783 13.770339\n",
            "EPOCH=31 4138.352227 3108.030382 277.877943 13.881413 276.634217 13.851902 281.010904    14.018976 287.759878 14.260905\n",
            "EPOCH=32 3824.467649 2902.361979 273.581772 13.945499 271.032258 13.899651 274.122949    13.961564 279.851466 14.133179\n",
            "EPOCH=33 3763.909512 2706.984406 272.974944 14.019515 270.396626 13.942449 273.578116    13.994074 279.794365 14.267142\n",
            "EPOCH=34 3673.808236 2659.277530 243.517090 13.155831 240.950929 13.103050 239.815682    13.043330 242.724665 13.129140\n",
            "EPOCH=35 3548.197941 2456.174913 238.922692 13.079863 235.454465 13.003859 235.149930    12.981329 238.288300 13.087774\n",
            "EPOCH=36 3352.953072 2348.536086 243.993948 13.383561 240.633044 13.295724 243.173557    13.385940 247.233002 13.559085\n",
            "EPOCH=37 3248.167848 2053.887393 230.343902 12.991831 225.508194 12.861219 227.838383    12.999432 233.755642 13.228803\n",
            "EPOCH=38 2997.770788 2153.661768 229.310453 13.241454 225.239717 13.139031 223.772949    13.081342 224.631913 13.118020\n",
            "EPOCH=39 3032.214496 1800.749403 206.461636 12.378127 203.102923 12.303572 203.128778    12.343403 205.188463 12.458764\n",
            "EPOCH=40 2774.743106 1783.154467 201.899651 12.248862 199.202582 12.205297 198.214513    12.216239 200.900293 12.378530\n",
            "EPOCH=41 2708.025658 1653.214751 216.886316 12.900377 213.531223 12.803130 215.742203    12.878848 218.736450 13.030082\n",
            "EPOCH=42 2797.298132 1744.865327 192.535955 12.062312 187.618929 11.912410 188.315671    11.960816 190.057129 12.091848\n",
            "EPOCH=43 2547.604565 1546.609670 232.053350 13.518311 227.066527 13.367513 228.712703    13.456033 230.868593 13.528393\n",
            "EPOCH=44 2460.390789 1547.248620 223.773147 13.344419 219.584964 13.236504 221.186473    13.296095 222.057163 13.372954\n",
            "EPOCH=45 2554.795449 1447.852818 197.861546 12.597229 190.576528 12.380192 191.050730    12.360977 191.484333 12.404092\n",
            "EPOCH=46 2811.102258 1480.324761 217.275566 13.255308 211.455592 13.084260 211.567983    13.009783 212.511465 13.110910\n",
            "EPOCH=47 2500.840922 1377.971013 205.097660 12.807356 199.017856 12.658939 202.391366    12.809943 205.953977 12.950517\n",
            "EPOCH=48 2601.264952 1307.420185 188.371716 12.133939 184.187531 12.035085 186.348454    12.141565 188.053606 12.250587\n",
            "EPOCH=49 2248.528103 1152.597811 182.198817 12.064845 179.300241 11.970546 180.217795    12.054613 179.738651 12.063269\n",
            "EPOCH=50 2027.385413 1184.925456 180.409285 11.855782 177.625880 11.781555 178.182051    11.826757 178.374653 11.829673\n",
            "EPOCH=51 2289.809221 1241.861251 208.150271 12.915279 203.716891 12.796733 205.617483    12.847232 205.176319 12.777749\n",
            "EPOCH=52 2317.003102 1096.869885 169.197978 11.398354 164.788768 11.281927 163.835346    11.285532 165.696393 11.395535\n",
            "EPOCH=53 2198.280143 928.147267 180.849595 11.809224 176.431262 11.697127 178.654925    11.728751 182.168585 11.841799\n",
            "EPOCH=54 2187.454804 1205.743707 180.062627 11.766682 175.767235 11.656728 172.395337    11.542057 172.098961 11.597386\n",
            "EPOCH=55 2038.643605 995.013606 192.290274 12.333481 187.557184 12.206845 191.272089    12.284734 195.500806 12.434186\n",
            "EPOCH=56 2184.436765 1136.145678 184.043935 12.182863 181.772232 12.120378 181.136914    12.137675 182.346463 12.227728\n",
            "EPOCH=57 1979.476975 1100.699382 195.607831 12.569659 191.657897 12.451938 193.724891    12.480354 196.163601 12.540526\n",
            "EPOCH=58 2077.772715 939.915241 206.637474 12.730610 202.931290 12.616817 205.512885    12.652697 209.579325 12.788122\n",
            "EPOCH=59 1719.074030 945.206272 204.964148 12.924998 203.876078 12.896662 205.114888    12.891917 208.093190 13.000514\n",
            "EPOCH=60 2107.457775 1198.206938 203.524410 12.714699 199.806337 12.593478 204.661753    12.682880 208.995913 12.849634\n",
            "EPOCH=61 2187.873394 978.443189 191.116864 12.027526 185.827488 11.870739 183.448270    11.733937 185.735956 11.781289\n",
            "EPOCH=62 2140.976270 954.657529 182.914343 11.889656 180.025990 11.831753 181.534329    11.833629 185.318369 11.977765\n",
            "EPOCH=63 2387.380184 1130.484716 173.067063 11.550641 169.465504 11.471683 168.504976    11.355314 170.780079 11.368257\n",
            "EPOCH=64 1823.465307 1075.575753 195.814137 12.531145 194.345001 12.497603 195.320683    12.488811 200.059699 12.658220\n",
            "EPOCH=65 1832.571528 1113.152096 169.248695 11.581531 166.648395 11.525325 167.469140    11.510077 170.231492 11.618521\n",
            "EPOCH=66 1966.022395 1139.987320 175.267381 11.724263 171.211354 11.621218 172.148899    11.592423 173.512547 11.662515\n",
            "EPOCH=67 1941.259633 1080.759936 180.467457 11.911918 178.432660 11.870276 183.819868    12.028734 187.817201 12.207496\n",
            "EPOCH=68 2194.654442 1438.855252 229.219241 13.181207 227.903315 13.129402 227.418969    13.099872 228.705051 13.070793\n",
            "EPOCH=69 1837.658387 842.088371 181.840510 11.860790 179.282136 11.790778 184.453722    11.924330 189.434605 12.070504\n",
            "EPOCH=70 1892.895906 1060.531297 232.411733 13.748975 230.824999 13.702154 236.259098    13.746421 243.308855 13.959369\n",
            "EPOCH=71 1558.947907 764.902499 158.474948 11.159000 155.081182 11.044890 159.950442    11.183845 164.939836 11.370184\n",
            "EPOCH=72 1775.420218 1052.697266 200.588952 12.590896 197.909719 12.510793 201.399105    12.564263 207.885546 12.800305\n",
            "EPOCH=73 1754.267835 905.063252 146.592486 10.410275 144.000944 10.351495 142.076303    10.314208 144.614948 10.340185\n",
            "EPOCH=74 1781.231638 1126.877356 201.408096 12.610836 200.241241 12.582244 203.140036    12.583101 208.339228 12.750807\n",
            "EPOCH=75 1682.003100 1177.022197 222.372294 13.264417 220.315593 13.211584 222.061129    13.226305 226.408680 13.319286\n",
            "EPOCH=76 1682.543892 918.738405 163.675931 11.209884 160.978684 11.166102 160.814860    11.139709 163.205502 11.203310\n",
            "EPOCH=77 1747.230584 1075.805928 175.606228 11.666998 172.845060 11.627045 175.113678    11.679057 177.226199 11.739802\n",
            "EPOCH=78 1866.834221 796.876589 145.380354 10.467141 140.317299 10.338149 138.999796    10.246972 141.526789 10.238201\n",
            "EPOCH=79 1480.815082 751.057183 158.514399 11.061846 156.323730 11.004029 155.976914    10.967269 157.299855 10.982439\n",
            "EPOCH=80 1811.572790 1167.299448 155.928435 10.938021 153.246529 10.860523 153.224686    10.847985 154.434587 10.891155\n",
            "EPOCH=81 1492.897008 1047.687376 144.647045 10.412617 141.540369 10.320740 141.562594    10.290038 140.899541 10.213252\n",
            "EPOCH=82 1848.451717 1017.169069 205.030550 12.693245 200.352081 12.577964 203.444931    12.522624 203.978555 12.447026\n",
            "EPOCH=83 1438.613162 843.476888 184.430498 11.934997 180.794183 11.851252 182.966232    11.833849 184.062409 11.767107\n",
            "EPOCH=84 1390.703390 846.960550 160.015668 11.078536 156.450455 10.980665 157.378899    10.905543 156.614505 10.873041\n",
            "EPOCH=85 1270.831432 802.053176 146.800906 10.539538 144.098444 10.458689 146.715995    10.532718 148.117155 10.580044\n",
            "EPOCH=86 2109.879475 1123.172766 174.220333 11.598884 170.972597 11.513759 172.526350    11.496421 175.328016 11.505591\n",
            "EPOCH=87 1743.599992 837.468618 149.280547 10.621386 146.207809 10.532940 147.709371    10.557755 149.588785 10.624877\n",
            "EPOCH=88 1795.596501 964.215371 139.260968 10.304320 135.934396 10.209219 136.835714    10.224818 136.815269 10.212574\n",
            "EPOCH=89 1659.001601 930.299247 170.583968 11.402104 168.091117 11.338471 172.318347    11.423876 176.411500 11.545675\n",
            "EPOCH=90 1611.224506 1117.006851 142.236913 10.466633 139.447112 10.386141 138.579919    10.385152 138.230673 10.354587\n",
            "EPOCH=91 1430.855536 1098.805680 184.442738 11.706184 182.905031 11.665174 181.872515    11.646298 184.846227 11.684088\n",
            "EPOCH=92 1951.015227 1215.415101 185.402827 11.763771 182.552875 11.698355 180.558883    11.604747 182.916548 11.607651\n",
            "EPOCH=93 1786.113397 1125.816685 154.685257 10.758784 152.523091 10.715333 151.960387    10.685967 153.094540 10.730476\n",
            "EPOCH=94 1531.382642 1196.131774 185.002947 11.895961 181.887181 11.837136 181.792807    11.791542 181.744676 11.762031\n",
            "EPOCH=95 1454.004883 1468.741629 188.845254 12.153710 185.430598 12.068147 185.182135    11.957449 185.128370 11.861213\n",
            "EPOCH=96 1754.603412 1235.575102 218.585335 12.909798 215.840418 12.844737 218.646149    12.769636 221.924819 12.746404\n",
            "EPOCH=97 1696.104368 1122.018431 204.100160 12.491150 202.545489 12.447119 205.127944    12.409057 209.002424 12.474630\n",
            "EPOCH=98 1477.229675 1294.930571 192.315751 12.093294 188.994123 11.996755 190.735419    11.995916 194.463503 12.074658\n",
            "EPOCH=99 1486.845617 1036.365436 163.640263 11.076111 160.084456 10.960236 161.784482    11.032625 163.817375 11.135518\n",
            "EPOCH=100 1693.462987 962.272259 172.790382 11.505733 169.439302 11.386683 172.161321    11.358014 175.858748 11.511370\n",
            "EPOCH=101 1458.652974 1039.741381 218.315407 13.027673 216.625839 12.953825 223.468286    12.954591 232.276132 13.170749\n",
            "EPOCH=102 1312.253187 1107.506286 182.178086 11.674922 179.165004 11.582373 182.288711    11.524524 186.432053 11.650499\n",
            "EPOCH=103 1523.039645 1091.902065 186.784417 11.913465 184.016949 11.835850 186.001203    11.763737 189.207690 11.801156\n",
            "EPOCH=104 1552.359697 1316.904002 179.892107 11.735795 176.418316 11.639368 176.458166    11.527517 176.532896 11.494072\n",
            "EPOCH=105 1591.427294 1438.612661 173.453720 11.524633 170.274762 11.427412 167.818572    11.263637 166.730662 11.180690\n",
            "EPOCH=106 1397.904722 1408.834449 170.613457 11.453361 168.088898 11.392167 169.730952    11.433841 173.177422 11.515854\n",
            "EPOCH=107 1855.748818 1517.643694 162.527220 11.056440 158.051822 10.931657 158.611780    10.918072 158.610739 10.929361\n",
            "EPOCH=108 1352.273782 1131.751736 167.576970 11.221876 166.348248 11.202566 169.283256    11.244735 172.790898 11.358891\n",
            "EPOCH=109 1256.183152 928.939321 162.897911 11.073350 161.283009 11.021648 160.298808    10.887121 160.936894 10.900693\n",
            "EPOCH=110 1358.481703 1154.885029 174.718029 11.502356 174.040608 11.497568 177.467923    11.606118 180.692543 11.711154\n",
            "EPOCH=111 1424.165973 1517.869017 211.654997 12.749702 211.280137 12.738423 213.986121    12.799119 216.812965 12.883494\n",
            "EPOCH=112 1477.598708 1287.901274 209.701734 12.806429 210.814146 12.841638 215.369227    12.959957 218.514757 13.120924\n",
            "EPOCH=113 1314.803729 1432.824761 216.703063 12.801277 216.096493 12.810452 220.517198    12.869426 223.795130 12.896673\n",
            "EPOCH=114 1745.346628 1151.723586 157.247648 10.789138 154.819517 10.727222 149.996728    10.580979 148.574752 10.541919\n",
            "EPOCH=115 1360.014685 1318.848703 156.965762 10.624496 154.933251 10.546857 153.704115    10.463131 153.423459 10.512154\n",
            "EPOCH=116 1357.358164 1450.723052 234.024327 13.650695 234.607042 13.658010 240.071071    13.652353 244.059709 13.742245\n",
            "EPOCH=117 1250.022774 1392.876798 198.476221 12.276446 196.126555 12.243288 193.854112    12.203940 194.874565 12.119513\n",
            "EPOCH=118 1337.774902 1106.977508 169.878356 11.446460 167.796552 11.401349 167.747512    11.328154 168.094704 11.265592\n",
            "EPOCH=119 1235.539224 1186.018322 175.462333 11.629538 171.093450 11.494064 167.663453    11.402129 165.165182 11.289001\n",
            "EPOCH=120 1525.337324 1197.616513 154.999709 10.845734 150.800397 10.740932 147.459642    10.631684 146.357073 10.565514\n",
            "EPOCH=121 1610.458880 1068.741226 198.333468 12.313870 195.924227 12.215381 197.945276    12.205678 198.812339 12.230050\n",
            "EPOCH=122 1497.004301 1118.187252 188.229634 11.888906 187.906465 11.869011 191.347933    11.934167 194.099416 12.055173\n",
            "EPOCH=123 1561.424689 1192.525422 198.267846 12.405992 197.896113 12.423729 198.845491    12.341119 202.149436 12.428449\n",
            "EPOCH=124 1372.788456 1299.831132 217.264109 12.933962 217.981961 12.953321 221.246710    12.967457 223.696937 13.057016\n",
            "EPOCH=125 1329.688260 1098.059896 197.457377 12.316805 197.596333 12.306837 199.593999    12.211622 203.500141 12.252418\n",
            "EPOCH=126 1285.110048 1116.624411 199.285400 12.377103 197.516961 12.339199 199.851274    12.357892 203.836414 12.413196\n",
            "EPOCH=127 1466.526521 1119.592572 167.740932 11.421238 166.023292 11.400651 166.151626    11.355261 168.338302 11.363903\n",
            "EPOCH=128 1432.966070 1129.534397 173.315360 11.666830 171.079839 11.615922 173.099029    11.654083 173.996481 11.730322\n",
            "EPOCH=129 1348.352543 1251.503224 181.198523 11.880584 180.345745 11.831110 180.530509    11.742596 183.199259 11.805600\n",
            "EPOCH=130 1542.570715 1229.757533 195.782592 12.377499 195.618794 12.377987 196.375158    12.310188 200.664909 12.413740\n",
            "EPOCH=131 1105.092105 1363.374783 193.641085 12.408351 193.826658 12.402357 194.166418    12.289425 198.051311 12.419620\n",
            "EPOCH=132 1317.467064 1424.006650 200.055725 12.337409 199.669338 12.343048 198.731912    12.302738 201.805052 12.387318\n",
            "EPOCH=133 1591.756402 1132.056284 215.456441 12.752678 214.976366 12.747994 214.571992    12.624439 216.004977 12.585400\n",
            "EPOCH=134 1509.637862 1147.463681 201.202796 12.378314 199.816289 12.359763 199.348888    12.266495 201.635375 12.203653\n",
            "EPOCH=135 1423.583803 1099.442460 190.991941 12.075693 192.164766 12.125088 196.491247    12.140205 202.322943 12.308124\n",
            "EPOCH=136 1669.835678 1190.837054 157.385838 11.044179 157.421741 11.082960 156.827694    11.031595 157.047629 11.069120\n",
            "EPOCH=137 1201.111030 1125.077381 210.383806 12.642353 210.224621 12.648909 212.549998    12.633433 216.500897 12.729265\n",
            "EPOCH=138 1185.777607 1134.468564 227.323337 13.228047 229.530247 13.290014 233.380402    13.311580 239.754141 13.488053\n",
            "EPOCH=139 1141.643523 1154.392563 189.991372 12.135810 190.359296 12.173845 190.193210    12.082545 192.459213 12.034837\n",
            "EPOCH=140 1296.160750 964.049867 236.691111 13.629082 236.783601 13.633416 240.427712    13.640514 244.740201 13.669671\n",
            "EPOCH=141 1525.799980 882.952350 215.491741 13.052167 213.836695 13.051739 214.682538    12.936417 218.748526 12.931871\n",
            "EPOCH=142 1228.254351 1238.233274 164.948520 11.311793 161.111464 11.228735 156.240084    11.011330 154.408896 10.948399\n",
            "EPOCH=143 1375.406458 1298.569971 206.754025 12.668032 205.189156 12.626944 205.698615    12.630456 205.523927 12.601535\n",
            "EPOCH=144 1394.245265 878.079272 220.228041 12.907750 219.859061 12.887572 226.061325    12.966725 232.396024 13.156763\n",
            "EPOCH=145 1117.819171 1088.385370 210.059892 12.870269 208.426059 12.832604 210.901667    12.834645 213.929351 12.847948\n",
            "EPOCH=146 1233.625952 1023.245660 200.457845 12.459363 199.921938 12.440250 203.185645    12.447070 205.917702 12.550481\n",
            "EPOCH=147 1172.736216 1149.765749 185.632559 11.752794 183.237904 11.715347 183.553276    11.716782 185.608937 11.748240\n",
            "EPOCH=148 1256.908917 1078.978702 204.661393 12.435522 203.623604 12.399195 204.996823    12.430676 209.504971 12.522781\n",
            "EPOCH=149 1180.652439 1096.129278 241.311550 13.500561 239.728200 13.475160 243.234862    13.450284 249.828595 13.492330\n",
            "EPOCH=150 990.281847 994.449932 185.614219 11.813702 185.062886 11.792021 188.680404    11.764327 194.257623 11.892822\n",
            "EPOCH=151 1193.474823 1092.540117 240.757974 13.514381 239.605397 13.461954 244.652191    13.443390 251.895878 13.570179\n",
            "EPOCH=152 1413.455702 1294.682199 161.473755 10.806938 159.754225 10.772986 160.120528    10.783029 162.757869 10.833460\n",
            "EPOCH=153 1399.910266 1379.806393 215.632728 12.775437 214.498820 12.723902 217.726378    12.739013 221.753749 12.912245\n",
            "EPOCH=154 1253.624009 1109.147786 205.918630 12.565668 205.138141 12.542932 206.606484    12.527298 209.254619 12.611305\n",
            "EPOCH=155 1395.622406 1578.894555 229.801788 13.541783 228.457974 13.496213 230.119510    13.522012 232.438804 13.580755\n",
            "EPOCH=156 1014.547806 1635.443204 255.145967 14.134511 252.188425 14.033571 255.367015    14.034184 261.654646 14.103991\n",
            "EPOCH=157 945.490203 1099.680571 192.112559 12.281483 190.157519 12.216296 190.248732    12.114942 192.268148 12.055920\n",
            "EPOCH=158 1397.928366 1676.203141 230.740170 13.504094 227.838242 13.431814 227.863559    13.360783 230.619583 13.335192\n",
            "EPOCH=159 1520.616660 1333.834930 216.437668 12.849600 214.522870 12.798672 213.573781    12.715451 214.528815 12.632474\n",
            "EPOCH=160 1256.476221 1368.975090 235.199087 13.712454 233.523116 13.658493 232.840871    13.523956 235.092499 13.502972\n",
            "EPOCH=161 1253.479589 1112.128147 199.229729 12.277632 198.074911 12.277343 198.077188    12.211036 200.871214 12.174978\n",
            "EPOCH=162 1460.925661 1176.724826 225.540816 13.037164 223.236673 12.958944 222.595310    12.824942 222.681964 12.709783\n",
            "EPOCH=163 1119.719044 1324.573366 190.557265 12.053315 189.594093 12.034164 188.524984    11.963504 189.298245 11.951913\n",
            "EPOCH=164 1256.232087 996.811089 179.826388 11.690432 178.698288 11.684627 176.006715    11.548735 176.835779 11.577656\n",
            "EPOCH=165 1093.182580 1102.682385 177.813410 11.736420 177.025037 11.724896 176.120826    11.653709 178.885790 11.787055\n",
            "EPOCH=166 1262.019660 1138.335736 228.257122 13.202822 227.173735 13.146668 224.173158    12.994815 226.620233 13.019607\n",
            "EPOCH=167 1041.947359 1086.308160 184.563705 11.745717 183.023552 11.737568 179.787394    11.615515 181.805321 11.629177\n",
            "EPOCH=168 1048.520282 1227.653708 206.980953 12.550536 204.541058 12.474801 197.951243    12.198416 195.874403 12.107501\n",
            "EPOCH=169 1035.102582 1123.163737 183.297996 11.701272 180.630948 11.605573 174.540735    11.303216 174.785100 11.205170\n",
            "EPOCH=170 1249.136140 1207.779948 204.972336 12.344273 203.647369 12.317074 202.204689    12.221279 206.976295 12.298923\n",
            "EPOCH=171 993.137786 995.322746 215.289935 12.968242 215.597895 13.017079 215.780112    12.919444 219.516345 12.956446\n",
            "EPOCH=172 1050.598967 1103.713898 203.237168 12.547039 202.501924 12.545738 200.894981    12.366829 201.723296 12.417391\n",
            "EPOCH=173 1107.115416 1127.698079 211.049082 12.820910 210.650075 12.817161 206.752885    12.590443 205.532324 12.515374\n",
            "EPOCH=174 1307.690612 1061.927765 175.383422 11.488626 173.820715 11.439047 166.946261    11.156344 163.096966 10.946339\n",
            "EPOCH=175 1330.043751 1267.641338 193.744480 12.122530 191.925742 12.055541 186.656576    11.761960 185.747579 11.687877\n",
            "EPOCH=176 922.122445 1207.459752 180.356164 11.861916 179.181918 11.801420 174.505864    11.543582 174.126708 11.487017\n",
            "EPOCH=177 1066.334734 1340.668000 201.084737 12.222005 201.108839 12.222699 196.392125    12.053092 198.497636 12.046153\n",
            "EPOCH=178 1156.015761 1292.373310 235.744069 13.338672 235.457402 13.303071 229.837041    13.047800 232.259241 13.091048\n",
            "EPOCH=179 1099.540147 1176.024693 186.377649 11.879479 185.460668 11.841931 179.860388    11.617230 178.177918 11.494161\n",
            "EPOCH=180 1246.712175 1486.154979 192.956048 12.058075 193.101981 12.056034 189.508338    11.838362 192.612097 11.836395\n",
            "EPOCH=181 1177.899560 1295.227493 202.484174 12.432682 201.187193 12.378799 196.788027    12.182945 197.511262 12.190848\n",
            "EPOCH=182 1191.078677 1262.206210 185.565581 11.779105 183.389675 11.733228 178.023568    11.606343 177.337369 11.570429\n",
            "EPOCH=183 1253.453732 1093.077939 188.777223 11.921070 187.507608 11.898350 185.652123    11.734763 188.450278 11.739033\n",
            "EPOCH=184 1198.949258 1194.883030 168.343017 11.135363 166.336099 11.060514 160.751299    10.847748 161.311346 10.794015\n",
            "EPOCH=185 1277.553816 1194.385169 195.788845 12.352706 195.035909 12.329981 190.738792    12.103386 190.578625 12.087550\n",
            "EPOCH=186 1106.142683 936.827071 178.473349 11.605035 178.200960 11.587413 177.042601    11.480456 179.467591 11.574032\n",
            "EPOCH=187 1033.645671 1156.377449 194.685164 12.184474 193.460024 12.162430 188.705955    11.933383 189.282947 11.822302\n",
            "EPOCH=188 1392.795125 1414.528274 254.935406 14.091423 257.124713 14.160144 255.886148    14.018298 260.696656 14.015824\n",
            "EPOCH=189 1134.280274 1255.834325 189.169604 11.862986 188.982478 11.843201 185.498611    11.619846 186.429433 11.616387\n",
            "EPOCH=190 1232.629243 1164.544069 159.951923 11.021172 159.814040 11.036184 156.719562    10.865567 157.776374 10.908274\n",
            "EPOCH=191 1068.672791 1187.852214 198.521181 12.329665 199.002031 12.351406 198.672134    12.256735 199.994672 12.322060\n",
            "EPOCH=192 1196.012048 1145.570607 168.959400 11.195797 168.250020 11.153526 166.747385    11.039015 169.236435 11.154996\n",
            "EPOCH=193 1061.213692 1283.242257 233.870211 13.235157 233.322518 13.235647 234.293908    13.168701 238.837314 13.305764\n",
            "EPOCH=194 1045.156650 1310.792752 185.153792 11.857671 182.533279 11.789020 175.226958    11.526389 171.515708 11.392190\n",
            "EPOCH=195 1414.808304 1298.421565 182.097603 11.812705 178.689097 11.709991 170.906615    11.424130 165.753752 11.237077\n",
            "EPOCH=196 1091.379029 1402.684183 205.875587 12.407005 201.741299 12.270970 193.628748    11.972658 187.684069 11.791842\n",
            "EPOCH=197 968.174498 1265.266865 229.123232 13.261843 226.705983 13.190911 225.066317    13.073582 226.070977 13.037036\n",
            "EPOCH=198 1202.692565 1193.198560 225.318529 13.201392 220.722463 13.091951 213.822071    12.779107 209.384014 12.556263\n",
            "EPOCH=199 902.774345 1223.948506 227.559950 13.317863 224.878296 13.244610 220.787534    13.018502 217.644990 12.886038\n",
            "EPOCH=200 1287.188655 1265.266834 197.616349 12.179524 194.553508 12.113812 184.127376    11.718018 180.056186 11.504483\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-aO28mTm7sfr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "56a51d3c-20b5-492a-a291-656b6735e495"
      },
      "source": [
        "# Bi-GRU\n",
        "class Model(nn.Module):\n",
        "  def __init__(self, hidden_size=256, output_size=10):\n",
        "    super(Model, self).__init__()\n",
        "    self.hidden_size = hidden_size\n",
        "    self.output_size = output_size\n",
        "    self.avg_pool = nn.AvgPool1d(kernel_size=7)\n",
        "    self.batch_norm = nn.BatchNorm1d(224*224)\n",
        "    self.gru = nn.GRU(1568, 256, batch_first=True, bidirectional=True, num_layers=1, dropout=0.6)\n",
        "    self.out = nn.Linear(256*2, 10)\n",
        "\n",
        "  def forward(self, input, hidden):\n",
        "    input = input.permute(0, 2, 1)\n",
        "    input = self.avg_pool(input)\n",
        "    input = input.squeeze(2)\n",
        "    input = self.batch_norm(input)\n",
        "\n",
        "    input = input.view(-1, 32, 1568)\n",
        "    gru_out, _ = self.gru(input)\n",
        "    output = gru_out[:, -1, :]\n",
        "    output = self.out(output)\n",
        "    return output, 1\n",
        "  def initHidden(self):\n",
        "    return torch.zeros(16, self.output_size, self.hidden_size*2, device=device)\n",
        "\n",
        "model = Model()\n",
        "train(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:60: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.6 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EPOCH=1 122504.721340 111644.103175 9544.843378 93.722100 9723.277183 94.716909 10053.854241    96.461082 10222.794916 97.289400\n",
            "EPOCH=2 92443.986883 83694.150794 7002.107174 79.059605 7145.848332 80.014491 7426.470536    81.774751 7562.816257 82.538653\n",
            "EPOCH=3 68637.865079 61857.728175 5001.406424 65.163847 5111.238641 65.979856 5336.092721    67.678382 5448.249206 68.427137\n",
            "EPOCH=4 50056.335207 45537.315476 3512.865335 52.543936 3594.261732 53.211395 3763.457791    54.508363 3849.633042 55.167035\n",
            "EPOCH=5 36398.791171 33833.308036 2475.311268 41.403080 2529.926239 41.864524 2647.398401    42.684909 2710.855954 42.990899\n",
            "EPOCH=6 26645.731233 25837.251984 1810.971778 32.221114 1843.713382 32.521314 1915.434658    32.912113 1960.564396 33.006952\n",
            "EPOCH=7 20664.406278 21103.552083 1446.279219 28.240898 1465.543856 28.429943 1504.599434    28.757518 1538.372194 29.102451\n",
            "EPOCH=8 16668.152585 16842.432788 1136.280143 23.254536 1152.349885 23.457417 1182.025812    23.670228 1208.182217 23.886253\n",
            "EPOCH=9 13769.777254 13991.689732 926.839926 20.335163 939.121007 20.555702 957.343578    20.708821 976.211912 20.868046\n",
            "EPOCH=10 11154.245698 11354.927083 763.738534 19.259597 775.025976 19.489789 793.352218    19.703558 810.709002 19.964442\n",
            "EPOCH=11 9249.271667 9275.646577 602.039866 16.379689 610.615267 16.451968 624.616001    16.481769 638.916697 16.640492\n",
            "EPOCH=12 7803.495050 7788.448661 488.949049 15.204488 494.663182 15.179001 502.250321    15.129612 512.868713 15.241538\n",
            "EPOCH=13 6819.935323 6612.927579 425.639426 14.536851 429.599087 14.511120 434.251751    14.446407 442.865943 14.540545\n",
            "EPOCH=14 5876.516562 5573.356399 400.900606 15.461980 404.010496 15.431006 406.649688    15.343367 413.538084 15.445838\n",
            "EPOCH=15 5309.566847 4788.671193 344.288740 14.432139 346.077178 14.399434 346.582968    14.309396 350.093243 14.394996\n",
            "EPOCH=16 4674.494289 4425.623698 309.619326 13.765380 309.486647 13.713764 308.427078    13.528043 310.352863 13.540918\n",
            "EPOCH=17 4103.964467 3336.710627 301.433387 14.342992 300.681631 14.271666 305.458731    14.376197 307.990450 14.463954\n",
            "EPOCH=18 4094.123219 3389.617498 280.421878 14.005747 279.010986 13.915011 279.172315    13.855924 279.761675 13.846942\n",
            "EPOCH=19 3662.849544 2961.257750 258.387615 13.014584 255.482604 12.900978 255.106073    12.896984 256.468671 12.887387\n",
            "EPOCH=20 3268.215533 2397.614645 293.836479 14.880666 293.824328 14.919227 297.422803    15.051671 302.279381 15.164638\n",
            "EPOCH=21 3348.329699 2419.505487 235.546917 12.795552 233.796576 12.719263 230.616688    12.567855 231.452179 12.601442\n",
            "EPOCH=22 2792.920480 1878.497287 237.545788 13.372507 236.414131 13.347226 241.617426    13.522509 247.150569 13.761260\n",
            "EPOCH=23 3318.752652 2131.000930 196.732682 11.457673 193.158847 11.326966 191.130634    11.238963 189.974793 11.318789\n",
            "EPOCH=24 3248.266166 2042.780056 179.955761 10.775818 174.790239 10.592089 168.765662    10.437882 165.824702 10.279492\n",
            "EPOCH=25 3235.049272 1896.013331 232.848458 13.000191 229.730984 12.921221 226.674575    12.863092 230.894137 12.971608\n",
            "EPOCH=26 2633.632196 1629.966921 200.398257 12.207224 197.797089 12.135602 197.314428    12.132319 198.874884 12.263746\n",
            "EPOCH=27 2603.107422 1724.788814 201.081795 12.061894 197.388175 11.957213 196.123940    11.949997 197.442216 12.097384\n",
            "EPOCH=28 2366.556548 1701.175626 194.236865 11.715479 189.482130 11.562034 187.520147    11.577898 187.690974 11.669420\n",
            "EPOCH=29 2255.336724 1395.179874 163.007343 10.830179 158.157914 10.721226 153.704495    10.594130 155.592270 10.663350\n",
            "EPOCH=30 2224.952079 1426.061694 170.607243 11.295545 163.767865 11.047855 159.843339    10.911407 157.929961 10.813938\n",
            "EPOCH=31 2532.778286 1699.978175 168.358730 10.752671 163.544463 10.610656 157.721037    10.493757 157.535474 10.570175\n",
            "EPOCH=32 2359.832438 1457.893880 211.027249 12.795305 205.871035 12.662605 205.085578    12.607617 206.847009 12.699203\n",
            "EPOCH=33 2095.219086 1331.179129 194.620628 12.124916 189.288870 12.005865 185.178849    11.908200 189.120612 11.951763\n",
            "EPOCH=34 2674.659767 1500.388067 168.066588 11.217019 163.258364 11.077189 159.458113    10.929740 161.416534 10.991898\n",
            "EPOCH=35 2003.604525 1239.532940 197.713643 12.185785 193.524306 12.071806 190.653724    11.940211 194.097857 12.035459\n",
            "EPOCH=36 1837.245889 1140.312453 179.609474 11.929209 177.852598 11.900919 174.930943    11.793162 178.422628 11.874530\n",
            "EPOCH=37 2062.592286 970.287156 204.417082 12.661660 204.262999 12.687100 203.209590    12.642103 211.638307 12.849794\n",
            "EPOCH=38 1656.181907 1048.626984 224.364549 13.477092 221.856520 13.407799 221.009084    13.309689 226.221077 13.458688\n",
            "EPOCH=39 1851.389924 1053.620303 170.387104 11.540999 166.107030 11.405713 162.537163    11.372861 163.389446 11.365067\n",
            "EPOCH=40 2221.947440 1305.813058 152.063387 10.714797 148.652275 10.630329 142.200120    10.383239 143.609292 10.405699\n",
            "EPOCH=41 2014.261014 1236.649926 140.714856 10.337685 136.649911 10.233764 132.743740    10.050258 132.903617 10.042964\n",
            "EPOCH=42 1747.786482 1318.370133 182.488590 11.850547 178.949267 11.752790 173.696122    11.482429 174.282764 11.431138\n",
            "EPOCH=43 2135.725796 1192.002914 210.224130 12.862439 207.128033 12.782243 203.756422    12.626751 207.042171 12.742751\n",
            "EPOCH=44 2038.013209 1183.473943 172.404597 11.539182 169.247505 11.462317 167.150209    11.325334 170.027339 11.418913\n",
            "EPOCH=45 1934.576776 1039.710240 220.174997 13.184043 217.429547 13.156687 219.802714    13.166960 226.914275 13.333286\n",
            "EPOCH=46 1921.528090 962.965340 150.973177 10.540288 147.541802 10.469017 144.366131    10.323151 145.604553 10.322536\n",
            "EPOCH=47 1719.052330 907.793147 185.308870 11.952275 181.912924 11.868421 184.554079    11.917574 187.057216 11.998433\n",
            "EPOCH=48 1880.563289 985.450846 175.300471 11.680276 170.514698 11.552787 168.735813    11.537883 171.395761 11.586335\n",
            "EPOCH=49 1775.246936 1057.187547 149.490115 10.761880 145.051161 10.610877 143.941554    10.534389 145.512082 10.560132\n",
            "EPOCH=50 1846.746235 1136.959573 163.327983 11.167922 158.993458 11.029240 157.460452    10.932685 158.713639 10.948388\n",
            "EPOCH=51 2002.349456 1079.644485 226.698874 13.012922 224.009939 12.947744 222.488348    12.837751 226.557246 12.889285\n",
            "EPOCH=52 2051.757285 1027.952086 210.243205 12.651317 206.200214 12.531934 207.075426    12.455174 209.148283 12.462272\n",
            "EPOCH=53 2032.408893 1109.791450 218.061062 13.229238 215.594784 13.156685 211.821996    13.052841 209.111669 12.934130\n",
            "EPOCH=54 2125.734452 1253.430277 159.204952 11.088341 155.726290 10.990994 150.369449    10.857927 146.514601 10.745657\n",
            "EPOCH=55 2060.036065 1167.740165 154.266799 10.817097 149.434309 10.697786 139.804083    10.345565 134.172346 10.075466\n",
            "EPOCH=56 1698.677516 999.262618 174.408921 11.640258 172.006305 11.568837 168.411722    11.381413 169.567679 11.447342\n",
            "EPOCH=57 1517.580950 771.305618 178.956738 11.736958 176.237799 11.678980 173.231760    11.538172 176.678241 11.542158\n",
            "EPOCH=58 1848.433105 942.533498 196.117240 12.379643 193.356082 12.286019 193.108252    12.218249 197.316533 12.274598\n",
            "EPOCH=59 1748.907326 1157.557338 258.716155 14.314357 254.359894 14.168614 257.955004    14.249355 263.142689 14.354576\n",
            "EPOCH=60 1890.849322 902.510897 206.406666 12.777229 203.115938 12.683850 205.399147    12.645374 211.104605 12.774031\n",
            "EPOCH=61 1725.229205 1021.759068 231.449977 13.598173 227.723303 13.473034 229.363385    13.425208 233.300988 13.455183\n",
            "EPOCH=62 1864.215631 1063.321506 220.604860 13.148922 216.967552 13.032834 216.664424    12.938957 219.894163 12.915399\n",
            "EPOCH=63 1775.043036 1416.193948 301.422487 15.079494 301.027666 15.040444 302.520244    14.957624 308.731483 15.045179\n",
            "EPOCH=64 2078.786529 1189.247628 182.944505 11.862856 179.170827 11.730893 173.359918    11.423417 173.665994 11.298928\n",
            "EPOCH=65 1617.009449 1084.225028 231.234209 13.495544 229.987730 13.446052 227.131462    13.249953 232.619830 13.397481\n",
            "EPOCH=66 1604.899552 1388.777685 206.105236 12.844197 203.848652 12.776379 198.732476    12.582388 198.878778 12.562538\n",
            "EPOCH=67 1461.643956 1005.879867 223.560510 13.277196 222.534180 13.237811 222.099427    13.167994 224.089503 13.149755\n",
            "EPOCH=68 1923.293310 990.080884 170.676632 11.483884 168.221802 11.426261 163.859384    11.214435 165.174245 11.225196\n",
            "EPOCH=69 1462.771566 833.362568 220.802106 13.200995 219.214626 13.157383 217.971030    13.090275 221.970753 13.201699\n",
            "EPOCH=70 2105.598599 1002.978082 145.455901 10.446206 141.530897 10.332520 140.107761    10.276380 141.608620 10.360731\n",
            "EPOCH=71 1534.635792 933.801448 206.044168 12.582054 203.132288 12.500549 203.252262    12.377544 208.004844 12.502061\n",
            "EPOCH=72 1500.376951 996.624705 196.484820 12.463586 194.530906 12.420623 198.814489    12.502876 203.757478 12.698339\n",
            "EPOCH=73 1433.971995 924.469789 195.755976 12.509345 194.944719 12.489360 198.551165    12.524411 205.844873 12.722616\n",
            "EPOCH=74 1583.544097 1100.789915 163.321711 11.228384 161.220700 11.146119 160.385029    11.053524 163.315591 11.123081\n",
            "EPOCH=75 1555.699753 852.979686 193.112661 12.372970 191.132087 12.323210 194.167580    12.313057 199.721111 12.487738\n",
            "EPOCH=76 1609.326690 900.011207 210.643097 12.797430 208.362623 12.721536 211.570527    12.695247 218.783205 12.841919\n",
            "EPOCH=77 1512.868614 825.753379 206.596918 12.911116 203.626673 12.811743 204.108138    12.716747 209.173878 12.820836\n",
            "EPOCH=78 1466.197453 888.229864 194.141584 12.560313 191.236415 12.493236 189.785126    12.421487 192.821433 12.512599\n",
            "EPOCH=79 1707.846385 1054.288690 203.870539 12.774064 199.872979 12.624981 199.293147    12.415319 203.578165 12.504547\n",
            "EPOCH=80 1565.386689 825.607670 185.647232 12.144526 182.025279 12.000395 180.418146    11.765798 185.413686 11.918792\n",
            "EPOCH=81 1420.147718 885.129108 210.893652 12.892842 207.830513 12.801543 207.021009    12.629323 211.394263 12.741546\n",
            "EPOCH=82 1469.479329 967.879805 191.167555 12.087476 188.661892 12.045524 190.243389    12.021834 197.879417 12.242102\n",
            "EPOCH=83 1455.492980 1104.772631 189.637813 12.054664 186.567712 11.938896 184.696506    11.867769 188.040235 11.936981\n",
            "EPOCH=84 1434.413448 992.783854 239.013250 13.522148 236.616280 13.439983 239.074329    13.398521 245.871773 13.584620\n",
            "EPOCH=85 1478.976431 781.566274 227.339442 13.349088 224.702205 13.256406 228.970671    13.311299 235.034063 13.439600\n",
            "EPOCH=86 1620.465670 895.239614 210.154123 12.718073 206.271067 12.603572 201.626527    12.360437 202.327418 12.331100\n",
            "EPOCH=87 1538.737830 800.279142 176.126651 11.621592 173.511358 11.568263 171.473190    11.519352 174.068302 11.608607\n",
            "EPOCH=88 1623.120773 1217.221540 280.144093 14.878364 277.592383 14.812535 280.011062    14.758413 287.970966 14.930426\n",
            "EPOCH=89 1612.472179 902.467746 212.403466 12.864692 207.829470 12.760726 205.478969    12.609029 209.635400 12.656148\n",
            "EPOCH=90 1481.500251 1031.619048 247.946348 13.900491 246.045090 13.873639 245.259099    13.799585 252.341247 14.015312\n",
            "EPOCH=91 1578.225297 840.811647 192.102656 12.254488 189.392608 12.154465 187.373395    12.070929 192.650278 12.215459\n",
            "EPOCH=92 1606.208078 1044.487088 206.978122 12.477706 204.866934 12.426304 201.524396    12.274187 206.902670 12.461735\n",
            "EPOCH=93 1464.597925 907.030971 224.556746 13.147820 222.083507 13.078521 222.252234    12.999386 227.816309 13.162201\n",
            "EPOCH=94 1374.751806 897.072467 216.389087 12.830412 212.852004 12.752874 210.347072    12.540147 214.073957 12.585459\n",
            "EPOCH=95 1275.938217 1076.793372 237.738501 13.472649 235.338177 13.423543 236.100814    13.354682 242.512819 13.472831\n",
            "EPOCH=96 1418.806033 1216.393028 187.088860 11.777933 184.012039 11.688289 183.949495    11.609603 187.299343 11.623431\n",
            "EPOCH=97 1473.658048 1395.564469 229.216254 13.028725 225.232318 12.946231 225.900687    12.884508 228.531020 12.858161\n",
            "EPOCH=98 1477.593058 926.150747 183.095826 11.709888 180.255929 11.632689 180.745142    11.597603 182.741929 11.685639\n",
            "EPOCH=99 1344.673150 863.661117 180.861812 11.763866 177.665857 11.664659 175.889541    11.534281 176.661219 11.532333\n",
            "EPOCH=100 1413.393744 1200.980577 218.788214 13.078703 214.780613 12.946245 213.713344    12.773286 213.752789 12.768171\n",
            "EPOCH=101 1466.437249 1048.110197 165.444571 11.331777 161.842533 11.240970 158.349875    11.128877 157.449222 11.045689\n",
            "EPOCH=102 1326.064094 983.250202 204.895424 12.508275 201.641197 12.453261 197.913812    12.305969 196.852039 12.245193\n",
            "EPOCH=103 1455.274726 1069.688740 219.071858 13.124406 216.379782 13.055133 215.345224    12.925500 217.170421 12.930257\n",
            "EPOCH=104 1505.039086 1252.340138 236.990604 13.459733 236.653346 13.424190 237.280646    13.355498 242.429638 13.465628\n",
            "EPOCH=105 1562.321096 881.666512 191.745137 12.176674 189.544845 12.113410 185.376583    11.948118 183.730430 11.885386\n",
            "EPOCH=106 1150.459413 971.734096 201.159849 12.463519 200.688817 12.482342 199.946017    12.451523 203.471213 12.524685\n",
            "EPOCH=107 1231.105711 1225.722191 229.354441 13.506785 228.298206 13.497456 226.767091    13.356925 230.535190 13.427787\n",
            "EPOCH=108 1238.539687 886.494699 208.777703 12.710235 206.832222 12.679520 202.953333    12.557738 203.448449 12.569717\n",
            "EPOCH=109 1391.118249 1233.127868 265.162998 14.231399 261.887999 14.159196 260.947008    14.081231 265.583793 14.107970\n",
            "EPOCH=110 1401.500143 1250.611328 241.074678 13.750777 237.851210 13.664357 237.053129    13.484654 238.555029 13.484480\n",
            "EPOCH=111 1326.600177 1234.934725 206.406862 12.504605 205.326233 12.495158 204.355539    12.428502 207.392063 12.514052\n",
            "EPOCH=112 1637.667505 959.591983 214.129142 12.745579 213.830970 12.754313 217.333338    12.806352 224.009174 12.994350\n",
            "EPOCH=113 1026.081532 913.658149 174.979763 11.393637 172.495885 11.297239 172.016408    11.258891 174.541486 11.388686\n",
            "EPOCH=114 1318.159249 1133.869063 189.425991 12.125364 187.922908 12.057389 187.217404    11.896521 190.440073 12.034717\n",
            "EPOCH=115 1465.103652 1071.218099 234.280475 13.564421 232.710893 13.512732 230.198234    13.261600 235.582526 13.333447\n",
            "EPOCH=116 1170.707734 949.359142 193.557314 12.097781 192.415775 12.055564 188.127232    11.767205 193.406871 11.796033\n",
            "EPOCH=117 972.018274 1066.077210 227.174615 13.076794 224.726067 12.988521 217.637047    12.607580 219.925305 12.530530\n",
            "EPOCH=118 1420.003983 1130.513083 246.769592 14.119813 246.263907 14.115289 244.529392    13.977805 251.656379 14.068426\n",
            "EPOCH=119 1717.613731 1347.784257 202.649278 12.639859 200.121960 12.573411 192.433736    12.301248 193.033988 12.255094\n",
            "EPOCH=120 1597.895215 1269.342975 227.025585 13.275033 226.131798 13.261697 225.329239    13.187437 232.398775 13.353578\n",
            "EPOCH=121 1457.426450 1136.002558 207.162662 12.736462 206.753116 12.718797 200.562083    12.383962 205.775841 12.432648\n",
            "EPOCH=122 1203.411277 1000.951916 195.577394 12.085805 194.974628 12.060275 191.276314    11.790948 195.143800 11.908043\n",
            "EPOCH=123 1298.108085 1007.673627 201.407712 12.180622 199.231080 12.124697 196.395086    11.903659 200.064207 11.937304\n",
            "EPOCH=124 1553.052859 1328.804563 195.705774 12.346906 192.278460 12.271325 184.478562    12.011877 181.421065 11.930569\n",
            "EPOCH=125 1299.050239 1299.701637 248.385762 14.046089 244.980761 13.964705 239.281048    13.750021 238.673831 13.695938\n",
            "EPOCH=126 1299.265706 997.403677 222.284913 13.281170 219.304397 13.202836 211.253095    12.942380 209.927566 12.943752\n",
            "EPOCH=127 1395.966246 904.698095 223.354200 13.191051 220.864271 13.137336 214.114994    12.884063 214.732251 12.782454\n",
            "EPOCH=128 1316.080586 1164.296162 173.661958 11.430550 171.819785 11.402822 166.622633    11.266159 167.309367 11.245678\n",
            "EPOCH=129 1314.584789 1139.308074 203.991426 12.607147 203.252207 12.609849 199.503206    12.449904 202.472611 12.378696\n",
            "EPOCH=130 1209.886756 1173.570925 233.765759 13.443535 234.137549 13.451910 232.736208    13.321773 238.416499 13.385984\n",
            "EPOCH=131 1372.613503 1302.004480 199.720447 12.424145 197.948425 12.376032 196.084419    12.303705 200.374420 12.445746\n",
            "EPOCH=132 1232.321593 959.072018 203.337435 12.313300 200.968248 12.255734 199.759264    12.164839 204.419306 12.313543\n",
            "EPOCH=133 1123.653298 1076.142346 210.656538 12.713244 207.887920 12.610093 205.450402    12.350900 206.202465 12.279184\n",
            "EPOCH=134 1046.987642 1127.167147 206.298991 12.577545 204.167233 12.521795 204.933347    12.539243 206.882098 12.592820\n",
            "EPOCH=135 1210.846223 1287.852524 203.174225 12.578102 201.191934 12.508842 199.731855    12.387068 202.973136 12.390954\n",
            "EPOCH=136 1258.654207 928.429982 237.383720 13.517996 235.297596 13.448565 232.357823    13.246159 234.969156 13.316579\n",
            "EPOCH=137 1600.257179 1014.406467 188.026355 11.878588 184.219506 11.764856 172.809842    11.333084 169.965774 11.262724\n",
            "EPOCH=138 1205.761994 1438.414776 277.585961 14.647669 275.878213 14.610831 275.979907    14.525917 283.137178 14.750329\n",
            "EPOCH=139 1434.420470 1197.525794 184.353604 11.855151 181.492361 11.753883 176.842739    11.531986 176.330326 11.543481\n",
            "EPOCH=140 1057.298571 856.900484 230.869456 13.364544 228.825175 13.313001 226.647188    13.227817 230.194715 13.369431\n",
            "EPOCH=141 1182.264110 949.089495 205.939132 12.440572 203.537010 12.383732 202.422595    12.241344 206.209050 12.351966\n",
            "EPOCH=142 1208.216750 940.948304 199.692113 12.289270 197.431586 12.208393 194.531240    12.035500 197.329648 12.157097\n",
            "EPOCH=143 1319.206017 998.292291 205.334374 12.662913 202.415279 12.580282 197.763523    12.465936 198.970038 12.544463\n",
            "EPOCH=144 1217.635083 1125.633169 236.856923 13.655579 234.154897 13.585600 228.158481    13.343642 226.794420 13.341401\n",
            "EPOCH=145 1110.564301 939.612041 215.662165 13.038626 214.135542 13.020381 208.237337    12.838538 207.290909 12.815460\n",
            "EPOCH=146 1199.864729 1056.046084 209.509650 12.922668 206.798072 12.835802 198.880579    12.473264 194.855735 12.278616\n",
            "EPOCH=147 1403.598694 1109.395926 242.593428 13.874811 240.049193 13.800391 235.089777    13.555340 235.495353 13.437899\n",
            "EPOCH=148 1348.164695 930.033436 195.558386 12.236148 192.532828 12.182980 180.440667    11.769558 177.027626 11.562762\n",
            "EPOCH=149 1320.811327 1122.012959 214.761417 12.765533 213.341723 12.744492 210.303472    12.606528 213.100235 12.622584\n",
            "EPOCH=150 1722.555626 1419.982670 233.756000 13.419832 232.885655 13.385739 227.701193    13.115214 229.890853 13.034460\n",
            "EPOCH=151 1149.779448 1045.658110 200.511639 12.497365 198.848643 12.428265 194.622293    12.273703 193.096566 12.225319\n",
            "EPOCH=152 1216.551701 1116.337519 265.615376 14.383216 264.631246 14.344621 263.004857    14.177420 271.519165 14.388117\n",
            "EPOCH=153 1322.495414 1143.180757 202.693592 12.509411 200.418571 12.471211 194.287955    12.266560 195.394510 12.312446\n",
            "EPOCH=154 1166.160223 1007.095277 202.780693 12.458466 199.717141 12.365524 191.036591    12.017722 189.597678 11.933673\n",
            "EPOCH=155 1554.967501 1000.983197 208.380563 12.620134 205.796438 12.560628 196.919002    12.279134 195.821073 12.270010\n",
            "EPOCH=156 1428.875195 1211.461775 178.644098 11.690028 173.530494 11.528507 162.511282    11.097281 159.373195 10.948702\n",
            "EPOCH=157 1143.252665 888.442127 240.835517 13.834583 238.444061 13.760554 233.428683    13.552763 234.152839 13.586426\n",
            "EPOCH=158 1004.944930 896.841921 229.577394 13.453921 227.404721 13.398816 222.573047    13.173057 225.302777 13.249991\n",
            "EPOCH=159 1390.726358 1209.587209 236.155600 13.659810 233.305191 13.576459 230.276221    13.372819 236.123633 13.451079\n",
            "EPOCH=160 1140.643190 1249.982329 226.847968 13.187105 222.146000 13.047683 213.387378    12.637379 214.416370 12.425150\n",
            "EPOCH=161 1149.627615 1245.578978 251.607903 14.013212 248.479123 13.938245 241.426999    13.661817 244.240136 13.617303\n",
            "EPOCH=162 1137.609052 901.514865 211.942777 12.938719 207.851778 12.807842 197.015157    12.306540 196.539871 12.277902\n",
            "EPOCH=163 1333.204549 1060.230872 207.446145 12.870576 203.628327 12.750770 189.819970    12.251520 186.465005 12.151115\n",
            "EPOCH=164 1212.815473 1093.879449 220.135330 13.045900 215.615255 12.918363 203.933610    12.508834 202.048725 12.389539\n",
            "EPOCH=165 1054.275243 1209.274337 256.945009 14.484848 256.160494 14.443513 251.839377    14.188538 255.333169 14.296474\n",
            "EPOCH=166 1705.655160 1279.051804 221.502862 12.972678 217.889342 12.895909 213.503308    12.717431 216.833286 12.656055\n",
            "EPOCH=167 1327.354075 1046.087472 223.223588 13.037627 220.855958 12.957586 216.804244    12.731593 219.692441 12.747162\n",
            "EPOCH=168 1164.924524 1019.084310 210.963058 12.679622 206.504279 12.571878 198.886472    12.302743 198.510049 12.301844\n",
            "EPOCH=169 1767.504463 1234.707527 189.005135 12.024416 182.809644 11.841619 170.044623    11.398161 164.513588 11.190335\n",
            "EPOCH=170 1303.384017 1036.901274 183.259134 11.783948 178.573125 11.639953 173.672994    11.463532 174.440125 11.489517\n",
            "EPOCH=171 1126.115442 1127.489537 211.481551 13.004314 207.507039 12.896925 207.332588    12.785916 211.784641 12.836682\n",
            "EPOCH=172 1222.488279 1439.888346 288.895519 15.220187 286.740726 15.164508 287.595611    15.105504 295.031459 15.280551\n",
            "EPOCH=173 1376.163365 1057.383014 205.613158 12.542293 202.725193 12.439161 200.107497    12.223236 201.945308 12.198369\n",
            "EPOCH=174 1262.535979 1098.392144 293.277979 15.014612 291.219358 14.934517 292.102900    14.817669 296.998922 14.807771\n",
            "EPOCH=175 1013.725848 1058.393725 216.861198 13.081445 214.017542 12.986722 210.089005    12.827608 211.098794 12.826783\n",
            "EPOCH=176 1004.763317 1125.991722 275.555998 14.909519 273.818932 14.836174 273.488230    14.687762 277.706156 14.713064\n",
            "EPOCH=177 1093.576982 935.412667 219.019170 13.014541 217.206756 12.957879 212.454956    12.802093 212.666541 12.784194\n",
            "EPOCH=178 1137.577300 813.359088 230.243756 13.339590 227.354122 13.243743 221.688062    12.954568 222.868130 12.868438\n",
            "EPOCH=179 1050.251132 935.766950 226.538193 13.301295 223.267328 13.197973 217.140142    12.959162 216.565210 12.935665\n",
            "EPOCH=180 1085.904826 925.750946 237.243045 13.715499 232.560354 13.588482 224.569563    13.329204 221.553549 13.266638\n",
            "EPOCH=181 1185.102995 1142.463154 193.495830 12.111398 189.547586 12.012121 182.392397    11.783535 179.962880 11.662462\n",
            "EPOCH=182 1181.448122 981.695623 237.597973 13.563146 234.531570 13.469779 229.821556    13.248657 233.166156 13.231701\n",
            "EPOCH=183 993.654044 980.059462 240.602398 13.692305 237.519044 13.603310 231.396155    13.401035 234.219620 13.455257\n",
            "EPOCH=184 1232.300219 1254.774662 235.075924 13.412388 229.929499 13.270359 221.475545    12.968538 221.872198 12.907036\n",
            "EPOCH=185 1459.444750 1393.282738 259.341250 14.161395 256.487585 14.078873 253.493508    13.965306 258.794021 14.122467\n",
            "EPOCH=186 1281.217128 1283.729120 267.204553 14.375279 264.804961 14.300605 262.453473    14.126743 266.547195 14.114438\n",
            "EPOCH=187 1200.277551 1094.757750 266.267642 14.570862 263.426244 14.465072 259.381607    14.216964 262.585207 14.331487\n",
            "EPOCH=188 1591.472266 1235.902390 256.349178 14.334795 253.837825 14.258074 245.684905    13.981477 248.697096 14.113737\n",
            "EPOCH=189 1321.416576 1209.271159 209.716548 12.749960 205.935610 12.658943 198.027634    12.357062 198.533379 12.272413\n",
            "EPOCH=190 1330.519038 1066.407118 244.430945 13.894225 243.179862 13.889329 237.874278    13.691360 243.378227 13.743050\n",
            "EPOCH=191 1265.643467 1089.049557 248.177559 13.987911 248.033948 14.001303 245.893951    13.875243 250.896258 13.951119\n",
            "EPOCH=192 1292.733013 1067.431114 226.579004 13.354763 224.367953 13.298562 217.546554    13.028982 219.998898 13.033965\n",
            "EPOCH=193 1257.678849 1067.935330 248.422167 14.010157 244.689221 13.900084 243.445565    13.791409 249.913359 13.976805\n",
            "EPOCH=194 1161.705173 1043.050859 229.275309 13.345887 225.996412 13.237096 224.910903    13.113196 228.724596 13.142279\n",
            "EPOCH=195 1184.112202 1025.551502 234.504844 13.715191 231.791141 13.625047 231.584456    13.499306 235.881384 13.587389\n",
            "EPOCH=196 1119.238745 1546.786768 314.239408 15.898435 313.739421 15.899972 311.829343    15.784452 315.083528 15.860117\n",
            "EPOCH=197 1070.886311 925.164194 218.955501 12.915590 216.330485 12.840284 210.637097    12.557888 210.917852 12.538604\n",
            "EPOCH=198 992.501778 1132.237770 254.704336 14.277365 251.782497 14.204221 248.153703    14.029300 249.629161 14.033999\n",
            "EPOCH=199 1214.519855 1162.567445 335.448500 16.514326 335.278275 16.499415 337.324662    16.480578 345.362133 16.694662\n",
            "EPOCH=200 1242.775096 1172.567429 255.686467 14.375626 252.019674 14.276148 244.828023    14.041412 243.343674 14.025165\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qv5Bs70Q_ow9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "outputId": "25b350e2-3b72-4e71-b2b5-817b5579d7f2"
      },
      "source": [
        "# M1\n",
        "class Model(nn.Module):\n",
        "  def __init__(self, hidden_size=256, output_size=10):\n",
        "    super(Model, self).__init__()\n",
        "    # Encoder\n",
        "    self.hidden_size = hidden_size\n",
        "    self.output_size = output_size\n",
        "    self.encoder = nn.Sequential(\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7, 7*9, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 112*112\n",
        "\n",
        "          nn.Conv2d(7*9, 7*18, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 56*56\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*18, 7*36, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 28*28\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*36, 7*72, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 14*14\n",
        "\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7*72, 7*72, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 7*7\n",
        "          nn.BatchNorm2d(7*72),\n",
        "\n",
        "          nn.Flatten(),\n",
        "          nn.Linear(7*7*7*72, 256*10),\n",
        "        )\n",
        "    self.dropout = nn.Dropout(0.5)\n",
        "\n",
        "    self.gru = nn.GRU(self.hidden_size, self.hidden_size, bidirectional=True, batch_first=True)\n",
        "    self.out = nn.Linear(self.hidden_size*2, 1)\n",
        "\n",
        "  def forward(self, input, hidden):\n",
        "    input = input.view(-1, 7, 224, 224)\n",
        "    # Encoder\n",
        "    encoder_outputs = self.encoder(input)\n",
        "    encoder_outputs = self.dropout(encoder_outputs)\n",
        "    print\n",
        "    encoder_outputs = encoder_outputs.view(-1, 10, 256)\n",
        "    \n",
        "    hidden, _ = self.gru(encoder_outputs)\n",
        "    output = self.out(hidden)\n",
        "    output = output.squeeze(2)\n",
        "    return output, 1\n",
        "\n",
        "  def initHidden(self):\n",
        "    return torch.zeros(16, self.output_size, self.hidden_size*2, device=device)\n",
        "\n",
        "model = Model()\n",
        "train(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH=1 103627.627094 89095.571429 7455.358829 81.507042 7521.314298 81.952413 7781.390340    83.570301 8031.393849 84.982208\n",
            "EPOCH=2 70018.101687 61462.852183 4916.933625 64.225171 4967.350558 64.584343 5171.004111    66.133268 5371.763176 67.545051\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-32d491e13866>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-16-18c8f75791bd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, lamb, max_iter)\u001b[0m\n\u001b[1;32m     86\u001b[0m       \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m       \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m       \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m567\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0mval_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse_5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_5\u001b[0m\u001b[0;34m,\u001b[0m     \u001b[0mmse_9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_9\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriteria\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EgwZGnOrp8hP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bf9cc236-162a-4118-a93b-1d7b8d456a2f"
      },
      "source": [
        "# SMAML\n",
        "class Model(nn.Module):\n",
        "  def __init__(self, hidden_size=256, output_size=10):\n",
        "    super(Model, self).__init__()\n",
        "    # Encoder\n",
        "    self.hidden_size = hidden_size\n",
        "    self.output_size = output_size\n",
        "    self.encoder = nn.Sequential(\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7, 7*9, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 112*112\n",
        "\n",
        "          nn.Conv2d(7*9, 7*18, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 56*56\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*18, 7*36, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 28*28\n",
        "\n",
        "          # group convolution\n",
        "          nn.Conv2d(7*36, 7*72, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 14*14\n",
        "\n",
        "          # depthwise convolution\n",
        "          nn.Conv2d(7*72, 7*72, 3, groups=7),\n",
        "          nn.LeakyReLU(inplace=True),\n",
        "          nn.MaxPool2d((2, 2), padding=(1, 1)), # 7*7\n",
        "          nn.BatchNorm2d(7*72),\n",
        "        )\n",
        "    self.linear1 = nn.Linear(7*7*72, self.hidden_size)\n",
        "    self.batch_norm = nn.BatchNorm1d(self.hidden_size)\n",
        "\n",
        "    # Decoder\n",
        "    self.linear2 = nn.Linear(224*224, self.hidden_size*2)\n",
        "    self.dropout = nn.Dropout(0.8)\n",
        "\n",
        "    self.gru = nn.GRU(self.hidden_size, self.hidden_size, bidirectional=True, batch_first=True)\n",
        "    self.out = nn.Linear(self.hidden_size*2, 1)\n",
        "\n",
        "  def forward(self, input, hidden):\n",
        "    input = input.view(-1, 7, 224, 224)\n",
        "    # Encoder\n",
        "    encoder_outputs = self.encoder(input)\n",
        "    encoder_outputs = encoder_outputs.view(-1, 7, 7*7*72)\n",
        "    encoder_outputs = self.dropout(self.linear1(encoder_outputs))\n",
        "\n",
        "    # Decoder with attention\n",
        "    input = input.view(-1, 7, 224*224)\n",
        "    embed = torch.tanh(self.linear2(input))\n",
        "    embed = self.dropout(embed)\n",
        "\n",
        "    hidden = hidden[: len(embed)]\n",
        "    attn_weights = F.softmax(torch.bmm(hidden, embed.permute(0, 2, 1)), dim=2)\n",
        "    attn_applied = F.relu(torch.bmm(attn_weights, encoder_outputs))\n",
        "    attn_applied = self.batch_norm(attn_applied.permute(0, 2, 1))\n",
        "    attn_applied = attn_applied.permute(0, 2, 1)\n",
        "\n",
        "    hidden, _ = self.gru(attn_applied)\n",
        "    output = self.out(hidden)\n",
        "    output = output.squeeze(2)\n",
        "    return output, attn_weights\n",
        "\n",
        "  def initHidden(self):\n",
        "    return torch.randn(16, self.output_size, self.hidden_size*2, device=device)\n",
        "\n",
        "\n",
        "model = Model()\n",
        "train(model, lamb=0.1, max_iter=200)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH=1 118463.774802 94043.333333 9043.519370 91.062185 8056.841890 85.764545 7893.786694    84.488335 9745.336086 94.940172\n",
            "EPOCH=2 72697.528715 58860.495040 5608.949975 69.858468 4744.764661 63.420966 4684.373264    62.409243 6150.306250 73.681959\n",
            "EPOCH=3 46295.298969 38620.038690 3880.039140 56.951715 3053.175068 48.634580 2941.057448    46.629834 4283.381926 60.217296\n",
            "EPOCH=4 29705.876605 26284.920139 2722.508411 46.462668 1977.620784 36.967849 1804.495081    32.653568 3210.292361 51.750891\n",
            "EPOCH=5 20281.908771 19649.910218 1807.378550 34.175558 1284.360585 26.379551 1214.437168    24.479630 2341.745094 39.958682\n",
            "EPOCH=6 15584.617746 14443.286706 1341.475879 29.647570 1056.135941 24.905594 946.581402    22.411608 1512.782896 31.734637\n",
            "EPOCH=7 12909.148658 18352.522073 4149.826426 56.323244 3536.084253 49.149991 3267.630784    44.967031 4501.612754 59.874736\n",
            "EPOCH=8 11628.009018 11008.171627 1093.026264 28.314322 865.109255 23.918081 812.211672    22.640930 1291.839621 30.877517\n",
            "EPOCH=9 9620.408751 15443.465526 2569.155804 42.590505 1982.929903 37.022329 1423.167143    29.584839 2662.526414 45.380922\n",
            "EPOCH=10 7898.058959 7478.076761 1146.036721 27.656402 593.523580 18.759789 476.179455    15.924646 1306.175314 29.696119\n",
            "EPOCH=11 7398.530616 6162.422371 784.806351 22.778890 573.620514 18.687233 450.030698    16.283878 888.299423 24.389300\n",
            "EPOCH=12 5956.989215 4636.683160 863.905893 24.028735 540.392447 17.928950 351.285308    14.318441 923.737230 24.987881\n",
            "EPOCH=13 5612.073550 4886.306734 1310.639369 32.622503 873.881048 25.878165 538.916561    20.040473 1269.954624 31.816698\n",
            "EPOCH=14 5035.078511 6387.183780 683.375295 21.128132 517.766982 18.511603 482.820076    18.553826 668.926537 21.953211\n",
            "EPOCH=15 4501.569568 5670.637153 1616.131340 34.048927 910.380090 24.611547 597.453645    20.108108 2139.764280 39.948583\n",
            "EPOCH=16 4562.765952 3796.887091 651.034267 20.872780 332.354469 14.426762 221.363562    11.613596 629.920239 20.559907\n",
            "EPOCH=17 4092.149364 3667.629650 560.004152 18.707419 288.302493 13.389112 182.718515    10.802623 590.383052 18.743986\n",
            "EPOCH=18 3754.047089 2568.215805 633.133955 20.624215 344.462726 15.295070 223.239185    12.665767 599.664638 19.881978\n",
            "EPOCH=19 3369.136832 9299.415427 457.613983 16.425958 437.839679 15.989513 443.442578    17.129241 917.295909 21.232115\n",
            "EPOCH=20 3343.647211 4754.302331 1567.583327 33.543135 676.784347 22.046835 355.811410    16.910720 1444.247675 32.373375\n",
            "EPOCH=21 3005.569260 1890.020461 356.489006 14.605401 163.063718 10.012446 121.738888    8.750750 377.193651 14.985525\n",
            "EPOCH=22 3088.724792 4329.409660 942.871605 24.431909 506.591052 17.584743 239.493272    12.792696 1062.043539 25.268786\n",
            "EPOCH=23 2500.337668 8759.673983 715.735327 20.620406 822.018125 21.583418 833.244550    23.685412 633.458822 18.984361\n",
            "EPOCH=24 2390.673826 1848.549386 422.149480 16.285923 219.619823 11.801209 160.849131    10.299015 384.054917 15.364021\n",
            "EPOCH=25 2183.041461 2640.049820 256.617014 12.952975 229.106976 12.039900 203.084962    11.431555 266.133126 13.017036\n",
            "EPOCH=26 2634.913814 5045.272073 445.882335 16.859084 432.127827 16.295844 482.084613    17.814505 478.536669 17.320515\n",
            "EPOCH=27 2194.340822 1731.118645 331.266359 15.331106 239.138414 12.649659 196.156085    11.792446 323.824009 15.158219\n",
            "EPOCH=28 2086.000861 2095.791140 841.264384 20.108647 311.641229 12.423133 134.988503    9.047781 673.114168 18.758950\n",
            "EPOCH=29 2060.133670 3329.286210 684.802310 18.372839 246.328022 12.214498 183.682944    11.104091 898.245138 21.420008\n",
            "EPOCH=30 2011.500291 1813.294333 624.137951 21.185273 392.319201 16.949407 323.083966    15.564570 646.638009 21.959257\n",
            "EPOCH=31 1827.826856 2679.746280 804.003265 24.655149 428.153066 18.427316 368.675724    17.253336 632.327949 21.746354\n",
            "EPOCH=32 1555.891384 1172.856554 303.534851 14.283123 167.530382 10.527915 134.311437    9.155236 285.772404 14.364614\n",
            "EPOCH=33 1967.579867 1530.353872 384.600870 15.761924 252.975394 12.531426 217.516938    11.783260 373.468218 16.073340\n",
            "EPOCH=34 1622.157523 1860.406002 501.676257 19.272587 333.203172 15.950574 311.212912    15.497321 487.068180 19.010609\n",
            "EPOCH=35 1689.361020 1272.723540 335.079781 15.053863 270.129648 13.242579 207.358903    11.420797 321.662002 14.964449\n",
            "EPOCH=36 1620.649104 12946.547371 767.280660 21.137937 1187.500048 27.046618 1307.525349    29.507225 966.296053 24.887575\n",
            "EPOCH=37 1539.419598 1767.654715 314.482965 14.543942 232.593281 12.353220 218.015248    11.506936 295.541233 14.123369\n",
            "EPOCH=38 1489.359492 1678.380332 331.673640 14.166599 257.628956 12.185909 324.440127    14.542141 264.586940 12.594374\n",
            "EPOCH=39 1481.742477 6259.051277 388.744935 15.107411 445.053654 16.245027 474.571381    17.450403 438.157480 15.652745\n",
            "EPOCH=40 1967.201096 9611.390377 1035.578168 23.694780 629.979840 19.595304 453.454364    16.938269 1039.687573 21.824683\n",
            "EPOCH=41 1955.010935 5144.392113 728.597960 20.776220 537.171181 17.612979 483.221416    17.146809 577.722938 18.710497\n",
            "EPOCH=42 1982.933309 1489.786288 262.755116 13.401079 205.493385 11.678342 191.170066    11.387080 237.256393 12.853208\n",
            "EPOCH=43 1917.863224 1583.374101 377.989128 15.981587 276.081699 13.958257 241.333137    12.813683 275.131229 14.145922\n",
            "EPOCH=44 1579.931973 1171.016509 410.082957 17.602367 305.422766 14.996246 270.981738    13.901340 368.246839 16.568352\n",
            "EPOCH=45 1272.863149 906.343006 318.115685 15.240372 238.124783 13.132919 208.221245    12.038015 268.183393 14.035369\n",
            "EPOCH=46 1466.222377 1403.688678 175.945385 10.858796 115.306971 8.316021 85.095582    7.176223 170.285390 10.225943\n",
            "EPOCH=47 1445.500530 1699.942383 492.637457 17.922938 335.039273 14.851351 259.892514    12.873891 449.306929 17.245553\n",
            "EPOCH=48 1426.378848 3713.882626 596.696518 22.021053 436.446880 18.811929 354.347707    16.954356 503.084818 19.723248\n",
            "EPOCH=49 1782.712561 1039.077536 546.453220 19.780188 374.065134 16.758553 324.843942    15.551481 478.233194 18.051738\n",
            "EPOCH=50 1415.671878 17726.428323 999.935561 25.023688 1328.625715 28.613682 1799.450390    34.408882 1233.947835 28.272907\n",
            "EPOCH=51 1261.973400 1526.115792 140.920586 9.901601 107.209246 8.460804 83.505238    7.010912 112.254090 8.470291\n",
            "EPOCH=52 1461.157619 1057.853826 456.907127 14.289333 208.397245 11.105780 133.788110    8.699060 222.851509 10.938290\n",
            "EPOCH=53 1340.117594 1781.564732 267.784017 12.911239 242.432674 12.303288 247.083880    12.680324 206.764010 11.137307\n",
            "EPOCH=54 1181.754955 938.813112 389.633815 17.110905 310.664638 15.073785 280.963163    14.116467 356.802753 16.445921\n",
            "EPOCH=55 1363.924539 1155.548301 210.237966 11.865119 171.908599 10.308148 147.151339    9.339627 152.009824 9.551150\n",
            "EPOCH=56 1589.570195 726.190213 261.686625 14.179920 257.943789 13.558373 238.428496    12.646677 202.017563 11.748452\n",
            "EPOCH=57 1472.290808 2229.118769 529.942267 19.154978 383.520080 16.036440 298.710807    13.973292 395.735637 15.757818\n",
            "EPOCH=58 1362.136879 904.049440 147.279447 9.597757 122.125181 8.510375 93.036049    7.351156 107.770555 8.417627\n",
            "EPOCH=59 1478.156920 1874.813771 398.993362 16.609355 269.254366 13.400127 184.523870    11.013035 331.327548 14.603691\n",
            "EPOCH=60 1339.502727 1788.626395 248.717243 12.569970 229.321004 11.961787 212.893759    11.520683 211.356081 11.243386\n",
            "EPOCH=61 1133.073914 6992.152902 379.810161 15.304572 400.467149 16.116663 447.279630    17.087494 338.962123 14.444716\n",
            "EPOCH=62 1465.097056 2129.362723 173.069250 10.410435 144.843818 9.537274 93.851313    7.791009 108.390114 8.354962\n",
            "EPOCH=63 1033.490393 659.106275 229.770608 13.001603 204.388568 12.052441 185.861874    11.105362 195.338084 11.838044\n",
            "EPOCH=64 1348.459102 897.805587 253.408163 13.260671 210.347209 11.942562 199.748172    11.652335 202.476590 12.006894\n",
            "EPOCH=65 1253.898557 551.896787 196.247433 11.600945 174.811040 10.737557 159.881825    10.116621 141.577007 9.731661\n",
            "EPOCH=66 1430.941193 3504.211217 228.547643 10.941212 239.567334 11.276247 265.592350    11.951288 302.222362 11.887953\n",
            "EPOCH=67 1228.093848 1146.799417 215.668090 12.474974 168.027356 10.758384 130.069620    9.033572 160.271941 10.561588\n",
            "EPOCH=68 1267.717783 901.820654 231.084025 13.088753 197.348784 11.816536 182.615101    11.009052 183.120579 11.499255\n",
            "EPOCH=69 1247.879422 537.243792 170.874935 10.933001 158.232936 10.254086 158.643862    10.241671 126.178387 9.349953\n",
            "EPOCH=70 1102.906340 1686.417287 273.570197 13.359275 322.920423 14.222571 343.308852    14.842472 256.864716 12.735830\n",
            "EPOCH=71 1142.839647 788.997721 191.028332 11.454993 177.944749 10.862323 166.847694    10.298642 162.695688 10.471978\n",
            "EPOCH=72 1130.659651 1108.782180 289.566085 14.454325 328.524804 15.166454 321.045718    14.866329 258.502294 13.326394\n",
            "EPOCH=73 1330.509500 627.533381 210.205204 12.141540 168.710675 10.703006 153.294880    10.119225 165.661486 10.812814\n",
            "EPOCH=74 1143.647988 1129.817445 259.000466 13.759303 235.779057 12.969791 216.905744    12.356843 218.059320 12.898723\n",
            "EPOCH=75 1337.285347 3312.727121 246.621923 12.503445 228.036530 12.186885 266.909274    13.221262 286.784227 12.956284\n",
            "EPOCH=76 1200.359203 613.806176 315.658101 15.275419 292.172725 14.447980 308.397087    14.839482 234.438597 12.951242\n",
            "EPOCH=77 967.270152 1569.021453 280.829999 13.413266 206.486988 11.549518 162.661124    9.692158 231.755471 11.421406\n",
            "EPOCH=78 1236.864686 1092.393702 245.560834 12.955880 260.692088 13.120928 251.523358    13.002943 203.619657 11.703377\n",
            "EPOCH=79 1218.753471 1301.954210 253.662062 13.521003 216.204808 12.494865 186.470640    11.350067 224.284702 12.736421\n",
            "EPOCH=80 1024.921295 786.031568 205.112609 12.122935 204.675107 11.947300 178.262710    11.038815 161.345236 10.673178\n",
            "EPOCH=81 1175.792871 510.979977 132.007569 8.978194 115.009729 8.468572 104.456996    8.119015 107.370490 8.368641\n",
            "EPOCH=82 1062.241623 813.846060 311.483397 15.012878 295.801488 14.482926 267.514510    13.393876 251.392820 13.472629\n",
            "EPOCH=83 1124.492367 764.982616 187.706738 11.121006 183.804927 11.063397 177.678022    11.040371 162.126908 10.326810\n",
            "EPOCH=84 1016.961529 896.065476 204.339871 12.278065 201.859288 12.058769 215.703864    12.583957 166.341287 11.124996\n",
            "EPOCH=85 1113.640665 1150.982034 275.007604 13.769723 242.803443 12.835101 224.229697    12.360937 207.808500 12.014317\n",
            "EPOCH=86 886.028828 671.703776 138.467875 9.730645 137.651560 9.633479 129.903297    9.330558 113.420577 8.734393\n",
            "EPOCH=87 1149.062588 2240.889044 239.436720 12.027438 202.173806 11.012433 168.118657    9.963699 222.189292 11.159038\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-51-1336bd5a6a32>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlamb\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-50-18c8f75791bd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, lamb, max_iter)\u001b[0m\n\u001b[1;32m     86\u001b[0m       \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m       \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m       \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m567\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0mval_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse_5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_5\u001b[0m\u001b[0;34m,\u001b[0m     \u001b[0mmse_9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmae_9\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriteria\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1JbgQO-FMs5N",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "06c8a1fd-bb85-4f13-c091-17d277471934"
      },
      "source": [
        "def pred(model, val_loader, test_loader):\n",
        "  model.eval()\n",
        "  hidden = model.initHidden()\n",
        "  pred_label = []\n",
        "  mse = 0.0\n",
        "  weights = []\n",
        "  for i, (inputs, labels) in enumerate(test_loader):\n",
        "    inputs = inputs.to(device)\n",
        "    labels = labels.to(device)\n",
        "    output, weight = model(inputs, hidden)\n",
        "    weights.append(weight.to('cpu').detach().numpy())\n",
        "    pred_label.append(output.to('cpu').detach().numpy())\n",
        "    step = 0\n",
        "    labels = labels[:, step]\n",
        "    output = output[:, step]\n",
        "    mse_sum = torch.sum(torch.pow((output - labels), 2))\n",
        "    mse += mse_sum.item()\n",
        "  mse = mse/315\n",
        "  return np.vstack(pred_label), mse, np.array(weights)\n",
        "\n",
        "preds, mse, weights = pred(model, val_loader, test_loader)\n",
        "print(mse)\n",
        "for k, i in enumerate(preds):\n",
        "  print(','.join([str(k) for k in list(i)]))\n",
        "\n",
        "torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "301.031153312562\n",
            "67.92358,69.876335,70.66485,70.82065,70.22916,71.713646,71.50754,71.13899,74.98229,75.46991\n",
            "77.708534,76.96415,76.63735,76.083664,83.72402,86.329346,84.43171,86.252045,79.54052,80.48531\n",
            "66.88258,63.362286,65.71035,65.65867,66.35354,66.270195,67.031624,68.09753,72.02675,72.84423\n",
            "69.19776,66.36722,67.028656,66.32448,66.88923,65.03648,65.23697,66.31954,66.984886,65.884155\n",
            "71.759026,77.98195,78.42031,78.73118,78.997765,79.24675,79.34009,80.354744,80.70208,82.50687\n",
            "72.83006,73.10901,72.99141,72.89564,72.59561,73.06616,73.45066,73.41811,74.432175,75.27235\n",
            "81.96655,87.12719,85.36069,85.49302,85.55965,85.64179,87.09851,84.55969,84.375404,86.47033\n",
            "71.18769,70.435394,70.33931,69.89865,70.08787,69.96808,69.78139,70.58192,72.01757,72.7356\n",
            "75.52231,75.11677,74.18165,74.92341,77.933624,76.703705,76.8431,75.48066,76.693016,77.77889\n",
            "76.88742,82.5917,83.304634,78.25418,77.524956,77.886055,78.27253,78.57934,82.09176,82.98129\n",
            "77.785286,88.027596,80.82183,80.04474,79.89751,80.68089,82.88915,83.659454,84.02604,86.169426\n",
            "97.52711,87.13172,87.086174,86.887436,86.58818,86.63343,89.63767,90.01867,90.85704,91.62152\n",
            "95.94714,93.15959,93.24794,92.94553,83.715614,84.34144,84.094795,84.69474,88.51085,89.173615\n",
            "81.89255,84.35607,84.66841,84.79444,84.75737,88.52974,98.26013,99.05529,98.2201,87.2916\n",
            "81.63119,81.25934,80.7783,80.54605,80.99965,80.37504,81.09733,81.33841,94.47786,86.614044\n",
            "77.8831,89.91058,80.81718,81.00801,78.3894,78.44973,78.372856,79.0733,82.552574,82.77049\n",
            "79.289024,78.14787,79.314064,79.584694,90.88933,81.73347,81.2082,83.02672,83.16159,83.941345\n",
            "87.56121,87.199196,86.6424,87.10017,87.09681,87.524345,87.61261,87.71069,89.245384,102.899\n",
            "84.02197,97.25636,94.82833,97.141045,97.15134,97.312225,97.51064,98.8475,86.929825,87.506615\n",
            "82.9255,82.78317,81.09485,92.60704,90.46126,90.99087,92.663445,83.16825,85.20732,86.05062\n",
            "76.98393,76.807785,76.79641,76.4668,76.34069,79.45594,76.2687,76.32601,77.001915,78.31459\n",
            "91.591644,91.057655,94.34545,92.59669,92.046486,92.61075,83.05526,84.17194,87.85852,89.02554\n",
            "87.002304,100.19699,99.23572,99.07179,99.15481,97.777306,97.95461,98.747444,102.081696,106.20924\n",
            "81.32613,93.62191,83.60555,83.91508,83.23778,82.492,83.9989,84.10787,84.28718,84.67301\n",
            "72.27832,72.154015,72.32823,71.79797,72.2043,71.96912,72.48239,72.55353,73.60069,74.33172\n",
            "81.076324,81.10477,93.88585,82.256775,82.4248,81.862526,81.782455,81.38437,81.573044,79.30272\n",
            "86.3784,85.72724,85.760796,87.534424,87.650345,100.48677,97.74506,100.288284,91.595184,93.9518\n",
            "89.07712,89.000595,88.49972,88.3255,101.51397,98.74752,101.135506,89.1703,89.33819,90.595146\n",
            "82.38842,81.63792,81.826065,81.82972,81.94283,95.81838,84.44092,84.452324,84.581406,85.47288\n",
            "99.612526,88.59038,88.57437,88.430115,88.104126,88.204506,88.44173,88.26139,88.599594,89.44707\n",
            "82.00189,80.627754,93.85233,84.24905,87.143555,86.99438,86.646095,86.51602,85.33165,86.579056\n",
            "70.86203,70.36062,70.33094,69.94248,70.03342,70.12646,69.98096,70.40038,70.589615,74.850975\n",
            "98.902245,97.185555,97.89785,92.89292,92.65205,92.53971,92.55759,86.89802,87.05992,87.74749\n",
            "79.1174,82.981384,83.55042,83.68981,87.09235,86.757126,91.24828,85.961266,87.6823,91.669586\n",
            "86.7921,99.717224,100.01076,97.74774,97.56657,97.67339,98.05861,98.78888,88.06847,88.99875\n",
            "88.46005,87.59727,86.76391,93.29246,89.86831,89.81961,90.35103,92.34814,92.623314,86.41617\n",
            "80.566635,80.727066,80.70535,80.70908,80.46154,80.26675,80.1153,90.151184,91.48032,95.29127\n",
            "96.509186,95.60641,94.95064,94.69715,94.73188,94.78273,95.212975,95.25393,97.98371,87.560036\n",
            "80.34544,93.81965,91.28127,91.1914,90.96214,91.03919,91.86556,94.370285,94.50288,96.80226\n",
            "88.02167,87.37044,86.27354,86.020836,85.96031,86.6013,82.54082,82.38461,83.07751,84.08667\n",
            "80.06531,79.76334,79.64501,92.04169,88.661026,88.64167,88.90303,82.213486,83.0305,87.19266\n",
            "87.05858,86.46843,101.01106,99.94256,101.75422,99.97256,99.99755,101.23345,97.24135,89.35907\n",
            "91.54712,91.88914,91.99375,101.95342,101.45586,101.252304,105.27156,105.25412,105.269585,105.97055\n",
            "103.0986,102.22297,100.29251,100.96902,89.49449,89.353584,89.44998,89.46946,90.00307,91.08769\n",
            "85.01583,98.34432,92.70562,92.38374,92.32239,92.406204,92.77888,93.078896,86.42404,87.73066\n",
            "103.634346,102.38516,101.96846,102.00361,103.08082,101.909676,102.0413,100.48636,101.16709,91.05233\n",
            "79.60702,83.77247,83.91275,84.11757,84.444695,95.51832,91.29857,91.70041,84.7821,85.78439\n",
            "79.09351,79.07191,78.703804,78.65807,90.38411,80.12436,79.692245,80.05119,80.992744,85.52167\n",
            "83.57688,83.40824,83.15153,96.123276,84.23157,84.33816,84.29741,84.31213,84.72576,86.308014\n",
            "80.41108,79.98866,86.227036,87.53085,86.48749,86.226295,86.59702,87.83684,82.725136,83.72481\n",
            "93.513885,83.08663,82.08156,82.01636,81.92009,81.25759,81.72074,81.27975,81.84395,79.62303\n",
            "99.99606,103.69431,102.56701,102.103905,102.81636,103.461624,102.80113,101.37852,90.34958,91.37636\n",
            "86.33166,92.48558,99.14775,95.88233,95.674225,95.55414,95.745964,96.28446,95.79868,97.75077\n",
            "95.993515,97.62978,96.07169,97.00428,96.32281,92.95728,92.53895,92.486465,93.063675,82.3947\n",
            "81.699455,81.09578,87.410736,86.781525,87.24834,90.39375,90.51111,90.648094,90.74117,95.86368\n",
            "89.00816,89.09586,88.61225,88.64777,88.460205,89.15641,89.10545,90.19535,94.86551,85.2177\n",
            "79.376144,78.92919,78.75417,78.64749,90.896576,83.16717,86.3795,84.46989,84.85236,86.045586\n",
            "83.020836,82.44809,82.12857,81.41034,81.33962,81.445496,92.4078,93.0784,96.38531,99.48306\n",
            "80.98059,92.509125,88.27275,87.602844,87.47526,88.06223,91.75247,91.915306,92.14484,95.02373\n",
            "84.58961,84.05856,84.1511,83.96664,83.53835,84.09113,87.40772,97.602425,97.632065,87.14541\n",
            "75.38447,74.44031,74.04454,74.29088,77.771774,78.24239,81.770294,75.86608,76.105034,77.21211\n",
            "81.74572,88.54003,90.39197,94.721306,92.892006,93.90474,89.70759,89.48181,83.18823,80.43948\n",
            "79.79561,79.174774,77.97852,82.159164,87.84337,84.445274,84.432076,84.8883,78.84753,78.22803\n",
            "77.85445,78.471634,78.37114,89.678406,87.27442,87.11687,87.08243,86.80645,79.63666,76.8186\n",
            "79.194466,79.89258,79.14486,78.94661,79.11735,82.89099,93.752914,86.075554,86.73893,88.32891\n",
            "81.48695,86.35597,86.07355,86.2628,86.42772,86.51388,94.7509,92.91762,95.130264,82.33119\n",
            "100.36085,102.23133,100.23775,100.175896,99.59287,102.91115,103.164185,103.85975,90.970604,92.13631\n",
            "82.99828,82.73442,81.9519,94.920425,82.876045,82.82675,82.931244,83.57227,83.72853,85.71448\n",
            "102.60444,98.99812,98.87246,97.934166,97.96689,98.10369,98.00847,98.35154,97.76818,99.93494\n",
            "84.70725,84.86359,85.0497,85.08663,97.97625,90.325966,90.31165,90.320885,90.68673,87.25384\n",
            "80.112686,80.14388,79.63269,79.49645,90.9324,86.020966,86.05997,86.09179,80.06243,78.267395\n",
            "85.34572,84.7546,84.53552,84.38075,84.16654,84.50564,84.79819,85.019295,87.862404,101.31517\n",
            "80.842674,80.76721,80.937744,93.412766,89.59394,89.16349,89.93733,83.1372,83.507866,87.632515\n",
            "84.78682,95.0264,97.586,95.952156,94.91933,97.00784,97.11864,97.80652,86.50797,88.20643\n",
            "84.51354,84.49764,84.50858,91.79011,90.67223,93.2449,94.02387,98.57046,95.62241,97.37482\n",
            "94.602425,94.4245,93.916084,94.01116,93.70466,94.02851,100.486534,101.02902,99.91487,102.43909\n",
            "75.13232,76.46597,76.619514,76.582214,76.15209,75.92452,76.38012,82.45851,76.51098,80.505135\n",
            "82.52268,85.327835,85.315056,85.06554,85.21834,85.539536,91.81056,83.00707,83.07355,84.17302\n",
            "79.26217,80.10113,80.20471,84.41254,84.59802,92.62101,80.841415,81.25116,80.937485,78.70276\n",
            "77.23141,76.41353,79.0687,79.246895,79.43877,79.90484,82.8977,83.51041,83.853386,79.53267\n",
            "86.21375,85.918365,98.73821,96.6971,96.970604,98.35369,87.065346,87.41602,87.59582,88.441925\n",
            "84.89399,84.651085,84.455154,88.615654,89.059364,90.03408,90.14151,91.11711,93.30068,100.31196\n",
            "96.052574,90.848335,90.7835,90.586685,90.16261,93.23199,93.102844,87.510086,88.28039,89.41753\n",
            "104.25428,102.681244,102.46147,102.78435,103.30286,101.72642,103.06394,91.51115,91.541374,91.63741\n",
            "85.2794,95.25706,94.612526,93.74157,93.8341,99.25271,97.87127,98.05654,98.904076,100.234505\n",
            "84.36171,83.90365,85.92039,93.841675,96.747,94.34619,94.5,95.20037,96.385544,87.77257\n",
            "85.52558,95.29976,94.03515,96.634384,96.8357,96.87762,96.82755,98.71612,97.28718,101.29869\n",
            "79.67392,90.045334,88.778076,87.86475,87.39793,88.54689,87.53648,91.21635,91.67898,85.97842\n",
            "81.005905,79.30834,79.30912,82.39633,93.1618,88.13933,88.53389,83.55716,84.04462,84.95883\n",
            "81.96839,92.10843,90.916664,90.899925,90.57773,92.34138,94.69018,88.14464,87.64842,80.94491\n",
            "83.8216,95.5654,90.9417,90.78422,90.26209,92.99699,93.16822,93.28535,94.08474,96.35381\n",
            "93.30824,84.688484,84.08834,84.055405,83.36794,83.771164,83.94818,87.310936,88.10221,89.11659\n",
            "82.49429,82.873146,82.62349,81.91572,83.582726,95.4251,93.69933,93.22109,85.95187,87.16282\n",
            "102.4913,102.51574,103.01919,102.32345,102.74498,102.36605,101.89004,105.18988,104.39863,104.3765\n",
            "74.88401,75.24665,75.07527,77.11155,76.720184,77.594124,76.86586,77.444824,74.4628,71.22523\n",
            "77.14515,89.91022,81.67472,80.68585,80.01342,79.885,79.785416,80.995476,81.71652,86.18856\n",
            "80.39217,80.947174,89.81376,89.42547,89.08111,89.115906,88.98979,91.53402,92.843994,98.16546\n",
            "75.37489,75.614334,84.35525,78.5474,77.00354,76.95388,76.76487,77.1798,81.90175,82.845055\n",
            "70.37939,70.146126,68.22184,68.01086,68.750984,68.36491,68.47643,68.853264,73.7464,75.04489\n",
            "98.32789,97.43832,95.96183,99.40706,98.53124,98.749146,99.25948,99.454445,100.02819,91.42306\n",
            "88.94645,102.03945,101.457726,101.11687,102.4088,101.68145,104.33252,103.65077,101.90617,104.075584\n",
            "89.32033,88.841354,88.74446,100.36766,98.10281,98.31328,98.29355,100.17707,89.746864,90.39115\n",
            "88.30718,89.06942,95.1956,94.98771,100.38324,96.67601,96.81,97.01298,97.36892,99.62952\n",
            "101.21728,100.66498,99.91242,100.82901,99.900986,100.0973,99.49386,99.16482,103.11169,102.52498\n",
            "93.2762,105.88835,107.470276,106.80583,106.860954,107.601364,107.34067,106.86711,105.91249,108.34074\n",
            "85.62191,98.112785,99.243065,98.446846,98.8536,98.58165,98.57212,99.65029,100.05513,103.73649\n",
            "91.10555,90.97825,91.03955,101.22048,100.84858,103.80974,103.51827,103.69254,102.22587,103.23459\n",
            "84.88063,97.726746,95.90166,95.58178,94.99377,97.65382,90.2265,89.654816,90.13063,91.61979\n",
            "82.17249,81.6291,81.32588,80.80999,80.088234,80.758896,84.42959,82.73923,86.91135,82.35294\n",
            "95.1803,94.45415,93.35452,92.55224,92.84364,91.89885,92.27033,84.51418,85.370384,90.11941\n",
            "83.89754,84.15391,83.507805,90.00834,92.233215,91.0654,90.50045,94.72757,85.755005,87.17615\n",
            "89.55589,102.17398,100.58271,102.034805,99.80796,99.78466,100.07949,99.299866,90.91442,90.7108\n",
            "78.66415,77.44442,76.58373,80.47162,80.44929,80.70442,80.01331,78.73233,79.29016,80.549835\n",
            "94.93389,93.20565,92.88304,92.789154,85.42098,83.59111,83.17469,82.744774,87.415825,88.57887\n",
            "90.55397,89.25229,89.07223,88.73894,87.82989,91.550865,92.080795,88.93305,89.364105,91.034035\n",
            "91.570244,90.99573,94.87987,92.66424,93.15243,96.99497,97.036964,97.78983,91.5942,92.45978\n",
            "88.388054,87.63298,89.66167,89.634315,89.45981,89.48077,89.56031,90.283676,92.22716,94.91557\n",
            "99.8919,99.34282,99.12396,102.40824,102.43049,100.37105,100.25949,101.13174,90.016884,91.569534\n",
            "92.15048,92.97596,92.735855,93.171646,99.082466,97.59036,97.54322,98.08358,97.11167,99.90483\n",
            "105.1402,105.44538,104.06994,104.73257,103.71613,104.29867,102.21127,102.045616,102.940834,94.86866\n",
            "91.65104,91.52173,104.3027,103.65851,105.07014,103.995544,105.18823,103.16856,95.57031,95.51053\n",
            "97.711174,97.99679,97.164,94.24962,93.95737,94.55373,98.66205,98.864494,99.17334,91.46558\n",
            "81.30011,82.36725,81.84047,91.4237,91.21976,96.3825,94.22291,93.51723,94.35087,96.55433\n",
            "104.83175,103.676895,103.851555,101.65887,101.80986,101.699905,101.538185,101.52355,102.49371,93.47698\n",
            "76.088776,76.792656,78.650925,78.85159,82.47844,83.35021,83.54515,88.34655,79.713356,81.54469\n",
            "91.96032,97.20168,96.00493,95.56748,95.87964,95.69321,95.84716,97.27893,95.89594,98.64428\n",
            "92.15774,91.873856,91.562,95.081154,93.85486,98.710266,95.55921,96.033134,90.20137,87.79588\n",
            "99.68945,96.60329,96.379196,96.046814,95.99055,96.093605,95.86998,96.50064,92.93234,94.48287\n",
            "93.86347,93.079994,92.7848,95.22731,98.51689,91.71205,92.25433,86.31511,86.82928,87.73397\n",
            "86.129456,97.23776,97.94021,96.92041,97.21293,96.838356,97.19672,97.0721,97.18421,99.91391\n",
            "77.89971,84.40744,86.218994,86.848335,85.57063,85.57016,86.32716,81.92512,85.40133,86.175514\n",
            "100.57683,100.19308,100.521736,102.8189,100.82146,99.95014,100.671165,103.93794,104.62881,93.187035\n",
            "88.98227,96.38495,98.366936,98.04239,98.04362,97.84212,98.09978,101.26899,101.47709,102.570076\n",
            "74.538765,74.65391,75.34126,75.51361,80.088104,76.20686,76.3077,77.65298,77.98565,81.34291\n",
            "84.97238,90.53556,90.156975,90.12667,89.92578,92.38783,92.44746,94.724884,93.98624,97.29514\n",
            "83.54285,84.68389,89.5631,97.67675,94.69058,95.37142,94.841446,93.87812,94.537926,82.13374\n",
            "88.82153,88.35053,101.61554,99.681946,99.689644,101.548874,97.30784,91.77601,93.9786,94.62858\n",
            "85.20918,97.49875,99.0821,96.9385,95.956665,95.92272,98.38205,98.15039,99.12315,89.406364\n",
            "102.692474,103.00773,103.457924,104.652885,107.43533,104.64324,104.97881,103.939865,104.23168,104.99961\n",
            "93.25556,97.34878,96.874954,96.98348,106.655075,106.038734,108.08298,106.412056,106.487274,109.56795\n",
            "84.0862,83.98441,83.866776,83.768036,84.096275,93.20247,97.285,97.42449,91.384315,87.504555\n",
            "95.311584,97.95343,96.14538,96.01226,96.22414,95.8195,95.915794,97.33262,88.09076,89.12481\n",
            "83.719284,85.66768,92.54945,92.26908,94.25971,97.23896,95.90574,99.49356,96.433876,98.886154\n",
            "85.956116,94.98712,94.86672,100.10126,99.33437,99.00443,102.617165,102.173195,101.67901,90.99598\n",
            "83.62389,96.786766,97.04672,96.05315,95.79199,95.89878,95.89558,97.20366,85.33487,82.158104\n",
            "91.81218,104.23711,104.72177,105.72864,103.64696,103.39686,103.44102,103.4788,103.6653,106.6199\n",
            "87.519646,97.63044,98.630714,99.30942,98.6794,101.145996,102.40266,99.52201,98.38125,99.671\n",
            "96.22596,95.21524,94.75081,94.83364,95.42096,94.48413,94.385506,94.9696,84.51054,82.86405\n",
            "90.22815,90.11279,91.50193,91.48983,92.18387,94.19202,103.69646,102.68058,102.23591,102.80646\n",
            "109.2198,109.54378,108.20398,107.724106,105.93767,107.49445,105.734344,105.897385,105.89326,95.24644\n",
            "97.71634,98.48436,97.98802,97.85623,98.55153,99.79998,98.60158,99.50915,99.68722,100.80481\n",
            "96.01449,95.538666,95.747894,97.53526,96.86139,98.22453,94.68689,95.172905,95.717926,89.19677\n",
            "100.35299,99.85611,99.5797,99.179886,102.07085,99.23323,101.82168,99.18469,99.602554,92.32911\n",
            "85.64302,96.63581,95.51415,96.3335,95.779594,97.93683,93.160835,86.0123,88.522125,90.153145\n",
            "91.790764,103.31621,102.85215,105.86413,105.95694,106.88207,107.48437,107.376335,105.9644,105.907814\n",
            "100.37515,99.47711,99.23859,102.43941,101.17373,99.32443,99.18915,99.14552,100.81502,92.46202\n",
            "95.68758,108.73921,109.78531,111.49663,110.96084,111.65246,111.519325,111.36964,111.430954,113.033714\n",
            "108.447845,107.99616,107.34194,107.607735,108.351105,106.8799,107.71717,106.6467,104.91951,107.5262\n",
            "88.24558,93.61013,103.3016,102.41637,102.28842,104.05512,103.36493,104.2839,104.73706,102.63457\n",
            "89.18283,88.411934,88.61566,88.742584,95.198166,94.550186,98.728065,98.393524,96.45683,98.70303\n",
            "81.24935,92.11702,92.511604,92.56695,91.81305,92.59478,82.91322,82.61393,82.4622,79.73761\n",
            "86.96249,91.63851,102.748055,102.36385,102.34731,104.13227,103.94425,103.28573,102.31726,105.19762\n",
            "106.11462,105.47689,105.49675,105.108536,105.03172,104.9974,105.74981,105.12035,103.650116,104.50223\n",
            "96.078285,95.59681,99.146736,99.362434,99.44214,98.42899,101.92362,102.66699,99.883896,90.99593\n",
            "96.719444,107.26223,107.89483,107.69502,109.14575,109.030754,109.08101,110.210655,110.37583,111.22819\n",
            "101.26266,101.58933,100.317825,99.88317,99.50289,97.3298,95.75813,95.92449,96.4272,84.91698\n",
            "112.01637,111.97153,112.09017,113.06041,112.716606,113.88117,114.23147,114.55929,113.57493,115.24348\n",
            "95.993355,101.478836,104.553764,105.53358,104.7404,105.523186,104.17926,104.0249,103.555885,89.932625\n",
            "93.34868,92.773254,94.76757,102.54803,103.59474,101.7906,102.64001,99.330956,99.340775,102.003395\n",
            "89.563934,101.95879,100.703094,101.61719,101.84097,101.871994,101.39682,103.76847,101.18341,87.46086\n",
            "110.50304,110.374825,110.19368,110.550385,111.537735,111.583824,111.72707,113.136284,109.45018,110.62086\n",
            "90.924576,92.13323,104.06911,102.13841,101.808655,102.17932,103.32128,103.406425,109.13985,114.024994\n",
            "113.34788,112.012856,113.15088,112.3619,112.299515,112.189255,112.42924,110.96899,107.19553,94.8566\n",
            "104.6318,106.259514,106.365746,108.46398,105.85709,103.75945,102.84821,96.024666,96.18743,100.09385\n",
            "110.330864,110.03313,109.8023,111.72827,110.94572,110.85291,110.81366,112.13704,111.055954,111.73038\n",
            "105.274734,105.264755,119.011055,122.28951,123.76363,124.23,123.98225,123.947975,125.15304,132.03648\n",
            "97.812065,107.18916,111.03985,110.848114,115.24746,111.69568,111.44306,110.16639,109.00516,99.45786\n",
            "95.899666,107.88857,109.8441,110.74674,115.88888,113.96903,112.13392,112.1416,111.57595,114.32695\n",
            "83.589806,89.17372,99.02421,95.58625,95.367676,95.17657,96.21469,90.404274,89.79719,89.50373\n",
            "115.67952,114.420494,115.42725,116.33994,115.45225,114.960464,115.71491,120.04237,121.06703,129.50502\n",
            "100.072426,107.53654,110.738045,110.388565,111.6898,111.41683,117.93943,118.83127,115.566666,116.745255\n",
            "98.70683,99.78408,100.00311,99.1648,95.65336,95.19041,94.666756,95.12275,87.33376,87.59142\n",
            "108.58962,108.76495,108.021614,111.17211,118.346794,120.95512,116.82296,115.97753,113.99394,117.37873\n",
            "125.134315,124.53762,124.23581,124.622215,125.50054,126.38634,126.17039,126.440216,127.62574,143.59451\n",
            "118.287315,116.10621,121.39158,140.43622,137.13985,137.52881,137.52649,137.5287,140.57043,154.6732\n",
            "100.110466,101.13078,100.502625,101.59746,100.89285,100.06495,99.64241,102.430305,95.80604,87.394485\n",
            "139.75491,126.52011,123.77014,123.556175,123.25184,121.37402,121.250786,121.390274,125.10633,126.02871\n",
            "117.18192,118.01287,118.248,119.38709,138.10887,145.76884,148.89473,146.51056,146.05342,160.0255\n",
            "101.084045,114.123665,129.62839,137.44481,134.12137,131.21346,131.21072,134.2824,141.27962,129.59204\n",
            "100.70324,107.430504,109.78806,110.3403,110.012474,121.78941,117.150986,120.20486,118.63151,119.90823\n",
            "97.05656,100.8923,100.61119,102.61763,101.90662,101.62086,99.271355,100.37148,90.08026,87.17939\n",
            "126.02193,124.839485,126.18779,125.907074,126.778244,159.5243,134.79254,133.33928,131.0169,124.818474\n",
            "125.537964,150.27354,141.25897,141.2249,141.24304,150.29153,134.38904,131.78734,131.12059,131.17021\n",
            "114.60322,113.50979,113.20455,114.50959,114.461,117.25201,145.09717,119.94489,115.6968,106.685356\n",
            "134.28288,134.47925,134.77232,134.6225,134.39758,134.20317,140.00606,164.41052,156.73575,167.56372\n",
            "91.64729,102.36326,104.48109,104.25837,103.60714,105.88743,104.078476,104.44017,103.840096,104.73115\n",
            "100.19622,99.155235,99.60584,99.139626,98.43607,97.94413,98.54064,98.56973,102.29128,103.53979\n",
            "99.45175,99.43938,99.7355,98.88754,102.11679,101.27391,100.45374,100.42934,99.10888,98.57955\n",
            "107.95351,109.909424,112.34448,112.06344,111.639114,112.20801,110.444374,111.9979,110.70316,113.3937\n",
            "127.435356,126.940056,130.11261,131.83731,169.4599,142.35239,141.55638,141.11656,139.38652,136.22438\n",
            "116.354225,115.902596,116.21431,117.84655,118.45613,118.16962,118.79503,119.09668,121.773056,120.46815\n",
            "105.22228,103.90067,103.07373,106.38337,106.83885,107.61517,107.31992,106.55463,107.51305,105.23218\n",
            "108.09366,106.66068,108.563286,108.28333,108.32921,109.027435,107.930176,111.36607,110.308525,110.68728\n",
            "114.88103,113.49541,113.64547,116.56354,116.58787,118.80291,122.877525,133.51564,173.94322,134.1687\n",
            "108.17435,108.44597,108.34602,107.43657,111.07359,110.963455,111.16946,110.88227,109.080414,111.803444\n",
            "177.4062,156.27747,156.38411,155.79066,156.26715,148.65195,148.83774,148.31029,147.4538,147.69211\n",
            "113.66971,114.1594,114.882256,115.57982,115.284904,115.99139,115.041176,115.285095,113.18355,116.05874\n",
            "128.1575,137.54027,176.3362,169.00711,168.96266,168.44669,168.4688,169.54584,173.7583,178.91287\n",
            "118.80932,134.98369,176.65665,147.70125,147.75095,147.37231,146.65865,146.04422,145.30078,148.50945\n",
            "119.55565,119.26411,119.08302,119.79805,128.7298,124.5214,123.4433,123.61912,122.00539,122.85363\n",
            "106.02473,108.16305,108.699615,108.17949,107.66297,107.4132,109.15803,106.935715,106.331055,112.128105\n",
            "121.7496,121.569954,120.88416,131.62625,132.57568,135.63724,139.85298,178.46469,168.5614,179.23167\n",
            "113.871956,179.05734,163.359,157.61215,181.53232,156.29306,178.29642,148.72673,147.9354,147.95753\n",
            "106.36871,123.35351,125.853226,129.03683,129.1665,131.9845,133.08179,133.74799,138.8936,177.67888\n",
            "179.9525,178.2723,179.9764,181.65381,181.28943,181.08351,178.67247,181.44464,180.01726,144.73196\n",
            "179.76263,176.69778,179.87704,161.64561,161.68198,161.52034,161.70837,164.00682,157.13925,179.66696\n",
            "110.94185,109.98095,108.30249,109.00606,110.43596,111.1219,110.57911,110.532936,107.71363,107.15899\n",
            "150.55731,148.71347,148.14835,148.98764,151.11227,151.19077,151.44778,151.45346,154.32442,179.28256\n",
            "149.25444,180.05415,160.8637,160.6677,159.59297,160.71698,160.17972,167.96286,147.01279,146.77855\n",
            "138.70421,140.61545,179.72601,171.78503,171.54016,170.5359,171.74841,177.00494,178.41927,178.46208\n",
            "106.444534,106.77691,107.39935,106.44691,106.22831,106.04807,105.77539,109.292755,108.63487,112.12691\n",
            "108.23168,111.31456,111.49191,120.400604,117.64126,118.22815,118.88974,119.37385,119.70948,122.279945\n",
            "138.87283,137.83597,138.22412,140.56433,146.43053,147.9596,180.51196,145.634,144.76785,149.19896\n",
            "116.191986,123.39172,129.43106,133.69559,172.86964,147.17807,145.29681,144.69556,144.59596,148.28793\n",
            "142.84236,147.42322,149.17467,180.76471,177.07004,179.85414,180.8182,182.03874,180.70378,181.44681\n",
            "117.375465,116.708084,116.680435,117.94347,118.12796,117.57725,117.28111,117.73148,114.74073,116.058754\n",
            "141.1383,142.15714,145.28801,149.93567,182.35019,175.76524,179.0321,180.90247,182.95753,146.03802\n",
            "180.22336,173.81995,173.61307,173.91766,178.01839,181.83972,146.15483,145.26141,143.27449,142.17032\n",
            "181.17029,177.35387,142.53899,136.91821,140.02596,139.36198,140.38559,138.75856,137.77158,137.85117\n",
            "128.5903,133.73444,137.7028,138.63301,139.05603,140.8406,145.31244,181.03824,133.8237,122.299034\n",
            "129.4754,133.27649,135.99458,143.41528,182.8018,145.28284,146.90393,144.79906,144.04364,148.03706\n",
            "148.32231,149.90042,182.2885,170.57321,170.30423,173.64365,177.52345,172.44724,181.28242,123.406845\n",
            "145.05898,148.88083,150.41081,182.06064,173.8297,174.01993,176.14905,180.44855,181.96928,181.66112\n",
            "136.96432,143.39874,182.60077,180.50685,183.69,182.8255,182.92812,183.67055,182.89009,144.77065\n",
            "118.49418,130.57631,124.481865,124.40548,127.15952,127.59938,129.0368,128.31844,123.452576,123.95\n",
            "121.03424,133.31912,136.51093,139.60106,146.87798,147.50734,176.39464,175.46487,178.54156,181.15628\n",
            "137.7436,141.88185,182.20103,174.4965,173.71237,177.26761,175.37091,176.02386,182.1028,143.904\n",
            "145.4631,147.5601,151.83649,181.91336,149.90727,149.99089,150.02362,149.49443,149.43634,147.92903\n",
            "148.00507,162.72014,159.67505,159.71104,162.10863,162.29285,163.75008,168.44112,183.88356,163.45073\n",
            "138.55579,137.05946,138.4575,138.86015,138.92735,138.24004,138.47737,141.32068,159.50418,153.21661\n",
            "126.9371,125.42705,126.46547,126.68126,127.5801,127.268135,127.42582,127.21635,126.66431,124.935074\n",
            "151.00717,181.62418,152.37189,151.822,152.30515,152.48557,152.43828,152.24449,153.12135,155.9356\n",
            "179.89136,156.78516,156.8795,156.84935,156.80992,156.79642,156.77039,156.88634,158.53593,163.59898\n",
            "142.74591,148.41875,147.83919,163.70892,161.37785,165.91438,166.98969,182.6197,181.8533,181.04387\n",
            "154.7239,155.19524,169.07161,167.45735,165.16856,169.7612,177.20634,182.80139,146.91878,146.68365\n",
            "180.2921,175.00362,178.23433,179.65729,181.57278,180.01984,180.1296,179.00014,179.7936,181.8228\n",
            "151.59344,153.20636,151.60075,176.21997,169.53769,168.7436,168.55385,172.71138,181.6933,144.93977\n",
            "137.11975,143.14693,152.89424,160.0372,183.43369,179.85463,183.83699,183.80353,183.56241,167.11118\n",
            "180.6532,172.70317,172.8653,172.97334,172.92459,172.26884,176.51524,179.28146,140.73419,142.90843\n",
            "117.38757,128.30954,134.08511,135.16565,135.87915,138.13039,140.14645,142.0133,145.64836,179.59744\n",
            "107.64391,108.473175,109.96369,109.64758,112.79208,119.775444,117.664276,117.23335,116.478806,115.50007\n",
            "122.09319,134.47133,138.33464,144.44244,153.26755,154.49217,177.19867,176.82172,179.55133,181.14159\n",
            "162.6392,159.22484,162.2296,160.51936,167.13638,155.59369,156.46098,155.82791,158.24109,164.96234\n",
            "164.03334,160.16704,164.76689,162.37979,162.60728,180.73257,179.97285,172.94366,165.55441,145.9449\n",
            "146.45767,181.5388,182.35944,177.59805,181.46155,149.84625,150.09009,148.66562,148.48589,150.34317\n",
            "141.66594,140.7486,141.52702,142.21281,144.98993,143.2687,143.16591,146.70694,161.83102,161.40765\n",
            "133.6028,137.51497,144.81706,143.01573,141.36513,146.93071,151.78009,152.89964,153.68419,179.76395\n",
            "128.71262,134.29424,135.86328,139.0671,137.82706,140.8728,178.53433,177.06631,183.30994,154.41269\n",
            "181.04155,158.64265,158.64938,156.96133,156.67712,156.67604,152.64877,156.71716,160.14032,167.47008\n",
            "144.20892,182.4908,173.88162,174.05502,174.0152,173.99825,175.542,179.8245,183.54349,154.12057\n",
            "118.68105,119.33096,130.55762,126.190094,125.95903,125.91624,125.31278,125.68991,124.89174,125.051125\n",
            "141.9205,152.27458,149.55154,151.88293,153.65309,153.42046,182.76654,150.05052,149.621,153.64458\n",
            "124.27037,124.58885,125.112816,124.694725,125.51727,125.47178,125.05206,125.68279,132.32281,123.37841\n",
            "118.10556,117.70296,118.79209,119.397194,119.71337,119.76789,119.86163,119.231766,119.5613,122.66908\n",
            "141.12122,143.22469,146.36734,183.02802,173.57635,173.91545,174.96056,178.64288,181.43266,180.44904\n",
            "141.59012,146.01198,157.7768,182.4444,150.0169,149.86433,149.49792,147.63593,147.66992,143.49063\n",
            "118.68664,121.69065,120.307915,120.05409,126.88027,124.86669,124.435425,136.71397,121.70709,124.193245\n",
            "167.17953,166.03145,182.58333,138.47252,138.39793,135.70538,136.26222,132.39444,129.5691,117.18233\n",
            "181.2464,174.90765,166.83073,175.93246,176.22643,178.22011,184.47142,180.54993,182.94775,118.92241\n",
            "139.35542,141.16617,147.45265,148.61774,149.21849,184.16838,181.35255,181.59528,184.10635,147.47798\n",
            "122.301285,123.77344,136.87018,133.43091,137.07887,131.77568,131.92032,131.62917,131.14436,131.44916\n",
            "166.32341,159.40869,164.79019,176.7989,138.97252,144.3184,143.64894,143.35463,142.71584,137.67734\n",
            "127.72085,131.44147,136.57541,145.80382,181.64874,169.80002,171.97464,171.3742,168.67497,157.66023\n",
            "93.309265,96.48931,102.74464,107.82126,110.47559,111.04355,113.726105,123.65385,110.09634,103.444115\n",
            "152.55606,151.66182,163.605,183.40535,183.40042,154.37431,153.80084,158.1676,154.23743,151.68454\n",
            "145.75807,162.17259,156.51381,159.88235,167.23917,167.97731,184.41536,184.52332,167.86658,181.25175\n",
            "124.483376,123.37067,124.11936,124.68198,124.83379,124.96194,124.86666,124.32908,123.69512,125.08699\n",
            "179.10187,181.78415,182.74101,182.92224,182.38689,182.19469,181.62132,176.11925,168.68025,181.52705\n",
            "168.9032,166.58957,163.4299,168.18457,184.50438,184.50679,167.37856,169.42792,184.49562,141.52837\n",
            "115.65646,131.17247,151.44652,147.69751,152.62006,155.25082,156.20973,167.06384,166.71938,181.98814\n",
            "126.13874,126.19721,128.33794,137.38829,134.21591,133.93378,134.38144,134.9569,150.02219,134.49728\n",
            "168.51631,163.36444,162.07692,166.58743,184.25568,181.5922,184.22276,184.10884,170.11716,173.42679\n",
            "149.36539,148.42068,155.04611,151.93388,170.04791,167.31168,170.10565,160.15251,129.45508,130.65903\n",
            "150.83391,184.15979,178.66641,176.32996,177.98051,182.69484,180.21658,183.35437,184.37329,152.47542\n",
            "156.45744,156.63982,156.89845,184.09534,174.33481,175.60344,178.39374,180.67514,179.43086,180.18303\n",
            "159.84337,158.9897,168.01009,168.19815,168.29897,184.1346,180.115,182.73505,184.45995,134.16481\n",
            "163.35591,169.40182,169.9251,172.66722,180.11772,181.37743,184.17941,171.69832,175.06635,173.69621\n",
            "122.28317,147.70564,143.54262,143.55414,143.58557,143.63065,144.4362,148.53168,155.52884,151.42973\n",
            "169.45757,181.42892,181.4,181.71516,182.53871,184.02159,155.74583,150.51344,150.70587,146.62817\n",
            "154.22314,153.59648,165.29402,168.4942,183.45016,180.78384,173.8849,178.20496,181.53828,165.24066\n",
            "151.33159,166.85132,166.25647,180.80453,181.90189,183.50499,182.59975,181.40186,181.91992,181.34297\n",
            "150.52881,166.06207,165.77403,166.28831,166.01015,166.13844,169.12671,172.49974,181.44888,148.88014\n",
            "120.71515,120.32217,126.6534,125.937355,125.88111,128.66782,129.10893,125.580666,124.66315,125.59128\n",
            "178.70961,181.14758,181.35587,181.45409,181.40135,181.96107,181.59956,182.60776,182.64539,181.2963\n",
            "137.1737,172.76347,179.31381,184.1915,182.45367,184.27502,180.73929,184.20534,184.13467,152.08043\n",
            "160.38536,166.65884,160.34694,160.57942,159.05676,164.23953,170.65169,160.32172,169.89531,152.68964\n",
            "177.54678,178.05453,171.1176,171.16603,172.13818,177.45386,176.34758,172.98135,170.39615,157.35988\n",
            "126.3448,180.21298,181.44368,176.0696,181.18187,181.92165,181.99228,182.10059,182.23866,181.6365\n",
            "140.09074,139.79549,145.11285,148.71925,184.39018,180.62144,179.57382,159.0813,163.30736,158.59962\n",
            "140.13756,140.6531,141.49539,142.27028,142.43494,146.4355,151.12082,157.01613,154.52338,135.15393\n",
            "159.40805,183.47536,183.03178,183.4021,182.45107,183.60146,183.3795,183.39369,183.81053,175.70456\n",
            "147.06627,156.82413,176.0751,178.22781,178.54489,184.41115,183.88121,183.63498,184.05614,137.61563\n",
            "139.45068,183.77711,183.78564,160.35092,159.86752,159.41095,159.03764,158.60222,153.31673,137.82675\n",
            "140.15625,154.12135,151.56377,151.45703,154.37947,158.98734,161.70905,163.55376,181.80127,145.958\n",
            "145.63554,156.28583,171.91586,183.25893,176.03558,181.85075,181.55331,182.25899,182.27332,181.30833\n",
            "132.5784,147.1178,147.87471,167.84361,173.65337,183.24657,181.04446,177.57358,176.24,162.67389\n",
            "126.64638,148.85902,150.61998,154.07558,184.03795,160.76477,160.04163,153.0019,154.01888,154.70966\n",
            "122.78323,180.16685,171.72897,181.13841,183.53123,184.22891,176.49504,176.14494,173.75443,161.48962\n",
            "141.38281,141.92699,142.04572,142.86783,143.24556,144.7021,151.35796,154.82669,156.84456,156.85356\n",
            "121.56041,129.27536,129.35358,131.8519,133.36467,133.3402,146.23615,140.53242,154.31854,135.07483\n",
            "153.39539,152.93713,184.17795,167.05502,167.28886,154.69838,150.58215,149.98428,142.20732,118.335754\n",
            "179.91243,181.79028,183.26248,175.1185,165.94986,166.35371,161.1635,162.35393,161.89005,121.66755\n",
            "177.25467,182.9943,183.11537,183.14397,183.17392,184.36296,182.8325,183.7408,164.74635,170.77846\n",
            "170.71301,171.23659,183.39404,183.46698,182.17702,182.5987,171.04913,156.91293,158.7959,163.39883\n",
            "134.36844,139.42538,163.7885,178.33513,180.1016,181.76268,182.3307,182.73491,184.09013,182.31665\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fWWfm56OdnnS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "outputId": "6f771754-c1f6-4f72-dc72-c07395406ce9"
      },
      "source": [
        "for i in weights[10][10]:\n",
        "  print(','.join([str(round(x, 4)) for x in i]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.0,1.0,0.0,0.0,0.0,0.0,0.0\n",
            "0.0,0.0,0.0,0.9974,0.0026,0.0,0.0\n",
            "0.0,0.0169,0.0,0.0,0.9831,0.0,0.0\n",
            "0.0,0.0,0.0,0.0,0.0,0.0,1.0\n",
            "0.0,0.7454,0.0,0.0108,0.0,0.0,0.2438\n",
            "1.0,0.0,0.0,0.0,0.0,0.0,0.0\n",
            "0.0,0.0,0.0,0.0,0.0,0.0,1.0\n",
            "0.0,0.0,1.0,0.0,0.0,0.0,0.0\n",
            "0.0,0.0,0.0,0.0,0.0,0.0,1.0\n",
            "0.0,0.0,0.0,0.0,0.0,0.0,1.0\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}