05_01_ConvNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "05_ConvNet.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN1g6IaEXVdiX5bLPnAj8F6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d66c888791a64644b27e2e2f7fcb5691": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c82f2747dbc4b72a323d517639eb8e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0322349e0ce84a88a0945937ab83313c",
              "IPY_MODEL_a5a7cecbfec447e3b59e9a2e9752e706"
            ]
          }
        },
        "5c82f2747dbc4b72a323d517639eb8e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0322349e0ce84a88a0945937ab83313c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c759e43f2b3a4114bd9710e9d03d45e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c5a6ed06f684cec857f2094ba6f27c0"
          }
        },
        "a5a7cecbfec447e3b59e9a2e9752e706": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bde5a79e71a74b3d8484340a0b463858",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "9920512it [00:02, 4169055.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a86b8c80c2f54705bf1cfafaeb160d6c"
          }
        },
        "c759e43f2b3a4114bd9710e9d03d45e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c5a6ed06f684cec857f2094ba6f27c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bde5a79e71a74b3d8484340a0b463858": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a86b8c80c2f54705bf1cfafaeb160d6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "23a58fcf473047968b33f264e86a6d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d7a12ce528ad46fb9306a935cdfb9e67",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6ac4e473d2e2457bb5fe4388be01f917",
              "IPY_MODEL_18e9eed8cc784120bf21234bee308759"
            ]
          }
        },
        "d7a12ce528ad46fb9306a935cdfb9e67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6ac4e473d2e2457bb5fe4388be01f917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7b010715cb564c3eb4de632ee33e477e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aa0cb6f49e8946bcaa49839642312eea"
          }
        },
        "18e9eed8cc784120bf21234bee308759": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fd1605c79b4b49b597fabb874950bd90",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "32768it [00:00, 38013.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_49b6639b046042a18679997ae001fd20"
          }
        },
        "7b010715cb564c3eb4de632ee33e477e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aa0cb6f49e8946bcaa49839642312eea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd1605c79b4b49b597fabb874950bd90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "49b6639b046042a18679997ae001fd20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98ffd464a497428f995a9322847fdc3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_02d2d7d813b845f0893f290102b1573f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_28dcdc81ed8042ca95a6d1471b2723aa",
              "IPY_MODEL_4c5b26f904ce4b71bbb6c5c2a4acc4b2"
            ]
          }
        },
        "02d2d7d813b845f0893f290102b1573f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "28dcdc81ed8042ca95a6d1471b2723aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_07d9bf55ebf5433f8ffa1734d385ab8b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_283c9709451443d2bb06db4882e771e9"
          }
        },
        "4c5b26f904ce4b71bbb6c5c2a4acc4b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_93a70f9d17ca45ddaa675d49cfa8309a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "1654784it [00:01, 948741.08it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e1738ce07d1c4afe8a0db35e5027f193"
          }
        },
        "07d9bf55ebf5433f8ffa1734d385ab8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "283c9709451443d2bb06db4882e771e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93a70f9d17ca45ddaa675d49cfa8309a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e1738ce07d1c4afe8a0db35e5027f193": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "489aab27f8f545eba9d21f198b88f50b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fd1a5c2e1a0c448bb2debd7f8231560c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3b0b4ce38db7436daa0741cabdc59775",
              "IPY_MODEL_08f911b041194fc3ad6c833d3f3de43b"
            ]
          }
        },
        "fd1a5c2e1a0c448bb2debd7f8231560c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3b0b4ce38db7436daa0741cabdc59775": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_512e899978bb45a3902b88ef44cec6ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_72a5e52d623b44798f04589b9a20e0a4"
          }
        },
        "08f911b041194fc3ad6c833d3f3de43b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_71bf76acb7b74cafb5b5be844254a17d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "8192it [00:00, 12717.10it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4724dfab39c7469a97e9e06d315b5b0e"
          }
        },
        "512e899978bb45a3902b88ef44cec6ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "72a5e52d623b44798f04589b9a20e0a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71bf76acb7b74cafb5b5be844254a17d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4724dfab39c7469a97e9e06d315b5b0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/OUCTheoryGroup/colab_demo/blob/master/05_ConvNet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J-s-Fmmpk4LH",
        "colab_type": "text"
      },
      "source": [
        "## 卷积神经网络（CNN）\n",
        "\n",
        "Outline\n",
        "- 今天我们学习如何使用 PyTorch 进行CNN的训练与测试\n",
        "- 我们还会展示池化与卷积操作的作用\n",
        "\n",
        "深度卷积神经网络中，有如下特性\n",
        "- 很多层: compositionality\n",
        "- 卷积: locality + stationarity of images\n",
        "- 池化: Invariance of object class to translations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KELFYNGfmQLX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchvision import datasets, transforms\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy\n",
        "\n",
        "# 一个函数，用来计算模型中有多少参数\n",
        "def get_n_params(model):\n",
        "    np=0\n",
        "    for p in list(model.parameters()):\n",
        "        np += p.nelement()\n",
        "    return np\n",
        "\n",
        "# 使用GPU训练，可以在菜单 \"代码执行工具\" -> \"更改运行时类型\" 里进行设置\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_PJQDVcRmhrV",
        "colab_type": "text"
      },
      "source": [
        "## 1. 加载数据 （MNIST）\n",
        "\n",
        "PyTorch里包含了 MNIST， CIFAR10 等常用数据集，调用 torchvision.datasets 即可把这些数据由远程下载到本地，下面给出MNIST的使用方法：\n",
        "\n",
        "torchvision.datasets.MNIST(root, train=True, transform=None, target_transform=None, download=False) \n",
        "- root 为数据集下载到本地后的根目录，包括 training.pt 和 test.pt 文件\n",
        "- train，如果设置为True，从training.pt创建数据集，否则从test.pt创建。\n",
        "- download，如果设置为True, 从互联网下载数据并放到root文件夹下\n",
        "- transform, 一种函数或变换，输入PIL图片，返回变换之后的数据。\n",
        "- target_transform 一种函数或变换，输入目标，进行变换。\n",
        "\n",
        "另外值得注意的是，DataLoader是一个比较重要的类，提供的常用操作有：batch_size(每个batch的大小), shuffle(是否进行随机打乱顺序的操作), num_workers(加载数据的时候使用几个子进程)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bYDOWS21moS7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386,
          "referenced_widgets": [
            "d66c888791a64644b27e2e2f7fcb5691",
            "5c82f2747dbc4b72a323d517639eb8e8",
            "0322349e0ce84a88a0945937ab83313c",
            "a5a7cecbfec447e3b59e9a2e9752e706",
            "c759e43f2b3a4114bd9710e9d03d45e7",
            "2c5a6ed06f684cec857f2094ba6f27c0",
            "bde5a79e71a74b3d8484340a0b463858",
            "a86b8c80c2f54705bf1cfafaeb160d6c",
            "23a58fcf473047968b33f264e86a6d7d",
            "d7a12ce528ad46fb9306a935cdfb9e67",
            "6ac4e473d2e2457bb5fe4388be01f917",
            "18e9eed8cc784120bf21234bee308759",
            "7b010715cb564c3eb4de632ee33e477e",
            "aa0cb6f49e8946bcaa49839642312eea",
            "fd1605c79b4b49b597fabb874950bd90",
            "49b6639b046042a18679997ae001fd20",
            "98ffd464a497428f995a9322847fdc3e",
            "02d2d7d813b845f0893f290102b1573f",
            "28dcdc81ed8042ca95a6d1471b2723aa",
            "4c5b26f904ce4b71bbb6c5c2a4acc4b2",
            "07d9bf55ebf5433f8ffa1734d385ab8b",
            "283c9709451443d2bb06db4882e771e9",
            "93a70f9d17ca45ddaa675d49cfa8309a",
            "e1738ce07d1c4afe8a0db35e5027f193",
            "489aab27f8f545eba9d21f198b88f50b",
            "fd1a5c2e1a0c448bb2debd7f8231560c",
            "3b0b4ce38db7436daa0741cabdc59775",
            "08f911b041194fc3ad6c833d3f3de43b",
            "512e899978bb45a3902b88ef44cec6ea",
            "72a5e52d623b44798f04589b9a20e0a4",
            "71bf76acb7b74cafb5b5be844254a17d",
            "4724dfab39c7469a97e9e06d315b5b0e"
          ]
        },
        "outputId": "04604efb-4181-45aa-db72-239fb825639b"
      },
      "source": [
        "input_size  = 28*28   # MNIST上的图像尺寸是 28x28\n",
        "output_size = 10      # 类别为 0 到 9 的数字，因此为十类\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=True, download=True,\n",
        "        transform=transforms.Compose(\n",
        "            [transforms.ToTensor(),\n",
        "             transforms.Normalize((0.1307,), (0.3081,))])),\n",
        "    batch_size=64, shuffle=True)\n",
        "\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, transform=transforms.Compose([\n",
        "             transforms.ToTensor(),\n",
        "             transforms.Normalize((0.1307,), (0.3081,))])),\n",
        "    batch_size=1000, shuffle=True)\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d66c888791a64644b27e2e2f7fcb5691",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "23a58fcf473047968b33f264e86a6d7d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "98ffd464a497428f995a9322847fdc3e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "489aab27f8f545eba9d21f198b88f50b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nC8Gx3708xoJ",
        "colab_type": "text"
      },
      "source": [
        "显示数据集中的部分图像"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L9h3cbDjwWBD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        },
        "outputId": "a56e42b2-6b38-47c7-e51e-652f0285e2ad"
      },
      "source": [
        "plt.figure(figsize=(8, 5))\n",
        "for i in range(20):\n",
        "    plt.subplot(4, 5, i + 1)\n",
        "    image, _ = train_loader.dataset.__getitem__(i)\n",
        "    plt.imshow(image.squeeze().numpy(),'gray')\n",
        "    plt.axis('off');"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAboAAAEeCAYAAAD8etB9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3decBN1f7H8bd0KyVCkuYSaURFuC4S\nJaKkwk2ibuMvqRuplEYZGiXFTaMGKaQ0KqSorgbdythAVyllSiSJ3x/ud699znnm55y9n7PO5/WP\n4+x9nrOs5zh7f9f6ru8qt3XrVkRERHy1XdwNEBERySRd6ERExGu60ImIiNd0oRMREa/pQiciIl7T\nhU5ERLy2fUEHy5Url1NrD7Zu3Vou7jaoz6Ol/o6W+jt66nNFdCIi4jld6ERExGu60ImIiNd0oRMR\nEa/pQiciIl7ThU5ERLymC52IiHitwHV0knuOOeYYAC677DIAevToERx74oknABgxYgQAH3/8ccSt\nExEpPkV0IiLitXIFbbwa1Yr68uXLA1C5cuU8j1t0sfPOOwfPHXLIIQD83//9HwB33nknAN26dQNg\n48aNAAwZMgSAm2++udB25HIVg/r16wMwbdo0ACpVqpTvuWvXrgWgWrVqpX7fuPs8m6pGnHDCCQA8\n9dRTALRo0QKAhQsXFvlnqL/zd/311wPuu2K77bbFAS1btgTg7bffLvbPjLu/oWz3eSaoMoqIiOSc\njM/R7bfffgDssMMOADRt2jQ41qxZMwB22203ADp37lzkn7ts2TIA7rvvPgA6deoEwLp16wD49NNP\ngZLdheWSRo0aATBhwgTARdUW6Vt/AmzatAlwkVzjxo0BN1dnx33TvHlzwP27J02aFEs7GjZsCMCc\nOXNieX8f9ezZM3jcv39/ALZs2ZJwTkGjXpIdFNGJiIjXMhbRJc/55Df/VhzhOy0bT//1118BN2+x\nfPlyAFavXg0Ub/4iF9g859FHHw3Ak08+CUDNmjXzPH/x4sXB42HDhgEwbtw4AGbNmgW438XgwYMz\n0OL42RxN7dq1gegjOpsrOvDAAwHYf//9AShXLvbpn6xnfQmw0047xdiS7HfccccB0L17d8DNIR9+\n+OEJ5/Xt2zd4/P333wNudM++jz744IO0tk0RnYiIeC1jEd23334LwMqVK4HiRXR2NV+zZg0Axx9/\nPJA4BzR27Ni0tDPXjB49GnDZqYWxyA+gYsWKgJv3tEjnqKOOSmMLyx5bS/jee+/F8v4WbV9wwQWA\nu+tdsGBBLO3xQevWrQHo3bt3yjHr11NOOQWAH3/8MbqGZaEuXboAMHz4cAB23313wI04zJgxA4Dq\n1asDcMcdd6T8DDvXzunatWta26iITkREvJaxiG7VqlUA9OvXD3B3R5988klwjmVMmrlz5wLQpk0b\nANavXw+4Md4+ffpkqrles2onAO3btwdS53csSnvppZcAty7RxtDB/e5s/rNVq1Z5/izf2BxZXMaM\nGZPw9/C8qRSPzQU9+uijQN4jTRZxLF26NLqGZYntt992yTj22GOD5x566CHAzf/PnDkTgFtvvRWA\nd999F4Add9wRgPHjxwevPfHEExN+/ocffpiJZiuiExERv2V8Hd0LL7wAuOzL8LqsevXqAXD++ecD\nLoqwSM588cUXAFx44YWZbaxnLPN16tSpwXNW8cTWBr366quAm7OzTCnLpAxHEz/99BPg1ihaFqxF\niTaf50MNzPC8Y40aNWJsSWrUEf59SvGce+65AOy1114px2wuyWq6SirLqEweZQD3ubQ5u19++SXh\nuD2fHMWBWxf9+OOPp6+xIYroRETEa7rQiYiI1yLbpic5jAVXHNhY+vSzzz4LpJbikaKpU6cO4BKB\nwkNfP//8M+AW1ttQgS28f/nllxP+LIoKFSoAcNVVVwFw9tlnl7jtZUW7du2Cx/bvi5oNmdpCcfPd\nd9/F0ZysZinv5513HuC+W2wJE8Btt90WfcOyhCWWXHfddUBiWbQHHngAcNMdeX3XAwwYMCDfn3/5\n5ZcDbnok3RTRiYiI12LdePWmm24CXPq7JULYYs433ngjlnZlK0vftaQei0rCCUC2+NnSeNMZrVgB\nbx/YNlBhlhQVFfs9WmS3aNEiIPH3KQU74IADAFe0PJltIgwwffr0KJqUVQYOHAi4SM6Kdrz++uvB\nOVYM+7fffkt4rZVUs+QT+34IL0eyKHry5Mlpb3uYIjoREfFarBGdLSOwuTlLS7cFiHaHZdHHyJEj\ng9dq64xUDRo0ABLnlwBOPfXU4LG2LSq5TGyPY8s92rZtC7j0bUhNw7Z5kvC8khTM+jW5TN1bb70F\nuLJVksi2Trv00ksB931rkdxpp52W72sPPvhgwBXaDxesAHj++eeDx1YoPtMU0YmIiNdijejMV199\nBbhNEK08zznnnJPw5y677BK8xhZ1WvagwN133w24MXCL3jIVxVlprFzJjq1atWqBx60AQngOwuab\n99lnH8BtQGyZqdaHNr8R3p7k999/B1zZpY8++qh0/4AcYhHHkCFDEp63clS2cDw581u2sc+pZasa\ny47cY489gud69eoFQMeOHQE44ogjAFcE3qJB+9OKkkNqcZBMUUQnIiJeKxMRnbENLa1orUUoJ5xw\nAgC33357cK5tmDho0CAgt9cWWcFsK/lld04vvvhiRt/XIjl7PyvK7YNwBpn9+0aNGgW4DLRkNg8U\njug2b94MwIYNGwCYN28eAI888gjg5p8t6g5vCWNlkSwzVtvyFK6wLMuvv/4a0NY7hbHsSlvXZtvn\nfPPNN0DBORJWCN7W09k2U7aG1wrHR0kRnYiIeK1MRXTm888/B+Css84CoEOHDoCbuwO46KKLAKhd\nuzbgtvbJRXbHb+PqK1asAFyFmXSxdXq2/tFYwe5rr702re8XJ8s2A7ddS9OmTQt8jW02bIXMAebP\nnw/A+++/X6T3DRcut7toi0KkcLamK7954+Q5O8mbZfbaXOeUKVMAN09teRXg1sA99thjgNuibdy4\ncYCL6OzvcVBEJyIiXiuTEZ2xu4qxY8cCiVtDWCZa8+bNAWjZsiXgttrIZZatl46MVIviwNWysxqa\nNod01113Aa5epm+GDh0a2XvZfHRYfvNNso3NTUPeW8CAizoWLlwYSZt8YVnANrpQFPadbJWuLLqO\nc2RCEZ2IiHhNFzoREfFamRy6tDTtM844A4CGDRsCbrgyzNK1Z86cGVHryr50LCuw4SAbpgS3Q7AN\nA3Xu3LnU7yOFs2U3krdw8fcqVaokHLMkICtGIZlnyXHJy4+UjCIiIpIhZSKisy1RLrvsMgBOP/10\nAPbcc898X/Pnn38CLuEiV8pQ5cUWKNuflhLcp0+fYv+sK6+8EoAbbrgBSNy01Yq02lY/ImVBtWrV\ngsfJ3wO2KaiviVJlUXgLn7JCEZ2IiHgt8oguHKV169YNcJGcle/Jj5VLAlf6K9NlrrJBctFU6+P7\n7rsPcOWmAFauXAlA48aNAVcw2woSW/FhW/wcvjuzu2OJhkXoderUAYq+6DxXWAEJK4ydl9mzZ0fV\nHPmfk046Ke4mpFBEJyIiXst4RFejRg0ADjvsMADuv//+4FjdunULfK0tVrzjjjuAxO3Wc3lOrjDl\ny5cHXBmrcHakFVq10mnJ7A7YNr0dOHBgxtopBbMIvaCIJRdZRrBtgRT+LrBixLZJs4o3R++ggw6K\nuwkp9D9IRES8lvaIzop+jh49GnB3X0W5yls0YSWlbH4ovGWKpHrvvfcAmDNnDuDWHZrwvKhF2Mbm\n7GyNS0kyNSWzmjRpAriiublut912A/LOyrbtuvr27Rtpm8R55513gLK1MbMiOhER8VqpI7rjjjsO\ncBU0GjVqBMDee+9d6GttM0rLDrSNVaPaXt0XVlzZ1h/aFkZWhDkvw4cPB+DBBx8E4Msvv8xkE6UE\nwhu4imQL22bNNtC20bxatWoBbjPXKCmiExERr5U6ouvUqVPCn8msFqVt3Ld58+bgmM3F2XY8UjpW\nJcY2Rk3eIFXKvldffTV4fOaZZ8bYkrJrwYIFgJvTb9asWZzNkXzYCJ1tr2Zrn3v37h2cY9eHTFNE\nJyIiXtOFTkREvFbOFqXmebBcufwPemjr1q2xz/6rz6Ol/o6W+jt6cfV5pUqVABg/fjzgFvhPnDgx\nOKdXr15AehMQ8+pzRXQiIuI1RXQhuXz3FZe4+1z9HS31d/Ti7nOL7CwZ5ZJLLgmO2Sbb6UxKUUQn\nIiI5RxFdiO6+ohd3n6u/o6X+jp76XBGdiIh4rsCITkREJNspohMREa/pQiciIl7ThU5ERLymC52I\niHhNFzoREfGaLnQiIuI1XehERMRrutCJiIjXdKETERGv6UInIiJe04VORES8tn1BB1X1Onrq82ip\nv6Ol/o6e+lwRnYiIeE4XOhER8ZoudCIi4jVd6ERExGu60ImIiNd0oRMREa/pQiciIl4rcB2d+Gv4\n8OEAXH755QB8/vnnAJxyyikALF26NJ6GiYj8z1tvvQVAuXLblsa1atWqRD9HEZ2IiHitTEd0u+66\nKwAVK1YEoH379sGx6tWrA3D33XcD8Pvvv0fcuux0wAEHANC9e3cAtmzZAsChhx4KQN26dQFFdOlS\np04dAP7yl78A0Lx5cwAeeOABwPV/UUyePBmArl27ArBp06a0tdM31t9NmzYF4Pbbbwfgr3/9a2xt\nkqK75557APf7e+KJJ0r18xTRiYiI18pURGfRRv/+/QFo0qQJAEcccUS+r6lZsybg5pqkYD/99BMA\nM2fOBKBjx45xNscrhx9+OAA9e/YMnjvzzDMB2G67bfeUe+21F+Aiua1bi16G0H5Xo0aNAuCKK64A\n4JdffilFq/1UuXJlAKZPnw7ADz/8AMCee+6Z8HcpW4YMGQLAxRdfDMAff/wBuLm6klJEJyIiXos1\norP5ILszPfvsswGoUKEC4DJt/vvf/wKwbt264LU2p3TWWWcBbs5jwYIFmW52Vlu/fj2gObhMGDx4\nMADt2rXL6Pv06NEDgIcffhiAWbNmZfT9fGCRnCK6sq1x48aAm2N99913ARg/fnypfq4iOhER8Vpk\nEZ2NmQ8dOjR4rkuXLoDLrky2ePFiAE466STAXeXBRW677757wp9SsN122w2AevXqxdwS/0ydOhXI\nO6JbsWIF4KIwm7NLzrq0LDOAFi1aZKSduchGhyRzLKN4wIABAHTr1g2AVatWFfpaO9fyMb766isA\n+vbtm5a2KaITERGvRRbRderUCYB//OMfhZ5rV/M2bdoAbo7u4IMPzlDrcsfOO+8MwH777Zfn8YYN\nGwIuYtZcXtE9+OCDALzwwgspxyx7rLC5oUqVKgWPrVqNZWoa+/kffvhhyRubYyy7daeddoq5Jf76\n17/+BUDt2rUBOOywwwA3z1aQ6667DoBq1aoBcMEFFwDw6aefpqVtiuhERMRrkUV0tp4oL0uWLAFg\nzpw5gFtHZ5GcsUxLKbnvv/8egMceewyAm266KeG4/X3NmjUA3H///VE1Lett3rwZSP3cFofNRwNU\nqVIlz3OWLVsGqBpQSRx77LEAvP/++zG3xD8bNmwAihc9169fH4D9998fcHPW6Y68FdGJiIjXdKET\nERGvRTZ0aZOLF154YfDcG2+8AcCXX34JuBTs/NSoUSNDrcs9t956K5A6dCnxsELN9v8EXOGEZAMH\nDoykTdnMhpHXrl0LuOVNtWrViq1NvrLvkiOPPBKA+fPnA/knkuyyyy7BY5umsiQ5G1J+/vnn09pG\nRXQiIuK1yCI6S4IoTQRhRZ4lffJbuCyZZeXurrnmGsAtnQkXRUg2d+5cwC1VkPxZMtU777wDuA2F\nJT323Xff4LGNQlgUfdlllwGugHwy21oNXJKiXR8ytY2SIjoREfFamdqmx7baCY/hhtkYcNjs2bMB\neO+99zLXMI+VZLsYyZttM3XOOecEz7Vu3TrPc5s1awYU3O+2/Y5Ffa+88goAv/32W6nbKlISVqJr\n0qRJwXNWfnHEiBEAvP3223m+1sp5hbexMoMGDUpnM1MoohMREa9FHtFZdg24EjE33ngjkFoMt6D5\nIxvT7dWrFwB//vln+hsrUgR2l/viiy8C+ZdXKy6bX7LSSlJ6VmJKimb77bddIrp37w6kFiUH9/1s\nORTXXnst4ObiqlatCrj5uHCB7SeeeAKA0aNHZ+Yf8D+K6ERExGsZj+gsi6xBgwYATJgwIThWs2ZN\nwM05WJRm821t27YFEqNAY3cap59+OgDDhw8HYNOmTen9B4gUkd2pFmVLmKJku1qm4MknnwzAq6++\nWtom5ryOHTvG3YSsYus7x4wZA7g55fDn1tZBW3k1+/PUU08FYO+99wbc9304G/O8887LWNvDFNGJ\niIjXMhbR7bDDDoCLyiZOnJhyzs033wzAtGnTAJg1axbgxnTteZsDCatevToAgwcPBuDbb78F3BYm\nKnhbNPlFFraJooo6F86202nZsiXg5jMAXn/9dQA2btxY4M84//zzAejdu3cGWpi7pk+fDmgdXXHZ\nptiPPvoo4NZu2vrEv//978G5q1evBuCuu+4C3IbBFtnZCIdFg+FNsq0Auv3fsS3a0k0RnYiIeK1c\nQet4ypUrV+zFVTYnd8sttwDQr1+/hOPheQZbb2R3CRal2Xqho48+GnDzbsOGDQtea1GejQObN998\nE4ChQ4cC7m4jzCpMJNu6dWvhkysZVpI+Lw3LVs3vc3DUUUcFj+fNm5f294+7z6Pu7/xYLcaVK1em\nHOvQoQOQnjm6XOvvzp07A/Dcc88BLh/AMr4zvbFw3P0NJetzG02z7XNuu+02wEV4ebE+tQxKy8JM\njujCnn76aQB69OhR3CbmK68+V0QnIiJe04VORES8lrZklPLlywNuywYr97J+/XrAlTEaN25c8Bob\nsrRJS0t8sKUIixcvBuCSSy4B3MQyQKVKlQBo2rQp4IrkWvrw1KlTE9oX3vX5wAMPLNG/0UejRo0C\n4KKLLsrzeHhbpSuuuCKSNuWi8M7ikj5WaNjYMNqOO+4YR3OyxuTJkwGXRBj+/syPJZkkJw9269YN\ncElbYcuWLStVO4tKEZ2IiHgtbRGd3flbJLdhwwbARQq2yWrjxo2D11j5LlsQaxtNWiKLTXzmdTdh\nBW9fe+21hD/t7iGc/gpw5ZVXlvBf5rcFCxbE3YSsYwlXJ554IuAm7ktSbNn+D1jBA0kvi0zsc163\nbl3AjU5ceuml8TSsjCvO59ESqazEl4222VKB8ePHp7l1xaeITkREvJa25QXLly8H3BIBW7Btd1K2\n9Y5tMJkX25TVFoFHXag5W1OB02HRokUA1KpVK+H5cPFW+92lc1Fn3H1enP62rXUGDBgAQJs2bQA3\n51uUeQwrhmAFzG1rk1133TXlXIsQbd45PEddUtnU3+l07733Ai6CrlGjBlD4Qv7Siru/IfN9bkWc\nLT/DSnw1bNgQiG4ezmh5gYiI5Jy0zdH98MMPgIvoLKupXr16CefZYnCAmTNnAq5s15IlSwBtuROH\nL774AoCDDjoo4fmCig7nGssKTs4qu/rqqwFYt25doT/DokArhpA8ojJjxozg8YMPPgikJ5KTbay/\nVfy9dGwhOcA//vEPwPWtbSsVdSRXEEV0IiLitbRFdFYE+LTTTgPcHeuKFSsAeOSRR4DEkly6qyo7\n7C7Myk1J0dk6z5Kw/x8vvfQSAH369AmOZXr+KBdZRqCVDpw0aVKczcla4XXKFt09+eSTgNtIuyxR\nRCciIl5Le1HnbJYLGVL5sbuyKVOmAHDooYdae4Jz6tSpA+Ru1mX9+vUBt5XOueeeW+T3sT6z9aXv\nvPMO4CLpvKpGZEI29Xc62abOVapUAVz1pUyvI427vyEzfW6ZluCyLW0dXdxRsrIuRUQk5yiiC/H1\n7qssi7vPS9LfllHcs2dPwG1hYtGCZRGDm8uwCh2WnRyXbOzvdLAauzZSYWsTtU2PfxTRiYhIztGF\nTkREvKahyxANM0Qv7j5Xf0dL/R099bkiOhER8ZwudCIi4jVd6ERExGu60ImIiNd0oRMREa8VmHUp\nIiKS7RTRiYiI13ShExERr+lCJyIiXtOFTkREvKYLnYiIeE0XOhER8ZoudCIi4jVd6ERExGu60ImI\niNd0oRMREa/pQiciIl7bvqCD2pk2eurzaKm/o6X+jp76XBGdiIh4Thc6ERHxWoFDlyIiPqpTpw4A\nr732WvBc+fLlAdh///1jaZNkjiI6ERHxmiI6EckZI0aMAKBLly4AVK1aNTg2ZcqUWNokmaeITkRE\nvKYLnYiIeC3WocvDDjsMgFNOOQWACy+8EIA5c+YA8MknnyScf++99waPN23aFEUTRSSL1ahRA4CJ\nEycC0LhxYwC2bt22tOzzzz8Pzj3//PMjbp1ERRGdiIh4rZzd2eR5MAMr6i+66KLg8Z133glAxYoV\ni/TaVq1aBY+nT5+e3oahKgZxiLvP8+vv8GfSEhc2btwIwDHHHAPArrvuCsDZZ58NwIwZMwD47rvv\nCn3fH374AYDJkycD8OGHHxa77SVRVvs73Wz5gH3HtGvXzt4fgGuuuQZI7Hd9p5T45wPwzDPPAK6v\nbcRu2bJlmXz7FKqMIiIiOSfyiC6czjt//nwA9thjjyK9ds2aNcFju8t+44030ta2XLj7Kmvi7vP8\n+nvYsGHB4759+2bs/bds2QLAvHnzAHdXHH68ZMmStL1fWe3vdLO5uHfffTf5/QHo3r07kNjfmRB3\nf0Pm+3znnXcGYOHChQDsvffegMu5GDNmTCbfPoUiOhERyTmRZ12uWrUqeHzjjTcCcNdddwHuzuDb\nb78FYL/99kt47W677RY8btu2LZDeiE6Kx0olVahQAYBu3boBcMkll6Sc+/LLLwPQq1eviFpXOqef\nfnqh56xcuRKA//znP4Wea3e7hxxyCOA+yw0aNADgiCOOAGDQoEHBa+znpjOi853NzT399NOAi+CM\n/V5tblRKb8OGDQAsXrwYcBFd9erVY2tTMkV0IiLitVjX0Y0aNQqAiy++GIB69eoB8MsvvxT62vvv\nvz9zDZM8tW7dGnB3xRbBVa5cGXBrk/JicybZ4qSTTgoeW5SwaNGihHPsTnb58uXF/vmWsfnZZ58B\nqaMXAB07dgRcNCyFO+eccwDXn6+88grgvmOKkhErJTNy5EgAWrZsCcChhx4aY2sSKaITERGvRZ51\nmZczzjgDgAEDBgBQv379Ql9jdwsLFixIWztyIUOqqCxT6sgjjwyea9iwYZ7nrlu3DoCnnnoKcJVt\nwhlttgYtWdx9Hld/WzRsfWZ+//334PHf/vY3IL1r7Hzs79mzZweP7bvj+++/B9xc/pdffpnuty2S\nuPsbovuM77vvvgAsXboUcNWrDjzwQKBkIx8loaxLERHJOWVim57nn38ecGteLJMyHE0ku+222wAX\nDUrpVKtWDYDBgwcDcN555wGJWbIfffQRAEOGDAFcncDffvsNcNmykmqHHXYA4L777gOgR48eeZ7X\npEmT4PHcuXMz37AsduqppwJw3HHHBc/ZCNVzzz0H5D+SIJljma72mbe55tGjR8fWJkV0IiLiNV3o\nRETEa2Vi6NKK4tryAls8W5Dk0j5SOjfccAPgtiqxnZgtQQjg119/jb5hWe74448HXNp7z549E47/\n8ccfAFx++eVAepOrfGWL7S1ZJy+rV68GCi8o3KdPn+CxJVOYTJZ+81lygqMNYcZJEZ2IiHgt8oiu\nbt26weNJkyYBcPDBB29rzPZFb86LL76Y3oblCCuz1r9/f8BFGldccQXgtip5/fXXAU3ml0SjRo2C\nx5ZYVb58+TzPtbtfS+T5888/M9y67Gd9ZNslbbedu1+3ItkzZ87M87VXXnllwt979+4dPLaSduaq\nq64CYJ999gG02DybKaITERGvRR7RhcvC2ELC4kRyxu7MwndkUrjrr78ecBHd+PHjARd5KIIrvbPO\nOit4nF8kZ2z+wsp8hReHv/TSS4Ab+bDlHLmuRYsWgJujsygOXGT8888/J7zGFpLbayzlPWz9+vWA\nm9ezAty2/Klr166AWxAt2UMRnYiIeC3yiM7uTgGuvvpqAIYOHQrATjvtVOSfU7NmzfQ2LEdce+21\ngJsbsjJdiuTSZ+LEicFjG8Gw8mm77757ga899thjUx7bdlb33nsv4DaFXbFiRZpanB2sELaNBBkr\n9wUwduxYwJX8soLc/fr1A9wic4v4wtt82XZhVqR82rRpCX+XorEF4wWVl4yaIjoREfFarOvorByS\nbdgX3lgV3NydbclTqVKlCFvnp3//+9+Aixasb62M19SpU+NpmEfCRYbbt28PuG1jLKKrUaMG4LY8\nspJryRuFgssq/Oc//wm4bMMTTjgBSJyj8lmzZs0AuOeeexKef+ihh4LHt9xyC+D698477wSgXbt2\ngCtAbnPT4bVytWvXBtz2YXbuW2+9BWhurqjKUiRnFNGJiIjXysQ2PQW8PwA33XQTAAMHDgyOffXV\nV4C7q03H3ZYvW2pYkdtPPvkEcNtlAFStWhVwlTisIopVPbHXRlWhI+4+j/szbqw6UDiLOLweLy/X\nXHMN4ObsiiKb+9syhQcNGpTwfF5Z27NmzQISCz6D+754++23gcQNgZOrLdmcaGkqpMTd3xDfNj3G\nqgNZn2eatukREZGcUyZqXebH1hiFIzljNQJVScJloE6ZMgVw80G21vDJJ58MzrVtd2xuziK6ihUr\nAi7ik2jZBqzPPvts8Nybb74JQPPmzfN8jVUUyhU2h28jPZMnT045x9bLHXDAAQnnWpUTiyosG/Pp\np58OXpt8rkV0Ujo2+hYnRXQiIuI1XehERMRrZXro0nYRz8vDDz8MFL4NRy74+OOPAbf8wibtw0OW\nycLbk4AbJlOZqXht3rw5eGw7uuc3dLlo0aJI2lTWWAJdQYl0tuTCzjnqqKMAVyLMilN88803wWus\nPNjatWvT3GKJmyI6ERHxWtqXF1SrVg2ARx99FHAlpuzPorDkCktxz2uheK1atQD4+uuvi9vEfGVr\nKrCV9bKCzRUqVMj3XFucb4tjLRW4c+fOgIsOoxJ3n6cj9do+rxdccAGQuDTDFiYXVbgItG2V1KpV\nq4RzLOqz54uzCXE297ctBYn1c0AAAA8USURBVEj+99pCcnDJKEOGDAFcklXo/QFXAiy8Ee6rr75a\n0qblK+7+hviXF9h3TVRJKVpeICIiOSftc3RW1qtDhw6AS+O1wqvhzQut8KqVNLJzrdhzciRnRVfD\nP09g8ODBgFty0aBBAwBat26dcm6VKlUAty2MLYa134UU3Z577gnAa6+9BsCRRx4JuD4uDitZZWW+\nIDWSM/PnzweKF8n5wD7fGzZsANwmwrY4HAovP5VcAiwTUZwksvJrI0aMiK0NiuhERMRraY/o7Kpt\nW2k0adIEgBkzZgCwZMmS4Nx58+YBLtvJtuEwdndmcx62XQloW5m8WAFbiYYtKLZIzoS3kVm4cCHg\nimYbm0e10QuL5JL/D4CbV7JoxMq35RrLQu3WrRvg+qxly5b5vubxxx8H4LPPPgNcWbyoylHlkh9/\n/BGAL774AoDDDz88zuYkUEQnIiJey1hRZ5tPs7mfBx54oNg/w8pVWSZnpuVShlRZEXefl6a/Lcty\n9OjR+Z5jEUTy2izbzNPmUwtiBbc7deoEuG1jSiKb+zsbxd3fEH2fz5kzB3C5F1aasGPHjpG8v7Iu\nRUQk52SsMooVRt1xxx2B1PUs4O5mbczd2N1vmzZtMtU8kVKzTWrHjRsHQNeuXVPOKUrEFhaujGJz\ngBMmTADggw8+KFE7RaI0d+5cwEV0eX33R00RnYiIeK1Mb7watVwcT49b3H2ejv62UQubQwuvf7N6\nlMnzE8kb206bNi3lebszTicf+jubxN3fEH2f2xZJVg3LMl9HjRoVyftrjk5ERHKOLnQiIuI1DV2G\n5OIwQ9zi7nP1d7TU39FTnyuiExERz+lCJyIiXtOFTkREvKYLnYiIeE0XOhER8VqBWZciIiLZThGd\niIh4TRc6ERHxmi50IiLiNV3oRETEa7rQiYiI13ShExERr+lCJyIiXtOFTkREvKYLnYiIeE0XOhER\n8ZoudCIi4rXtCzqonWmjpz6Plvo7Wurv6KnPFdGJiIjndKETERGv6UInIiJe04VORES8VmAyioiI\njw466CAABg8eHDzXqVMnAI466igAFixYEH3DJCMU0YmIiNd0oRMREa9p6FJEckbTpk0BeO211wD4\n6aefgmMjR44E4Mcff4y+YZJRiuhERMRriuhyzDnnnAPAiSeeCED9+vUBOOSQQxLOe//99wHo0KFD\n8NzatWujaKIk2WWXXQCYMWMGAHvttRcAf/3rXwFYsmRJHM3KKu3btwfg+eefB2DUqFEADBgwIDhn\nw4YN0TdMIqGITkREvFZu69b8y6CpRlr00tnnu+++OwBjxowJnrMIbc2aNQDMnj074TUtW7YEXBQR\nTrE+7LDD0tW0QNx9Hvdn3KKz6tWrpxxbvXo1AMcffzwAjz76KAALFy4EoFGjRgCsW7euyO+Xa/19\n8MEHA/Dpp58C8M477wDQrl07ALZs2ZLR94+7vyH+z3jUVOtSRERyTpmeo7vqqqsA2GGHHQA49NBD\ng2Nnn312wrkWeRx++OERta7ss8yyAw44IHhu2LBhANxxxx0ArFq1KuE1devWBeDf//43AHXq1AmO\nDRw4EIBbbrklMw320BFHHAHA5ZdfDsD++++fcNz6d7/99kt57ZAhQwAXSZcrt+1G9bvvvgPc/wtJ\ntdNOOwFuNOOzzz4D4KyzzgIyH8nlsqpVqwLQpUsXAK677jrAjV6EXX/99UDiwv1MUEQnIiJeKxNz\ndC1atADc3a/93Ury2J1sQewO7csvvwRKNp/ky3h6mzZtABfRjR8/PjjWrVu3Iv0Mi9rsjgtg6dKl\nABx44IGlbWIg7j7P9GfcIrl77rknz+O///47AM899xwArVq1Co4l3wHb/4MePXoA8OSTTxa7Pb73\nt7ERi8suuwyA2rVrA7Bs2bIo3j4Qd39DdH3euHFjwH3WbQ65oGuMGTt2LAC9evUqdTs0RyciIjkn\n43N0NWvWBOCZZ54BXDHVsMqVKwMu08/uXD/66CMAjj766ELfZ7vttkv4Gbls++23/Votuh03blyx\nf4atNwpHdDbvUalSJQB++eWXUrXTVzfddFPwuF+/fgnHHn/8ccBV5LjzzjsT/m7rGgFef/11wGXP\n2jn2u5FUO+64IwDdu3cH3NrDqCO5XGKfz4ceeghwuRT2eX3hhRcAmDx5MuBGJADOPPNMwEWDNu+8\nadOmtLZREZ2IiHgtYxFd69atAXeV33fffYv8Wptf+/nnnwF3xxCes7A1Rfvss0/Ca+fNm1fCFvtj\n+vTpADRo0AAoWcUHmzsKq1GjBgB///vfAVddQhKFRxUqVKgAuPlNq8SxfPnyhNfYei/LUAO3tm79\n+vWAixQ3btyYgVb74eqrrwagYsWKQGLlE8kMi9QsknvjjTcAt1Yx2eLFi4PHdp2w73H7GbbuMV0U\n0YmIiNd0oRMREa9lbOjShhAKGrK04bH+/fsDrpCwlTgyK1euBKBPnz7Bc8lDllbY1ooW57J0DG19\n/fXXAHzxxRfBc7YY31K1JW/hZJG2bdsCbjjeFoFfeumlgEvEuvvuuwFXfBjcYv5BgwYB8OCDD2ay\n2V6wYuWzZs0C4OOPP46zOTnht99+S/i7DWUWhyW22XRVuimiExERr6U9orM7KksXTfbtt98Gjy36\nsruvwiRHcWF2F5GpO4Jc88cffwCwefPmmFuSfebOnRs8tlEKi+hsQbgt6rfFtXmVALv55psBGDFi\nROYa64FmzZoFj+1758gjjyzwNVa8PLzxanj0QorOloPZn1aM3JYj1apVC4CePXsCcMwxxwSv/eGH\nHwBXyMLK26WbIjoREfFa2iM6K8S88847Jzxv28HYXSoUHslVqVIFcPMczZs3TznHfu4rr7xSwhZL\nXmzhrd2VhRVnW5hcFF6akbyo3pbITJgwAXB3wVYm6eGHHw7OtYW2UjBbHA4wf/58AL755puEcyya\nuOuuuwD33RL+XfXt2xeAkSNHZqytPrK5e/sM//Of/wTctSAcwQF07do1eBxV8QNFdCIi4rW0R3T/\n+te/ALfIe+3atYBbZGxjskVx8cUXA3DrrbemHLPxdNt2ozg/VwpnW/sccsghKcesWHQy+53Xq1cP\ngCZNmgTHrGhxckat72yheGFsRMJKggH897//zUibfHPeeecFj+17xiI1Kyl14403AnDRRRcBrrxa\neFGzFaH46quvgPw/55LIsuJ33XVXAI499lggdbTCClfEUdRDEZ2IiHgt7RGdzT3YnyXRoUMHwG30\nacIZgFZ+SpFceticnGW2Nm3aNN9zre+Ti27bhou2djI8l2clrmyuxGfly5cPHv/tb38D8t9q6uWX\nXwbcZ16KzuaGrIg5pGYJ22fTorPkOaFnn302eGzZm9dee23Ca6Rg9nuwjFf7Dgn3LcDEiRMBRXQi\nIiJpVyY2Xk32559/Aqkb9lk1CXBzgenkyyaJVkh4jz32ABK3ObK7rvAGn+CyK+3urCD2+0ne+uSx\nxx4DXJQSXtNolWuSxd3nmfiM23wkwOmnn17gudZXHTt2THcz8uRTf59wwgkATJ06NXjO1isuWLAA\ncPNGNldn80l5sdd+9tlnQGJkXlJx9zdE/z1uG2hbYWb7Hrf+XbRoUUbfXxuviohIzsn4xqvFcfvt\ntwNuE9UtW7YkHH/77bcjb1M2sAjOtnGx+Z66desW+lpb52XzaTbHEZ73MGPGjAHcHJ3qCG5ja+N6\n9eoFQOfOnYNjdjdrfWV3uXauRd2SHsmVNYqz5lObs6aHVaXJ73s8DoroRETEa7rQiYiI18rE0KVN\nFNuO2Bbq2rCPbc8T3plWHCsVZYWCbbGsJTqEyyFZ8Ws7x5JEbNjGJvHr1KkDuO16wJX2+fXXX9P/\nj8hilhRxyy23pBy7/vrrAbj//vsBOO200wA3dBlHqrUvkosJl1aLFi0AlbgrLdu2x77HZ8yYAcCm\nTZviapIiOhER8VusEZ0VfrairBaRmGeeeQaAp556Cigbk5plkW2NZJGbpbSHt4vJjyWdDB06FIC9\n994bgBUrVgCuxBookktmW73cd999Cc+Hlwq8+eabAOy5555AahGE/JZdSOFsxKegJVKF+ctf/hI8\ntpKDY8eOLV3DcpQlv51//vmA2wLJNgyO87OuiE5ERLwWeURnCzgBHnroIQDOOOOMhHOuvPJKwM1r\nKJIrmN3RrlmzBoDPP/+80NfYAnFb3Ny+fXvAzd3ZVhpaQpA/G4GoXLky4Ja/TJkyJTjHIoZTTjkl\n4VybVwpv/CnFY/Oby5cvD56z0SGLIvJjv5fweVbI/Nxzz01nM71nn2krlG2jQv379wei24qnIIro\nRETEa5FHdHa1h9RIzrbHSJ7zkIJZSZ369esDrjxatWrVALdIGVwWZb9+/QC3Dc8HH3wAwCWXXAIU\nbX4v1yVnB9uf4Xkfy7IcPnw4AKtXrwbc4vvCIg/Jn0VyVmgC3Maqxub3DzroIMBtIXXdddcBsHHj\nxuBcm+sOl66Twg0bNgxw3+2WW5H8u4iTIjoREfFaZBGdZeTY9uphFpGcfPLJUTXHK9a3tkFt3759\nAVeCp23btimvefHFFwH3+9CWJMWXXL7L5tvCRYZtmx5j6+deeumlDLcud4wcOTLlOYsmbJ7f2Bo5\nGzW67bbbgmNxrvPKNq1btw4e27yorZ8rC3NyyRTRiYiI1yLbpsfGyrt06ZJyrHfv3kD88xW5uKVG\n3OLu89L09xVXXAGkzkWEK3WsWrUKcFHHkCFDAHf3G7Vs7u9sFHd/Q3r73DJTbdNlcBncFtlNmjQp\nXW9XItqmR0REck7G5+hsI89KlSqlHLPswGnTpmW6GSJp9/jjjwOuVusNN9wAwIcffhicY3Oh99xz\nT8StE0kf2wrM5vRt7RzAhAkTgPgjuYIoohMREa/pQiciIl7LeDKKFQu2kHfp0qXBsXbt2gGwcOHC\n0r5NWvg2cZwN4u5z9Xe01N/RS0efWyEJW64xe/bs4JgtNbDygXFTMoqIiOScjEd0timlFfzs3Llz\ncMw2AS0rfLn7yiZx97n6O1rq7+iVps8bNWoEuISTRx55BHAF+cFt2lxWKKITEZGcE9mC8WyQ7Xdf\n2SjuPld/R0v9HT31uSI6ERHxXIERnYiISLZTRCciIl7ThU5ERLymC52IiHhNFzoREfGaLnQiIuI1\nXehERMRr/w+vLFuBlmg8HAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 576x360 with 20 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rIiFhqAb9G20",
        "colab_type": "text"
      },
      "source": [
        "## 2. 创建网络\n",
        " \n",
        "定义网络时，需要继承nn.Module，并实现它的forward方法，把网络中具有可学习参数的层放在构造函数__init__中。\n",
        "\n",
        "只要在nn.Module的子类中定义了forward函数，backward函数就会自动被实现(利用autograd)。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yC0fsdqU9NhK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class FC2Layer(nn.Module):\n",
        "    def __init__(self, input_size, n_hidden, output_size):\n",
        "        # nn.Module子类的函数必须在构造函数中执行父类的构造函数\n",
        "        # 下式等价于nn.Module.__init__(self)        \n",
        "        super(FC2Layer, self).__init__()\n",
        "        self.input_size = input_size\n",
        "        # 这里直接用 Sequential 就定义了网络，注意要和下面 CNN 的代码区分开\n",
        "        self.network = nn.Sequential(\n",
        "            nn.Linear(input_size, n_hidden), \n",
        "            nn.ReLU(), \n",
        "            nn.Linear(n_hidden, n_hidden), \n",
        "            nn.ReLU(), \n",
        "            nn.Linear(n_hidden, output_size), \n",
        "            nn.LogSoftmax(dim=1)\n",
        "        )\n",
        "    def forward(self, x):\n",
        "        # view一般出现在model类的forward函数中，用于改变输入或输出的形状\n",
        "        # x.view(-1, self.input_size) 的意思是多维的数据展成二维\n",
        "        # 代码指定二维数据的列数为 input_size=784，行数 -1 表示我们不想算，电脑会自己计算对应的数字\n",
        "        # 在 DataLoader 部分，我们可以看到 batch_size 是64，所以得到 x 的行数是64\n",
        "        # 大家可以加一行代码：print(x.cpu().numpy().shape)\n",
        "        # 训练过程中，就会看到 (64, 784) 的输出，和我们的预期是一致的\n",
        "\n",
        "        # forward 函数的作用是，指定网络的运行过程，这个全连接网络可能看不啥意义，\n",
        "        # 下面的CNN网络可以看出 forward 的作用。\n",
        "        x = x.view(-1, self.input_size)\n",
        "        return self.network(x)\n",
        "    \n",
        "\n",
        "\n",
        "class CNN(nn.Module):\n",
        "    def __init__(self, input_size, n_feature, output_size):\n",
        "        # 执行父类的构造函数，所有的网络都要这么写\n",
        "        super(CNN, self).__init__()\n",
        "        # 下面是网络里典型结构的一些定义，一般就是卷积和全连接\n",
        "        # 池化、ReLU一类的不用在这里定义\n",
        "        self.n_feature = n_feature\n",
        "        self.conv1 = nn.Conv2d(in_channels=1, out_channels=n_feature, kernel_size=5)\n",
        "        self.conv2 = nn.Conv2d(n_feature, n_feature, kernel_size=5)\n",
        "        self.fc1 = nn.Linear(n_feature*4*4, 50)\n",
        "        self.fc2 = nn.Linear(50, 10)    \n",
        "    \n",
        "    # 下面的 forward 函数，定义了网络的结构，按照一定顺序，把上面构建的一些结构组织起来\n",
        "    # 意思就是，conv1, conv2 等等的，可以多次重用\n",
        "    def forward(self, x, verbose=False):\n",
        "        x = self.conv1(x)\n",
        "        x = F.relu(x)\n",
        "        x = F.max_pool2d(x, kernel_size=2)\n",
        "        x = self.conv2(x)\n",
        "        x = F.relu(x)\n",
        "        x = F.max_pool2d(x, kernel_size=2)\n",
        "        x = x.view(-1, self.n_feature*4*4)\n",
        "        x = self.fc1(x)\n",
        "        x = F.relu(x)\n",
        "        x = self.fc2(x)\n",
        "        x = F.log_softmax(x, dim=1)\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dyngRKea9n8U",
        "colab_type": "text"
      },
      "source": [
        "定义训练和测试函数"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sBOOlkZf9V8p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 训练函数\n",
        "def train(model):\n",
        "    model.train()\n",
        "    # 主里从train_loader里，64个样本一个batch为单位提取样本进行训练\n",
        "    for batch_idx, (data, target) in enumerate(train_loader):\n",
        "        # 把数据送到GPU中\n",
        "        data, target = data.to(device), target.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data)\n",
        "        loss = F.nll_loss(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        if batch_idx % 100 == 0:\n",
        "            print('Train: [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                batch_idx * len(data), len(train_loader.dataset),\n",
        "                100. * batch_idx / len(train_loader), loss.item()))\n",
        "\n",
        "\n",
        "def test(model):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    for data, target in test_loader:\n",
        "        # 把数据送到GPU中\n",
        "        data, target = data.to(device), target.to(device)\n",
        "        # 把数据送入模型，得到预测结果\n",
        "        output = model(data)\n",
        "        # 计算本次batch的损失，并加到 test_loss 中\n",
        "        test_loss += F.nll_loss(output, target, reduction='sum').item()\n",
        "        # get the index of the max log-probability，最后一层输出10个数，\n",
        "        # 值最大的那个即对应着分类结果，然后把分类结果保存在 pred 里\n",
        "        pred = output.data.max(1, keepdim=True)[1]\n",
        "        # 将 pred 与 target 相比，得到正确预测结果的数量，并加到 correct 中\n",
        "        # 这里需要注意一下 view_as ，意思是把 target 变成维度和 pred 一样的意思                                                \n",
        "        correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()\n",
        "\n",
        "    test_loss /= len(test_loader.dataset)\n",
        "    accuracy = 100. * correct / len(test_loader.dataset)\n",
        "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        test_loss, correct, len(test_loader.dataset),\n",
        "        accuracy))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w4k3M5ir9wCT",
        "colab_type": "text"
      },
      "source": [
        "## 3. 在小型全连接网络上训练（Fully-connected network）"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-GvwOcZ093wt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 259
        },
        "outputId": "0053b7ab-9c3e-4f0f-ba90-4c1c4f8796cc"
      },
      "source": [
        "n_hidden = 8 # number of hidden units\n",
        "\n",
        "model_fnn = FC2Layer(input_size, n_hidden, output_size)\n",
        "model_fnn.to(device)\n",
        "optimizer = optim.SGD(model_fnn.parameters(), lr=0.01, momentum=0.5)\n",
        "print('Number of parameters: {}'.format(get_n_params(model_fnn)))\n",
        "\n",
        "train(model_fnn)\n",
        "test(model_fnn)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of parameters: 6442\n",
            "Train: [0/60000 (0%)]\tLoss: 2.302146\n",
            "Train: [6400/60000 (11%)]\tLoss: 1.951977\n",
            "Train: [12800/60000 (21%)]\tLoss: 1.290410\n",
            "Train: [19200/60000 (32%)]\tLoss: 1.000708\n",
            "Train: [25600/60000 (43%)]\tLoss: 0.829704\n",
            "Train: [32000/60000 (53%)]\tLoss: 0.553799\n",
            "Train: [38400/60000 (64%)]\tLoss: 0.596026\n",
            "Train: [44800/60000 (75%)]\tLoss: 0.720949\n",
            "Train: [51200/60000 (85%)]\tLoss: 0.557444\n",
            "Train: [57600/60000 (96%)]\tLoss: 0.593830\n",
            "\n",
            "Test set: Average loss: 0.4253, Accuracy: 8735/10000 (87%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gMD_x-bd95rU",
        "colab_type": "text"
      },
      "source": [
        "## 3. 在卷积神经网络上训练\n",
        "\n",
        "需要注意的是，上在定义的CNN和全连接网络，拥有相同数量的模型参数"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "akf4bHFR-K8f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 259
        },
        "outputId": "85798dfb-fe72-49a8-b412-2899c9c1c623"
      },
      "source": [
        "# Training settings \n",
        "n_features = 6 # number of feature maps\n",
        "\n",
        "model_cnn = CNN(input_size, n_features, output_size)\n",
        "model_cnn.to(device)\n",
        "optimizer = optim.SGD(model_cnn.parameters(), lr=0.01, momentum=0.5)\n",
        "print('Number of parameters: {}'.format(get_n_params(model_cnn)))\n",
        "\n",
        "train(model_cnn)\n",
        "test(model_cnn)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of parameters: 6422\n",
            "Train: [0/60000 (0%)]\tLoss: 2.328946\n",
            "Train: [6400/60000 (11%)]\tLoss: 1.301481\n",
            "Train: [12800/60000 (21%)]\tLoss: 0.472606\n",
            "Train: [19200/60000 (32%)]\tLoss: 0.415483\n",
            "Train: [25600/60000 (43%)]\tLoss: 0.179762\n",
            "Train: [32000/60000 (53%)]\tLoss: 0.350131\n",
            "Train: [38400/60000 (64%)]\tLoss: 0.177829\n",
            "Train: [44800/60000 (75%)]\tLoss: 0.121229\n",
            "Train: [51200/60000 (85%)]\tLoss: 0.140334\n",
            "Train: [57600/60000 (96%)]\tLoss: 0.333439\n",
            "\n",
            "Test set: Average loss: 0.1266, Accuracy: 9615/10000 (96%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5a8fJUY-PZh",
        "colab_type": "text"
      },
      "source": [
        "通过上面的测试结果，可以发现，含有相同参数的 CNN 效果要明显优于 简单的全连接网络，是因为 CNN 能够更好的挖掘图像中的信息，主要通过两个手段：\n",
        "- 卷积：Locality and stationarity in images\n",
        "- 池化：Builds in some translation invariance\n",
        "\n",
        "## 5. 打乱像素顺序再次在两个网络上训练与测试\n",
        "\n",
        "考虑到CNN在卷积与池化上的优良特性，如果我们把图像中的像素打乱顺序，这样 卷积 和 池化 就难以发挥作用了，为了验证这个想法，我们把图像中的像素打乱顺序再试试。\n",
        "\n",
        "首先下面代码展示随机打乱像素顺序后，图像的形态："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bc-LhqDsAUNm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "outputId": "739b3aab-05ac-4489-94f9-d4af5b09f1dd"
      },
      "source": [
        "# 这里解释一下 torch.randperm 函数，给定参数n，返回一个从0到n-1的随机整数排列\n",
        "perm = torch.randperm(784)\n",
        "plt.figure(figsize=(8, 4))\n",
        "for i in range(10):\n",
        "    image, _ = train_loader.dataset.__getitem__(i)\n",
        "    # permute pixels\n",
        "    image_perm = image.view(-1, 28*28).clone()\n",
        "    image_perm = image_perm[:, perm]\n",
        "    image_perm = image_perm.view(-1, 1, 28, 28)\n",
        "    plt.subplot(4, 5, i + 1)\n",
        "    plt.imshow(image.squeeze().numpy(), 'gray')\n",
        "    plt.axis('off')\n",
        "    plt.subplot(4, 5, i + 11)\n",
        "    plt.imshow(image_perm.squeeze().numpy(), 'gray')\n",
        "    plt.axis('off')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAADnCAYAAACpF9m0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2dd5gUxbbAf7vktEtOAouSkyKgAiLB\nRJIkAnIBBa+KeEHkPngCXoIiwWtAQAmiLhf0CVxQARVQgpJRFBSFVUCigCCwgOTdrffHUD2zMz29\nPWl3e/b8vq+/3emurq4+U9PV59Spc2KUUgiCIAiCk4jN6gYIgiAIQqDI4CUIgiA4Dhm8BEEQBMch\ng5cgCILgOGTwEgRBEBxHbquDMTExjnZFVErFZHUbAkVknrmIvDMXkXfmE60yF81LEARBcBwyeAmC\nIAiOQwYvQRAEwXHI4CUIgiA4Dhm8BEEQBMchg5cgCILgOGTwyqE0bNiQxMREEhMTSU1NJTU11fjc\noEGDrG6eIAiCJTJ4CYIgCI4jxiolSiQWt+XKlYv4+Hif/QMHDgSgYMGCANSoUQOAf/zjH7z66qsA\n9OzZE4DLly8zadIkAF544QW/15IFhb7Ur18fgDVr1hAXF2da5uzZs5QoUSKo+p0m8+yygPOee+4B\n4IMPPgCgRYsW/PLLLxmeJ/K2z7/+9S/A9cyIjXW9t7ds2RKAr7/+2lYdTpM3ZJ8+Hiz+ZG4ZYSNY\nKlWqRN68eQFo2rQpAM2aNQOgaNGidO3aNcM6jhw5AsDUqVPp0qULAOfPnwfghx9+sN3ZBBe33347\nAIsXLwYgPj4e/eKi5Xr16lUASpQoQePGjQH4/vvv0x3LSTRv3hxwyePjjz+O6LVuu+02AL799tuI\nXicn0rdvXwCee+45ANLS0oxjks/QuYjZUBAEQXAcYdW8PE1SZqZBO+i3Iq3i//XXX4Yp5dixYwCc\nOXPGlkklp6NNsA0aNOD9998HoFy5cj7l9uzZA8C///1vAObPn8/GjRsB9/cwceLEiLc3u6FNStWq\nVYuo5hUbG8uNN94IQEJCAgAxMY6zTmVbtEzz58+fxS2JHu644w569+4NuEzcAHXq1DGODx06FICj\nR48CLsubfgZt3bo1LG0QzUsQBEFwHGHVvA4dOgTAqVOnbGleegROTk6mVatWgHtuZd68eeFsWo5k\n1qxZgNvRxR/aNb5w4cKAa/Jaax0333xz5BqYzXnkkUcA2Lx5c0SvU65cOZ544gkA4+00KSkpotfM\nCdx7770ADBo0KN3+pKQkHnjgAQD++OOPTG+Xk+nRowcAU6ZMoWTJkoDbSvDVV19RqlQpAF555ZV0\n58XExBjHHn744bC0JayD1+nTpwEYNmyY0Tm2b98OuBwvNDt27ADgvvvuA+DChQuGyjl48OBwNilH\n0rBhQwDat28PpDdBaUeXZcuWGV6cWrXX39WZM2e4++67fc7NaWiPtEjzzjvvGP9rE64QGs2aNSMx\nMRHA50X6lVde4eDBg1nRLMeRO7driGjUqBEAs2fPBlxTEuvWrQNg3LhxAGzYsIF8+fIBsHDhQgDu\nv/9+o65t27aFtW1iNhQEQRAcR0Rc5T/55BPWrFkDuN2wb7nlFgD+/ve/G2/8Fy5cMM75+eefAXjy\nyScj0aQcgXaY+fLLLwGMdVxKKZYvXw64TYgtWrQwnDH0m//JkycB11IE7TijtbcGDRoYbvPRjjaV\nlilTJlOu56kZ6O9OCI1HH32U8uXLp9v31VdfATB37twsaJEz0U4ZntYBcPVTbUI8d+6csV/v89S4\nwLX06T//+U9Y2yaalyAIguA4IqJ5QfrRGFxRGzR6cnrBggVA+kWDQnBUr16dYcOGAe43+T///BNw\nLTHQbz1//fUXAJ999hmfffZZhvUWKFAAgP/5n/+hV69eYW93dqRdu3aA+94jhdbstJs8wO+//x7R\na0Y72ongscceM54rycnJALz00ktZ1i4nMm7cOEaOHAm4F3NPnz4dcC2h8X7GAzz//POmdT3zzDOG\nZSdciOYlCIIgOI6IaV7ejB07FnB5wulFbdqV9YsvvsisZkQd2rvn1VdfNTQGPc+oXb23bdsWshZR\nqVKlkM53EjqupkbPx4YbPfdbpkwZfv31V8D93QmBUblyZcAd/syTadOmAbB27drMbJJjGT16NAAj\nR440li6tXLkScIfYunTpklFeL/6+//77jeeE9lLW2u6SJUvC3s5MG7y0c8YTTzxhTPxrt8u1a9ca\nbpRvvfUWIDHH7HLrrbcCblMXQKdOnQD7wUYFa8IRbzAuLo42bdoA7klwz0lt7W6sTVxCYGjZeq5L\nXL16NeBakyRkTNGiRQF4+umnAdczWA9anTt39ilftWpVwB1MWi/RAVi0aBHgjtoTCcRsKAiCIDiO\nTNO8NPv27TOiPOtFhH369KFPnz4AFCpUCHC7s+p4hoI5r7/+OuBS07WmFQ6NSy/QFWcaKF68uOl+\nvfxDm0i0GbxChQpGVgXt5BIbG2uYWnRkmStXrgCuhaDfffddhFof/XTu3NlIkaTZsGEDjz76KJDe\nWUzwj+6z2ukFXI4WAKVLlwagX79+AHTs2JG6desC7sg8SinDYqYjxXguhwo3onkJgiAIjiPTNS/A\niNCtQ+G8/vrrRjK+CRMmAO5I0OPHjxf3YRN0+C29MFkpxdKlS8NWv9a49JuUDumVE9Aakr73mTNn\nGi7Dnuj5Fa15paSkAHDx4kV27doFwHvvvQe4nGa0Rqzj6emcdQUKFJBYhkFg5aTx22+/SdzCANHO\nGdqlvVSpUuzfvx8w90HQYeW0y3y5cuWM5TnLli2LeHuzZPDS/PTTTwB0796dDh06AG5TYv/+/QFX\nOgodA1Fwo70Htap/4sQJY91csGjPRe0ZChiRUkaMGBFS3U5CT1jr+Hc6oao3OhD1J598AsDu3bsB\n2LJli2X9OoqMDlT622+/hdjinIlZckmNtxlRyBjtLKSdMz799FPDZL5v3z7A7TU4Z84cI5bt/Pnz\nAdfgpf/PDMRsKAiCIDiOLNW8NMnJyUYKFB1DS0czbt68uZGeQ8cmE3y5cuVK0M4tWuPSsQ6HDRtm\nmLRee+01wB2ZIyfx8ssvR6RebSLXmJm9BP9oU7l3/DxwawaSrDZ4tEORtgz4o3nz5oA7GWVaWlqm\nWhFE8xIEQRAcR5ZqXnrC+6GHHuK2225zNSh3+ibt2rXLyBsj+CcYZw39BqtjIuqI0EuWLKFr167h\na5xgiXZgEuyhI/IUK1bM2KfnGfUyHCHy6Hl3T+cumfMSBEEQBAsyXfOqUaMGAwcOBODBBx8EoGzZ\nsj7lUlNTAdciZVko64t2z9Z/O3fuHFAW6iFDhjBq1CjAHYVeh3nRMREFITtSokQJIL2XoY52nhPn\nZrMKHToqq4j44KUHJp0EceDAgcb6DDN0jMPx48cDwZnDcgJ63YX+W7ZsWaZOnQq41xadOnUKgMaN\nGxsRTHRUiAoVKhiu3roT6geAkDnoF4/q1atn6F4vuJfR6OgvnmzatCmzm5Pjad26dZZeX8yGgiAI\nguOIiOZVpkwZateuDcCbb74JQM2aNf2W37p1K6+88grgdnUVU2Fg5MqVy1hcq50t9Mr3atWq+ZTf\ntGmTkSJCp0AQMhetNZtpEkJ66tevb8SO1M8GHRHirbfekmgaWcBNN92UpdeXX40gCILgOMKieekQ\nIrNmzQJcb0lWo7K2T+sFsCtXrkyX3EzImM2bNwPuXFN6qQG45xl1mnlwz39pV9ZAnDuEyNKkSRPm\nzJmT1c3I1hQtWtTHsUvHPB06dGhWNCnHs379eiDrMlAEPXjdcccdgGuN0O233w7ADTfc4Lf8xYsX\nAZg6daoRfDeS4fKjHR0BQ3ts9u/f34iQ4c2UKVOYMWMGAHv37s2cBgoZoh02BMGJ6Ni0OsD6TTfd\nRJUqVQB3cN9IImZDQRAEwXEErXl16dIl3V9Pdu3axaeffgq400RoE6GkOQ8vOp7h2LFj00WDF7Iv\ny5cvB6Bbt25Z3BLnkJSUZEw3NGvWLItbI3iiLWnvvPOOscRp0KBBAEZqoEggmpcgCILgOGLMkowZ\nB2Ni/B90AEopx00qiMwzF5F35iLyznwiLfO4uDgAFi5caCxn+OijjwDo168fEJp/gz+Zi+YlCIIg\nOA7RvLIZIvPMReSduYi8M5/MknlcXJwx5zVgwADAnTkklLkvfzKXwSubITLPXETemYvIO/OJVpmL\n2VAQBEFwHJaalyAIgiBkR0TzEgRBEByHDF6CIAiC45DBSxAEQXAcMngJgiAIjkMGL0EQBMFxyOAl\nCIIgOA4ZvARBEATHIYOXIAiC4Dgs83lFa1iR7IzIPHMReWcuIu/MJ1plLpqXIAiC4Dhk8BIEQRAc\nhwxegiAIguOQwUsQBEFwHDJ4CYIgCI5DBq8cwJQpU1BKoZRi586d7Ny5k4SEhKxuliAIOZjVq1ez\nZs0a1qxZE9T5MngJgiAIjsNynVdmUaRIEQoXLgxA+/btAShVqhQAr7/+OleuXMmytjmZypUrA9C7\nd2/S0tIAqFWrFgA1a9bk4MGDWdW0qKR69eoA5MmTh+bNmwMwffp0AEP+/liyZAkADz/8MABXr16N\nVDOjjjx58tC0aVMAJkyYAMCdd96ZlU0SLJg8eTIATZs2Ze7cuUHXkyWDl36oPvfccwA0adKEunXr\nmpYtV64czzzzTGY1Lao4efIkAOvWraNjx45Z3Jroo06dOgD07dsXgG7dugEQGxtL+fLlAfeglVHG\ncv39zJw5E4Bnn32Wc+fOhb3N0Uh8fDxr164F4Pjx4wCULVvW+F/IHkyaNAmAp556CoBr166xevXq\noOsTs6EgCILgODJN86pZsybgeqPs1asXAAUKFAAgJiaGw4cPA3D+/HnAbd7q3r27YXpJSkrKrOZG\nBRcuXAAQ82CEmDhxIgDt2rULW52PPPIIAO+++y4bN24MW705hbJlyxp/RfPKXjRu3BhwmXkBNmzY\nwMKFC4OuTzQvQRAEwXFETPOKj48H4OWXXwagR48egMs5w5s9e/bQunVrwD0qay2rZMmSlCxZMlLN\njGqKFi0KwC233JLFLYlOvvzyS8BX8zpx4gTvvvsu4Jr/gvQOG9q5oEWLFpnRzBxFTIzj4uY6kubN\nm/P8888D0LNnTwBOnz7tt3zPnj0Nv4Z9+/YBMHTo0JDaELHBq0uXLgA8/vjjfsvom7jvvvsMs2HV\nqlUj1aQcR8GCBQGoVKmSz7HbbrvNeEEQs2JwzJgxA4BPPvkk3f5r165Zmqzi4uIA+OmnnwAM5w7P\nurZt2xbWtuYUtGNM/vz5s7gl0c3bb79NtWrVAKhduzbgMgP6Y+TIkZQoUQKAJ554AoAffvghpDaI\n2VAQBEFwHBHTvLTbsDcHDhzg22+/Bdyu8lrrArejhhA6R48eBWDOnDmMHTs23bGxY8eSnJwMwJtv\nvpnZTYsKUlJSgPT91w7aRF6sWDGfY0eOHAGQtY0h0qhRI7Zs2ZLVzYhaLl68aEvLrV+/PgAJCQmG\n6TxcWrFoXoIgCILjiJjmpe2aTz75JABffPEFAHv37uXEiRN+zytTpkykmpRjGTdunI/mJWQ+OnqG\n/m3opSKejB49OlPbFA2kpKRw9uxZwO0oVqVKlaxsUtQybtw4AOrVq8fu3bsB87mrQoUKAW7rWsGC\nBQ1NeNGiRWFpi2hegiAIguOImOal51sCfeNv0qRJBFojmLlsC5FHL8gfPny44Umrl4N4smPHDsDl\nqSgERnJyMuvXrwfggQceyOLWRCcVK1YE3FaDlJQUBg4cCLjD0Hny+uuvA27fh6NHj4Y93mSWxDbU\nsQq1aulJvXr10n3etGkTmzdvzpR2RTN2Y+wJ9tExOvv06QPAvffe61OmWbNmgLncdezC4cOH8/nn\nnwNw6dKlSDRVEIJCr836+OOPAYw1t9OmTePrr7/2Ka/Xbul4n5rx48eHvW1iNhQEQRAcR8Q1L71Q\nVi9kGzNmjE9EgtjYWB9zljY79uvXj9TU1Eg3UxACom7duixduhQwXwRuB23qevvtt8PWLsGFXhAr\nBE7u3K5hoXfv3n4jxTRp0oQRI0YAbhNh8eLFDTOhjnSiU57MmjUr7O0UzUsQBEFwHBHRvPLkycOt\nt94KwOLFiwFXXi5w2fS1VqXnstq0aWNoaEbDro/+Dz74IFOmTAEkQZ+QvdBvl1bx9KwcZbRzQdu2\nbVm+fHkEWphzkfx1waOXdLzzzjvGXK3uv3v37gVci8AbNWoEQKdOnQC44YYbjOe8duJ47LHHItbO\nsA5eefPmBVyD0UcffZTu2AsvvADAmjVrjFQPxYsXN/Z5J6PUmZQnTpzIoUOHAHfcN4k+EDhmD1Gd\n7VcibATOTz/9RMuWLQGXeQVg5cqVAFy+fNn0nL///e8ADBo0KPINzGHoZJTibRg8Onh6YmIi4PJ8\n1VF4/va3vwFw5swZAF577TUjsLQexGJiYozBTjt26OgzLVu2NGLZhgsxGwqCIAiOI8bKdTomJsaW\nX7Vet/Liiy8CMGzYMOOYNodod+Lk5GRDq9LuwQ0aNDBMgv/+978Bt4umVkkBVq1aBbjSrOg3AI1e\nJ+OJUspx+RHsyjxQtNOL2fd98803A7Br166Qr+M0mUdK3mbo6A+nTp1Kt79Dhw5Bmw1F3i66du0K\nwH//+1/ANT2hncTCmTXBafIG+zJfs2YN4IpDCPDSSy8ZWpg3tWvXNpww9NpcT81L83//93+AO8lq\nMPiTuWhegiAIguMIec4rV65cRrwrvUDtwoULDB8+HID58+cDGLbTRo0aGXMs2qljz549DBgwAHDb\nrnXOo6ZNmxpRCvQkrE4CCG6b6o033hjqrUQ1M2fOBKB///4+x3T8yWeffTZT25TT0NHkhfCjI/xr\nYmJiyJcvXxa1xpksWbIEwPBXsMqWULJkSR8/hZ49exo56jQ6S0IkEM1LEARBcBwha15PPvmkoXFd\nvHgRcL3d6yjyjRs3BlyLjcHlFqyjaes5ssTERJ9RXofOWbFiBStWrADc6aa15wvAkCFDQr2FHIHO\nmiwEh57Xvf/++wHX/EAgoZz69etnLPkQwo/WGnQ/r1mzpmFJePrpp7OsXU7CTv/U87bdunUzrGPa\ni3DhwoWRa5wJITtsHDt2zHDA0C7sSUlJRtxCHYzUEx2sd+LEiQARi6ARzZOrwfLrr78C6VNGaDd6\n/V2F4tLqNJnbkXezZs14/vnnAbjvvvsAl5nayqyil4HoaDLTpk2jSJEi6crowa9jx46GuTxQolHe\nofDGG28ArpcFnV7J39KFYHCavCG8MtdRNcaNG2es5brtttuAyJkIxWFDEARBiBpCNhseP37c0Lz0\nBOktt9xiHNfu8OvWrQNcC40PHDgARE7jEvzz888/A3DTTTcZ+yRNijVvvvmmz+T0//7v/3L+/Hm/\n52gNrUGDBkD6JQpfffUVADNmzAAIWusS/KOUkog8YUS7zz/++OOAS746JmcknTKsEM1LEARBcBwh\na17Nmzenc+fOgPst88SJE7z33nuAO5yIvAVlD/TbUocOHbK4Jc5GL+2wy4kTJ1i2bBkAgwcPBsI7\nFyOkJy4uzghwoHNRCcGjlydpDez9999nzJgxWdmk8ETYyK7k9MlVM3Tn+/TTT6lVq5a+JgDVq1cH\nxGHDm/r16xvxCB999NEM69y3b5/heeuZ9sR7DUw4iEZ5h4IO+l2sWDFjHWk4PW2dJm8Ij8w9HTXA\n5W2YWS8F4rAhCIIgRA2ieWUzROaZi115a2cknd78pZdeolixYoA724E2rSxZsoTjx4+Hu6mmRKu8\ng0VH9KlVq5YRkUdiG0bnM0U0L0EQBMFxiOaVzRCZZy4i78xF5J35RKvMRfMSBEEQHIcMXoIgCILj\nkMFLEARBcBwyeAmCIAiOw9JhQxAEQRCyI6J5CYIgCI5DBi9BEATBccjgJQiCIDgOGbwEQRAExyGD\nlyAIguA4ZPASBEEQHIcMXoIgCILjkMFLEARBcBwyeAmCIAiOw3LwSktLU2lpaSo2NlatWrVKrVq1\nSsXExGT6Nnv2bDV79ux0+44ePaqOHj1qeV5mCTGc3HXXXequu+5SZ86cUZMmTVKTJk0KWX4LFizw\n2ffPf/7TZ19qaqpKTU31lqECbF8rq+UXKJ5t37dvn9q3b5/x2ZNA5D169GifOmJiYtTmzZvV5s2b\nLc+tVauWqlWrVo6Qt50tJSVFpaSkBNzn77rrLqWfX6H+fhYvXuxYeQM0btxYNW7c2Oe37f3b1/+X\nK1dOlStXLmA55c2bVx0+fFgdPnzY2LdkyRK1ZMkSv+f4+47weOb4u6+A83mlpqYCkCtXLsCVpTQh\nIQGArVu3AnDHHXf4rTM1NdU41w4FChTg0qVLPvvLlSsHQOPGjQH4+OOPfco4OffO+vXradq0KQB3\n3nknAFu2bDHKNWjQAIBhw4YB0LNnT791pqSkkDt37rC10bsPeOI0mT/44IMKXP2nV69eAFSsWBGA\nSZMm+ZRfsGABAD169PBbZ1paGrGxrvfC+vXrA7Bjx46A2jV9+nSjPTo7cP/+/X3KOU3edp4p165d\nI0+ePJnbMIv2eOI0eYO9fF5bt261fG5bUbp0aQBOnDjht0xGz319bT2GeOJX5kopvxvX37oD3fQo\nn5qaGtT5gCpfvrwqX768AtTnn3+uPv/8c9NyxYsXV8WLF0/3+dChQ+rQoUPK6t6y6xasvEaMGBG0\nrPV26tQpderUqYDrP336tPF/Vssvs+T9448/Gv9b9fULFy6oCxcuqHnz5hlvmf5+M40bNw64HVkt\nv8ySd/PmzUPu31bfk91nVVbLLzNlbrWtWLEi5Dp69+4dksxlzksQBEFwHAGbDa3o0qULkN6Ep80n\naWlpxmf9f7CkpqbSsGFDwG2OeeSRRwBITEw01FMVpSq+P6ZNmwbAoEGDgjrf01wyZcoUAAYPHgzA\n7bffzjfffGN63pkzZyhWrBjgPJkHKu/p06cDLhmZmfHAZVq0MitakZiYCEC/fv1slXeavFNSUhRg\n2yyYnJwMQNGiRY19JUuWBODPP/8Md/NM6du3L3PmzAGcJ2+ApKQkBVCrVi2fY1quWs4AZ8+eBSA+\nPj4zmpch/mQumpcgCILgOMKqeWn0GzyYT3raObd8+fIA/PHHH8ax06dPA1C8eHHLSVWNE9+SgpX5\n6dOnOXToEOB2ErBLSkpKus/+nDuiUeZO8CAbOnQoAK+++qrPMafJW2teYF/7ChUtt/fffx/w7zwT\njf0bgu/j586do0KFCsb/keS5554D4OWXX/Y5JpqXIAiCEDXY9p/+7rvvAIiLiwOgWrVqPmWKFy8O\nBK5teaLPLVSoEAATJ05kxIgR6epftGiRzzVeeeUVwOU63rFjx6Cvn53Qb4JaOzbTiLSWpWVjRkau\n8vrYmjVrLOvwRmvFJUqUoFmzZn7PzWksWbKETp06+eyfN28e4F7WYHf5woYNG3z2WVlMsjNm2lbL\nli0B+OqrryJyTa25ar7++mtatGiRbt+1a9eMZ8qMGTMAGDBgQETak5Xo3/jdd9+dYVn9rDfjjjvu\nMHVr96ZVq1YArF271tj366+/AlC9enVjX4kSJQCYOnUqAE8//XSGv4+ImA3N2LlzJwD16tXj7bff\nBuDJJ5+0da5e51WgQAFj3/r16wHYvXt3urrGjRvHqFGjgJyl4mfE4cOHAfcapmDwHKzA/OHrNJl7\nylv/FmJi7N2CHoz69OkT0DVHjx4NwIsvvmjsq1GjBgC//PJLQHU5Wd6BcPLkSUqVKpVuX9myZQE4\nfvw4165dA9IPjmb7gsFzjZLT5A3Byzyjl97PP/8cgHbt2gXVrr59+7Jt2zbAbcoN5JkiZkNBEATB\neVgtbstoUV8oi5ABVbRoUVW0aFHb5ZOTk1VycnK6fZUqVVKVKlVKt2/RokVq0aJFjl5QeH1iO912\n8OBBdfDgQVW9evWA5Gz2PS1dujTg72vq1Klq6tSpQS0ozK6b2T0899xz6rnnngu6X193TQ5qO3v2\nrDp79qzt8lktv3DIO5xbjRo1VI0aNWyXv3btmrp27ZoCVGJiokpMTIwqeWckc/0cHzduXNAy1yG8\n7I4HgY4d/u5LNC9BEATBeViN2GYj5JUrV9SVK1csR8qFCxf6PaaDLoJ12CdNly5dgn4jyOo3nnC9\nJek3G/354MGDtu5fBy/23Of5nfp7A3rxxRdzjMx1u9944w3jHl5++WX18ssvB3zvOgipXa32+PHj\n6vjx4+n2zZo1S82aNSvq5d25c+eg+5j3Fh8fb6tcixYtVIsWLUK6VlbLLxSZb9y4MWRZnzx5Up08\neVItWLDA2Ld69Wq1evVqy/NKlSoVdpnbumnPzfuBpz/bVQNbtWqlvvjiC/XFF1+oc+fOqXPnzlnW\n768N0d7RPDfvwSs1NVWNHTtWjR071tiXmJiobrnlFnXLLbeYnq//z5s3r8qbN2+64/v371f79++3\nlKXnA9m7/LPPPutYmdvpR54/VKvNc+Dp1auX6tWrl1q2bJlatmxZyA8Nf/0/q+UXCXkDqmXLlqpl\ny5bp9nma+Dy3w4cPGya/5s2bBxUHMSEhIWqfKdOmTVPTpk1Ldx/Lly9Xy5cvz/B+vZ89do+VKlUq\nwwGrdevWIfVxMRsKgiAIjiMsrvJ2VqZrChUqxIULF3zO0+7DY8aM8VtXoJE7lAPdWrt3764A/vvf\n/1qW0+uu7KwVat26NStXrgRgyJAhAEyePNlWHVZlunXrBsDs2bONGGlOk/l1jZLLly8HHY/QDvPm\nzTNc6s+fPw9AkSJF/JavUKECR44cybBep8k7KyKafP311wDGWkTPZ0eg7vROkzdERuaeqarMnhF6\n34MPPgjA0qVLfeo4duyYsUTEO4LHJ598QufOnQH/MhfNSxAEQXAcAWteerTVq+K3bt0a8RXyVm//\n3lrfvn37qFy5MgCxsbGOe0u6nu+JXLlyWcYcPHjwIICxcDNv3rxhSzhZuHBh/vrrL7/HdSLQY8eO\nAS4N7MMPP+R6ux0lc88+vn37dgBuvfVWv+XtlPGHznwwd+5cn2N6kb1ewG8Xp2kCZs+Ue++9F4BV\nq1YFVefq1au55557QmuYBblXHxYAAByDSURBVNG0SFk/U3bt2gXAzTff7FNe/66LFi2aLjCEJ+FO\ncGvGjz/+CEC9evVE8xIEQRCiBDueQbGxsba9oFJTU9WaNWt8jnku/OzTp4/q06ePcWzMmDEBeQbZ\ndRXPai+fSHpjecu8RIkStsqvWbPG+H78eQt98MEHxv/aDfaPP/6ISpmb3cOmTZvUpk2bAvoeAFPP\nQn8eoP62G264Qd1www22y2e1/CLdvz03f96GoWw7d+5UO3fuVOfPn49KefuTuQ54YPVMAVTt2rVV\n7dq1jWMdOnRQHTp0UICaPHmymjx5smmgCKstXIuUI9bRAtk8HxS5cuVSuXLlCrquf/zjH1HX0aw2\ns7Ur7du3V+3bt7c8z5+LazBbw4YNHStzs/sZPny4Gj58uPF53rx5mfI7CGbLavlFun+bbbNnz1az\nZ8+OmExPnDihTpw4ERXytivzF154IWS5hRpxyXNr3LixKl26tCpdurRfmYvZUBAEQXAcmRZV3gyd\nGmLlypVGokmNmVt8oJG3lYMnVyM1Ibps2TIAOnTo4LfMBx98QK9evWzX2bdvX2P5w8KFCx0lcy3v\nihUr0rhxY8C9TGHv3r0AVK1aNSzX6tu3L4CRUl6zYMECw02/Zs2aACQlJdmq02l93AnJP725du2a\n4UrvNHlD5GWuU6eEkrBSO56dPHkSSP/88ydz0bwEQRAE52HHVuodpiWYzTPas/ektFm4qVCvRxTb\np0PZdGzKAwcOGPuswrxEu8wjJeerV6+qq1ev2irrOb8m8kbVr1/f1r3bdeDYu3ev2rt3b47s35Hs\n496bdwgqf1ugzxt/92XbbDhp0iQAhg8f7re8J56J4oJh/PjxgCszsjYb6mzJZm3QmYQ9zY/KwSq+\n5xovM/OhTsZ51113Gft0VmWd2C1QXnvtNcClwuuEjFaJFitUqADAkSNHDDOv09bWhdOkcvXqVcC1\n5i4Q0tLSiI0NzgjitD4eirz1s0r3Tb2e88CBAyG3K4A2OEreYC7zQCL0hOM8wDCNL1iwIKBr+Vs7\nKmZDQRAEwXEE7bChJ7V1fLtgeOeddwB4/PHHjX1mcRL1hLmeQLeLk9+SPFf1azZv3gxAkyZNwnpN\n/Yaj32jtxI30h9NkbtbH09LSAAxtqF+/fiQmJvqc+9tvvwF4RnQBoHnz5qxbty7sbS1TpgwAf/zx\nh7EvGuSdXWnVqhUAX3zxhaMdNgoXLqwAzp4961dj+uWXXwyHuNq1awPuKBwZoetMSUkJ+Vmiz9+w\nYYMRucmfzEXzEgRBEBxHxFzl9VzJvHnz/Ja5cuUKAPny5Qv2MpY48S0pFJk/9NBDgFsj+P7774H0\nbqda1lr24cKpc17vv/++Auu5PbucOnUKgBIlShjx4bQGFu6I9Vo7jNGvuA4hlP4daAT4QLJdeLrD\nm5GQkADAgQMHHCVvsJb5woULAejevbvpcat5qk6dOgGwZMmSkNto9l3Nnz9ft0E0L0EQBCFKCNbF\nMiOX9vPnz/uNF6bP3bNnj9qzZ09IbrBmdev/s9pFNdxurRnJ3J8LaqdOnUyP6eyz+vM333yjvvnm\nm4BlPm7cOMfKPNB7tdrS0tJUWlqarQy1wWwLFixQCxYsUNczD+Q4eftzeX/kkUdsnT9mzBjLOKp2\nnjlZLb9Iybxx48YBfRf6WeT5PFq9erVPuSFDhqghQ4b4refChQvqwoULQcnc0mz4n//8R4E7MoAn\nOvlgcnKyz7Hly5fTtm1bv/VqChUqBGBEZzDDzHHB+ziYmwZUDjIb1q9fn2rVqgHWiSztOL9kFN1D\nL0fQyxMAIzrF5s2bHSXz+Ph4BaFFB8hKnNbHPVP+eBPuiCaBsG/fPqpUqZJhOafJG+D6ABOwe3up\nUqWMiBehcvz4cWP5lL9rAemup5dZlSlTRsyGgiAIQpSQWSYVvdWpU8dH3Vy/fr0tFTXQa2W1up7Z\nZhUr+QV6ToUKFVSFChWiXubhlPOSJUvUkiVLTI/p6DLh3rJafpklb6s+fPjwYVt1rF+/3njWVK5c\nWVWuXNnWeaNHj3asvD1lHhcXF7Z+d939PsOtdevWqnXr1iF93/7uSzQvQRAEwXHYcpXv3LmzkWZ7\n0KBB6cpkNCflzYsvvsjo0aN99gfi1mqG5/lOddsG8zmv8+fPA1CkSBHAda+bNm0C0oeH8ubDDz8E\nXNH7H3jgAQA+/fRT43jBggUBuHjxot867IaD0S74ly9fdpTMzeSt5++2bNkSUF1z584F4JFHHmH7\n9u0A3HrrrQHVYTaXrP/X8wLXrl1zrKv8dach8uTJY2SV0P00UPLnzw/A5cuXw9Q6F9E2j34935/l\nwvlAs1ikpKRQsmRJwNzvIVCsnjP+ZG45eM2YMUMBzJw5kx9//NG0zNmzZ4mPj/fZr3+8u3fvBuBv\nf/ub3+ucPXuWNWvWANCxY0cg+EHs559/pk6dOoAzO5odhw2vdAH6PBYvXgxA165dbV3Lu8NYrQGz\nSpPy+++/c8MNN+j2OErm15MOGtErMkIPGgBNmzYF3I5Hq1evNo7piBz9+vULT0P94DR56/5duHBh\n/vrrL9Myf/75p/FgNMOzz/vjqaeeMup46aWXgm4vpH9Bd5q8wTpqj8bzmTJ06FAAXn311YCuU7Vq\nVR9HMK2ovPjiiz7l7So+/mQuZkNBEATBcVhqXpF2Ix44cCDgikr+ySef+C03ZMgQACZPnpxhnZ5J\nLJ1oNrz11lsVZBwZPpTozgBTpkxh8ODBfo9XrFgRgMOHDxvX09qzXpWvcfKbaThj7WnTuqcGpiMX\neMssXORkeZuxYsUKANq0aWNZLthpCqfJG6Bfv34KfJOgehJK8luP6O+W2nCgdUoySkEQBCH6sHKx\nNHNRt+O23qVLl7C5ZIayZbWLajCblq+nK6p3dAwz11Or5G6bNm2yJa9y5cqpcuXKZViuYcOGqmHD\nhlEh82D7VmxsbMj908q1Plr7uNnzw05ki2Ci7cgzJbQ+bpVc0m4yyXAkF/Z3X6J5CYIgCI7Dcs6r\natWqClyhU/xRt25dfvrpJ5/93jbljGzMnnNVVuUyuo4nyoH26VDmBLRnZ61atQD3fJWev/LGM1sz\nBD9/5onTZG4mb+16rV2xJ0+ebMy7mqHDp1nNKdjl5ptvBkjn3XvixAkASpcu7VM+GuTtjT8vtEuX\nLgFQoEABW9cK1pXeLG+axmnyBnOZe4fmO3z4sN/nBNjP8TVgwAAApk2bBth/plhlDPAn86BTotx/\n//2AK1GbFVaDi16rpF2O7bJmzRruvvvuDMtFS0czwzsWWGpqquEooGXj2XH092C1PkwnFjWLjfjQ\nQw+xaNGiDNvlNJmr6z+A2NjYgNzb09LSjOSTZsc0/sqEC6fJ26p/f/fddwA0bNgw5HWfkcJp8gZr\nmXsO8IE6gelnkE7/E46XX0/0muKpU6eKw4YgCIIQHVhqXoIgCIKQHRHNSxAEQXAcMngJgiAIjkMG\nL0EQBMFxyOAlCIIgOA4ZvARBEATHIYOXIAiC4Dhk8BIEQRAchwxegiAIguOQwUsQBEFwHJaD16xZ\ns9SsWbOMsPapqakqJiYm3XbkyBGffYFulSpVUkeOHAmorjFjxqgxY8b47N+2bZvxf2YJMZxMmTJF\nTZkyxbYcuJ42IFCZjxo1SqWlpam0tDTT44ULF1aFCxdOt89fH/Dcl9XyCxTPe2rXrp1q166dz/15\nyqhevXqqXr16lrJdvHix6X5ved94443qxhtvVDExMergwYPq4MGDAX+PWS2/QClVqpQqVaqU7fv7\n5z//qf75z38GLJcmTZqoZcuWqWXLltk+Z/fu3Wr37t0++5VSjpU3uGIb6q1q1aqqatWqln3c3+/c\nc3vwwQdN93ufm9HvZc+ePWrPnj1BPVNsB+b1jg6vGTFiBBMnTrQtyLi4OCMzc8GCBQG4ePGi7fM1\nOmL6fffdB0BSUpJPmWgLoqnxyjKqzwtbG0LJquo0mXvKu0KFCgAcOHAAcAca9Yxybid4qaf87JQ/\nevQo5cuX99m/ZMkSANq3b++3DifL2x9WQY/DQSj1O03eYC7zSZMmATB8+PCg6vR8jofyDHrvvfcA\nuOmmmwBo2bKlT5mwR5UPN94/8nz58gFw5cqVgOopUqQI58+fB6Kno2UHBg4cyJtvvml6rHXr1qxc\nuRJwnsyDlbdnOhm7A713H/eMnN62bVsAli9fnmE9qampRvqVxx57LEfIO7vgtP4NofXxQF9iDx06\nBEClSpWCuaQp/mQuc16CIAiC4wir5hVoDp6qVasCrmSJa9eu9Vvu7NmzAMTHxwNQuXJlw7TjjZdJ\nzXFvSYMGDVKAXy3HGzOZm73hh5oXyW4dTpN5z549FcD8+fNtlbdjBly1ahX33ntvSO0ye+t95ZVX\nABgyZIhj+3igz5SlS5cC0LFjR2OfzpcWTtOiXVOi0+QNgct8xIgRAOmmg7zzB2YmonkJgiAIUYMt\nzWvhwoV0797ddqUpKSnGBLROYW4XrY1pBwx/b7h58+YF3GnTt23b5lMmJ7wlacqXL8/Ro0fD3ZyA\ncZrMg5V3xYoVDaehQJkwYQLgmp8Fd8ZYbzzn1SDnOGz4o2jRogAkJyeHrT2B4jR5g1vm8fHxhhXL\nLteuXQMgT548ETnv119/BaBKlSqAudVONC9BEAQharDlSjJhwgTjLXDUqFEAlu7xVvMBefPm5erV\nq+n2mbl+79u3z28dU6dOZejQoQCGl5bWxDZt2mR5L07CzhyinTkYu5jVlZCQAMDBgwctz9VvTtGA\nlUzDsTRh5MiRPtfT15o6dSoATz/9tLGvUaNGPnVoj1onY2fuauPGjQDceeedIWtcZtfT1gqzpQrR\nxtmzZwP2S/CnOV27ds1Sq9LHmjVrBsCGDRtMy1WvXh2A+vXrAzBu3DjAPc5YopTyu3F9AWxGW/36\n9VX9+vWNz48//rhleb1QznNfo0aNVKNGjWxdz3NLSUlRKSkppses7i27bp7tr1KliqpSpUrAMsnK\nLavlF4q8A9lWr17ts+/kyZPq5MmTClBxcXEqLi4uJFn669uLFi3KcfKO9DZ//vyo7N+hyLxQoUIB\nn9O8eXPVvHlz02OXLl1Sly5dCrhOf/clZkNBEATBcdhy2DBz2/UXcUOjJ6O9zRvNmjUzVMgdO3YA\nbpUxIzxVXu3QUbNmTb/llYMnV83Qk5ta1Q4GM7PBlClTABg8eLBP+UDNkk6TuZW8e/fuDcD7779v\netxf/922bZth6qtbty4AP/30k6322DXr6Gtu3749auT9xhtvAPDss88GXb82LWrnjowI1O3eaf0b\n3DI3czLSUWWOHDkSdP2BmiIDDUDhT+aieQmCIAjOw46tdNmyZX7t8A8//LAtu2XHjh1Vx44d1Tff\nfGPsS0pKUklJSQpQPXr0UD169AirHTurbc2ZaZ/++OOPbZVr3bq1at26tWUZf3OI0SjzfPnyqXz5\n8qW757Zt26q2bdsanwsUKOBzn4cOHbIlv3Hjxqlx48aZlsubN6/KmzdvjpJ3sPdZsmRJW+W859/N\nttjY2Bwjb0+Zb9myJej71puet7py5Yqt8pMnT1aTJ08O6Zp+7yvQjrZ37161d+/eDC/ob7I5JSVF\nnTt3Tp07dy7om/nXv/7l9zq33357VHQ0z23UqFFq1KhRxuc33ngjYJlZObboSNChdmwnyjwQ2XnL\n79ChQ6aD2IQJEwyZvvfee+q9996z/V3orXjx4n6P7d+/P6rkXbJkSduDk78tOTlZJScnmx7Lnz+/\nyp8/f47s3/5k/v3336vvv/8+JFkcPnxYHT58OGhHjGPHjhn/9+vXT/Xr1y/dbyQjmYvZUBAEQXAc\nYYltGOxao4oVKwKu9CblypUD3OsuzNbRlClTBoDff//d51qebdCryOPi4hw7uRqOeIRmmH1Xep+W\nudl1raLKe6IcNqEd6T7er18/ABITE42Jbf2b2717NwD16tXzuU5MTIzP92DWhmiVd7CYOWDMnTsX\ngFOnTgGu2JDB4jR5A1yf2mHBggW2yq9evRqAe+65x2+Z48ePU7ZsWb/H9fP82LFjfstYRa33dC7x\nJ3PRvARBEATHYal5nT9/XoEr8ZheMa1XontGXAjkrTSYHDHe9aekpBjx4GbMmGFaFiBXrlyOe0ty\nar4jLXenyfyBBx5QAJ999pnh1t6jRw/A5ip/E86fP28sFfGkVatWAKYZFKx+Q2bHtBYXGxvrKHk7\ntX9rjS4mnFlfMwlPmdt5Vme0RARcSzX0UhG7BBIn0U52ENG8BEEQBMdha85r3rx5xqI/HUvQbOT2\nXKx25513Au7YZJ7HzEb/CxcuAFCoUCG/7Qk0e60T7dNWb6Y7d+4E0s+RtG7dGsDIZOyPhQsXAgSU\nHSAYnCZzM3nb1YK8FyB7Hps8eTIAnTp1Alxpzu289UZ7Hw9U89KLsQN9y48UTpM3mMv83LlzgMuq\n5o9Lly4ZsWL1/JeOS5s3b16fufJ33nmH/v37Z9ienj17AvDhhx/aar8/mYfssBGMGTCcdRQrVgyA\nM2fO+NQVLR0tUOw+JPVxPZFdokQJ47juF4FaSZwm80D7uDYl6gCidhk/fjzPP/88AHXq1AHg559/\nDqgOjZ3J7OxKOPr3wIEDAeuErUopo+9qM5U2W4WC0+QNgffxPXv2AFCtWrWArlO8eHFOnz4dRAut\nEbOhIAiCEDWEnEcjFK1Lq42h1KE1Lk8CTZwWbVjJ0ywWpafGBa4Yc1YaVzjTsGQX4uLiDFOKN573\nGajGpfGMHac1Lm1ieeutt4xreJvbAWbNmpWu/P79+6NK9oFipXFpx4qWLVsa+wLVuLp06QLAxx9/\nHHjjsjF58uTxKwvP/hSoxmXneVC7dm127doFwJdffgnAfffdZ1nvmjVrLI+L5iUIgiA4joDnvJYt\nWwZAhw4djM/6fyt0dPkiRYowbNgwAB5++GHAFSXdzLU4VKLVPh0O7C6CNnMSscJpMo+UvLVjk3bm\n8NSWrN5UA41CL/I2Jy0tzVak+EuXLgFQoEABW/U6Td7glnn37t0Nx61A0VqQthropUr+CKd1JiSH\njZSUFJ566inA5VFiB7O1WZ6fA0GbBrVzhhl9+vQBXJ6RY8eOBWDMmDGO7WhZjfYuDTR7rdN+3J59\n3CrCiB3MUkPoKATHjx/3Kf/LL78A6bNQ2/FElAgbWYfT5A3hlbnuszVq1LBVPhwmWHHYEARBEKIH\nu9GIBwwYoAYMGGB81qHuQ0nn0LdvX9W3b990+w4cOKAOHDgQcF06UrdnNO6sjuYcrgjQ2X3zjEif\n1fILVt5169b126eef/75oGWTO3dulTt3btN6PffVqVNH1alTx1Yf92xrVssvO/fvH374Qf3www8h\n15OWluZYeXvK3PtZ67m1atUqy58jgPrwww/Vhx9+KFHlBUEQhOgkLFHlzbATpWD8+PEAjBkzJtjL\nmM4zNGvWDID169fnaPu0GbVr1wZg165dQc9DWp2nHDYnEOk+rgm3a7tTY0lm5pyXjnISaBR5q2g/\nTuvfEDmZh9rHPd3nzZ7jf/75JwAlSpSQOS9BEAQhOgj6dVDHGtOxx3wq9jMK2w0FZddF29szbPHi\nxXTt2jXD+p2IjmP42WefGTKsXLkyAAcOHPApr73Y9u3bZ3j7aO8fCPxNSYfl0eeZvS1FE2aaVfXq\n1QH49ddfLc+z47lopbndeeedrFq1CnC7cefPnx+Av/76yzMEmr2bcQDaM7No0aIkJSUFdO4bb7wB\nwLPPPgtAQkJC0Hm7rOKr5jT89dGdO3eGbE3wDCbh/TuZO3cuJUuWBCz6uNVE34gRI9SIESMCnnRr\n1aqVrVTndraUlBTVs2dP1bNnT2Nf0aJFbZ2b1ROlwWzByi0cstbb2rVrM7yWv+tltfzCKe/+/fur\n/v37mx7r3r27Wr58uVq+fLmlrLZv3662b99uWebo0aOWx8+dO6fOnTsXFfLWjl6B9snq1auHrX+X\nLFnS8vj06dPV9OnTo0LeSimClVPt2rXDJvPmzZsHfa6/+xKzoSAIguA8wjFi58qVS+XKlcv43LZt\nW8vyM2fOVDNnzrQss2XLFrVly5aAR+l33303R74lWW0JCQm2yr311lvqrbfe0rJTytWgsLwlZdct\nEvIGX+00Li4urHWLvAPfOnXqpDp16qQAtXTpUrV06dKo79+ZKXMz68CZM2fUmTNnFKBatGihWrRo\nYauuWbNmZShz0bwEQRAEx2HpKl+6dGkFcPLkybDFqkpKSqJmzZo++/W+QCdqdcxEPamdO3dux7oR\nA1xfEEmuXLmM+GHTpk1LV8ZuXEJNOHKu2UU5zJXYzI3Ye/L/zJkzlqHJNBn9Ruz8hnQ/1jH3wDcT\ngGdCV6f18YceekiBy7HKH2fPniU+Pt52nRcvXqRgwYKhN84GTuvfABs2bFAAd911l9GXevfuDdhP\nCKnR4aFGjRoVdJxEM3R/PnnyJADlypUzctZVqFDBXOZW6uaxY8fUsWPHLJ0Bqlatqvbv36/279+v\nmjVrppo1a6YA1aZNG9WmTRtbKuIzzzyjtHNIfHy8io+Ptyxfvnx5v8c8TQFZra6HouIH6oBRpUqV\ngMp7RsUI55bV8gt0S01NzVAWq1evNv7X5sCuXbuqtm3bqrZt26qPPvpIffTRR5Z1fP3110pfa9Cg\nQWrQoEG2ZZovXz6VL18+4/OECRMcK2/dbs+oFZHYgjEJRmP/9pS53WeKJpLfT0ab51SHv/sSs6Eg\nCILgOCxtSd9//z0A7du391tm79693HjjjT77V6xYAcCxY8cAlxrozYgRIwCYOHGi3/o9V3GXLl0a\ncIXj12lVvE0w7dq181uXkwjUzLdv3z5b5fRarQceeCDgNlld2zMyupOwY3695557jP/NvhdtujZD\n99+YmJiA18PpdTBXrlxJt3/q1KksXbo0oLqyG3bSlYTCb7/9Fra6SpYsaUR7cDJ2nylWiWjNGDp0\nKACvvvqqZTnv573Vc2PlypWm00ueiOYlCIIgOA5bsQ0rVarEoUOHTMusWLGCNm3apNt3/PhxY7V8\nsITDQUQ5cHLVM79Udk31npCQAMDBgwd9jjlN5tfnu9LJ2k7fC8f3YyVHuzhN3rp/200Wmd1wmrwB\nRo4cqQC+/fZbI2pLqHg6xllhZXmziz+ZO6/3CIIgCDkeW6+OnlqX91vpuXPnfMqXLVvWxw46Y8YM\nAAYMGGDekOv1Xb58Od3njIjW+HrheOu3q70GKsMOHToA8Oabb9oqn53ZuHGjzz7vPq2UYt26dQC0\naNECcMl0woQJAIwcORLIWI7ffvst4J4PHjVqlK02hjOlenbBSuuyq5UdPXoUgPLly1uWS0tLy/Ca\n0Yz2FfDUurz71JYtW2jUqFG6feCeS/eem/Kndb3wwgsANG7cGLCvcW3duhWAO+64w1Z5AFsulhlt\nHTp0UB06dDA+N2vWzMe9V2/r1q1L51Lvb5s0aZKaNGmS3+NWMeMuXbqkLl265Gi3VrPNjlt3Rluw\nsRMzuq6uN6vlF0559+rVS/Xq1UsBqlu3bqpbt27p7tefTJ988kljXygxPnfs2KF27NhhekwvKclq\n+YVT3mlpaSG70CcnJ6vk5OSAz9NRIPxtCQkJKiEhwXHyzkjmVtvGjRuN/+fMmaPmzJljfO7fv7+t\n59HgwYPV4MGDTY/t3LkzpGdKznwVEQRBEByNpcOGIAiCIGRHRPMSBEEQHIcMXoIgCILjkMFLEARB\ncBwyeAmCIAiOQwYvQRAEwXHI4CUIgiA4jv8HHYibkA7UByYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 576x288 with 20 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UGWDLRtEbTq7",
        "colab_type": "text"
      },
      "source": [
        "重新定义训练与测试函数，我们写了两个函数 train_perm 和 test_perm，分别对应着加入像素打乱顺序的训练函数与测试函数。\n",
        "\n",
        "与之前的训练与测试函数基本上完全相同，只是对 data 加入了打乱顺序操作。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pb-vBHD9bW0r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 对每个 batch 里的数据，打乱像素顺序的函数\n",
        "def perm_pixel(data, perm):\n",
        "    # 转化为二维矩阵\n",
        "    data_new = data.view(-1, 28*28)\n",
        "    # 打乱像素顺序\n",
        "    data_new = data_new[:, perm]\n",
        "    # 恢复为原来4维的 tensor\n",
        "    data_new = data_new.view(-1, 1, 28, 28)\n",
        "    return data_new\n",
        "\n",
        "# 训练函数\n",
        "def train_perm(model, perm):\n",
        "    model.train()\n",
        "    for batch_idx, (data, target) in enumerate(train_loader):\n",
        "        data, target = data.to(device), target.to(device)\n",
        "        # 像素打乱顺序\n",
        "        data = perm_pixel(data, perm)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data)\n",
        "        loss = F.nll_loss(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        if batch_idx % 100 == 0:\n",
        "            print('Train: [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                batch_idx * len(data), len(train_loader.dataset),\n",
        "                100. * batch_idx / len(train_loader), loss.item()))\n",
        "\n",
        "# 测试函数\n",
        "def test_perm(model, perm):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    for data, target in test_loader:\n",
        "        data, target = data.to(device), target.to(device)\n",
        "\n",
        "        # 像素打乱顺序\n",
        "        data = perm_pixel(data, perm)\n",
        "\n",
        "        output = model(data)\n",
        "        test_loss += F.nll_loss(output, target, reduction='sum').item()\n",
        "        pred = output.data.max(1, keepdim=True)[1]                                            \n",
        "        correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()\n",
        "\n",
        "    test_loss /= len(test_loader.dataset)\n",
        "    accuracy = 100. * correct / len(test_loader.dataset)\n",
        "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        test_loss, correct, len(test_loader.dataset),\n",
        "        accuracy))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tD7NwkpGvppG",
        "colab_type": "text"
      },
      "source": [
        "在全连接网络上训练与测试："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "obw8E60Ovtfx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 259
        },
        "outputId": "44d5216f-3729-47f7-93c4-10c21119cff2"
      },
      "source": [
        "perm = torch.randperm(784)\n",
        "n_hidden = 8 # number of hidden units\n",
        "\n",
        "model_fnn = FC2Layer(input_size, n_hidden, output_size)\n",
        "model_fnn.to(device)\n",
        "optimizer = optim.SGD(model_fnn.parameters(), lr=0.01, momentum=0.5)\n",
        "print('Number of parameters: {}'.format(get_n_params(model_fnn)))\n",
        "\n",
        "train_perm(model_fnn, perm)\n",
        "test_perm(model_fnn, perm)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of parameters: 6442\n",
            "Train: [0/60000 (0%)]\tLoss: 2.264858\n",
            "Train: [6400/60000 (11%)]\tLoss: 2.041490\n",
            "Train: [12800/60000 (21%)]\tLoss: 1.552290\n",
            "Train: [19200/60000 (32%)]\tLoss: 1.093537\n",
            "Train: [25600/60000 (43%)]\tLoss: 0.799383\n",
            "Train: [32000/60000 (53%)]\tLoss: 0.847221\n",
            "Train: [38400/60000 (64%)]\tLoss: 0.756004\n",
            "Train: [44800/60000 (75%)]\tLoss: 0.723714\n",
            "Train: [51200/60000 (85%)]\tLoss: 0.438829\n",
            "Train: [57600/60000 (96%)]\tLoss: 0.474032\n",
            "\n",
            "Test set: Average loss: 0.5751, Accuracy: 8348/10000 (83%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pZ9D2l0qv63d",
        "colab_type": "text"
      },
      "source": [
        "在卷积神经网络上训练与测试："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrKEZas2v-4c",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 259
        },
        "outputId": "39bdfdd5-c6ac-44b2-a316-28631fcb451e"
      },
      "source": [
        "perm = torch.randperm(784)\n",
        "n_features = 6 # number of feature maps\n",
        "\n",
        "model_cnn = CNN(input_size, n_features, output_size)\n",
        "model_cnn.to(device)\n",
        "optimizer = optim.SGD(model_cnn.parameters(), lr=0.01, momentum=0.5)\n",
        "print('Number of parameters: {}'.format(get_n_params(model_cnn)))\n",
        "\n",
        "train_perm(model_cnn, perm)\n",
        "test_perm(model_cnn, perm)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of parameters: 6422\n",
            "Train: [0/60000 (0%)]\tLoss: 2.300903\n",
            "Train: [6400/60000 (11%)]\tLoss: 2.282535\n",
            "Train: [12800/60000 (21%)]\tLoss: 2.261807\n",
            "Train: [19200/60000 (32%)]\tLoss: 2.111837\n",
            "Train: [25600/60000 (43%)]\tLoss: 1.717916\n",
            "Train: [32000/60000 (53%)]\tLoss: 1.320999\n",
            "Train: [38400/60000 (64%)]\tLoss: 0.960259\n",
            "Train: [44800/60000 (75%)]\tLoss: 0.961738\n",
            "Train: [51200/60000 (85%)]\tLoss: 0.636504\n",
            "Train: [57600/60000 (96%)]\tLoss: 0.507474\n",
            "\n",
            "Test set: Average loss: 0.6227, Accuracy: 8003/10000 (80%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ko050KVCyJWp",
        "colab_type": "text"
      },
      "source": [
        "从打乱像素顺序的实验结果来看，全连接网络的性能基本上没有发生变化，但是 卷积神经网络的性能明显下降。\n",
        "\n",
        "这是因为对于卷积神经网络，会利用像素的局部关系，但是打乱顺序以后，这些像素间的关系将无法得到利用。"
      ]
    }
  ]
}