A_1_x1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "A.1.x1.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMh7lQmFUVlce6FzP4RHHPq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1860be9fcc2540d3b0a44939cc8f97e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_af6b2cd852ce464297b0c87b793b4f7b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_159b79967bac4f689390134a61b0bcaa",
              "IPY_MODEL_84ad006a4abd45d6af2763acbde12f4b",
              "IPY_MODEL_ba3eec9dac204faea889cc5fe34ef881"
            ]
          }
        },
        "af6b2cd852ce464297b0c87b793b4f7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "159b79967bac4f689390134a61b0bcaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ff63067cfa3543e2bcba6b70d98ce583",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcf9d3ed27d5468e939ebaa0eeac02ec"
          }
        },
        "84ad006a4abd45d6af2763acbde12f4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5bce7fa5356840d3b0eb2ae95cbdbd86",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 170498071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 170498071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e553d12bca3643bfa6dc4635e2d378c8"
          }
        },
        "ba3eec9dac204faea889cc5fe34ef881": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f3792f6c6e8d4ada8ea85333b7dc95f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170499072/? [00:02&lt;00:00, 66037357.28it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bce97236b2354156ac9404aa096cb993"
          }
        },
        "ff63067cfa3543e2bcba6b70d98ce583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcf9d3ed27d5468e939ebaa0eeac02ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5bce7fa5356840d3b0eb2ae95cbdbd86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e553d12bca3643bfa6dc4635e2d378c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3792f6c6e8d4ada8ea85333b7dc95f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bce97236b2354156ac9404aa096cb993": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/secutron/Practice_Ignite/blob/main/A_1_x1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "na4WYXxkfI4C"
      },
      "source": [
        "### A.1.4. 데이터 분배 확인\n",
        "\n",
        " 본 절에서는 [기존 코드](https://github.com/secutron/Practice_Ignite/blob/main/A_1.ipynb)를 수정하여, 학습 데이터가 분산처리를 위해 생성된 각 프로세스로 배분된 상황을 확인한다. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HdhoeHGgsimM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b4b5a40a-5132-4767-a17f-98ce4f711484"
      },
      "source": [
        "import os\n",
        "\n",
        "gpu_gtg = False\n",
        "if int(os.environ.get(\"COLAB_GPU\")) > 0:\n",
        "    gpu_gtg = \"COLAB_GPU\" in os.environ\n",
        "\n",
        "tpu_gtg = \"COLAB_TPU_ADDR\" in os.environ\n",
        "\n",
        "if tpu_gtg: # tpu\n",
        "    print(\"TPU\")\n",
        "    #VERSION = \"nightly\"\n",
        "\n",
        "    # https://github.com/pytorch/builder/pull/750\n",
        "    VERSION = \"20210304\" # was 20200607\" \n",
        "\n",
        "    !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
        "    !python pytorch-xla-env-setup.py --version $VERSION"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TPU\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  5116  100  5116    0     0  69135      0 --:--:-- --:--:-- --:--:-- 69135\n",
            "Updating... This may take around 2 minutes.\n",
            "Updating TPU runtime to pytorch-dev20210304 ...\n",
            "Found existing installation: torch 1.9.0+cu102\n",
            "Collecting cloud-tpu-client\n",
            "  Downloading cloud_tpu_client-0.10-py3-none-any.whl (7.4 kB)\n",
            "Collecting google-api-python-client==1.8.0\n",
            "  Downloading google_api_python_client-1.8.0-py3-none-any.whl (57 kB)\n",
            "\u001b[K     |████████████████████████████████| 57 kB 1.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: oauth2client in /usr/local/lib/python3.7/dist-packages (from cloud-tpu-client) (4.1.3)\n",
            "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client) (3.0.1)\n",
            "Requirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client) (1.15.0)\n",
            "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client) (0.0.4)\n",
            "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client) (1.34.0)\n",
            "Requirement already satisfied: google-api-core<2dev,>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client) (1.26.3)\n",
            "Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client) (0.17.4)\n",
            "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (2.23.0)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (3.17.3)\n",
            "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (57.4.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (2018.9)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (1.53.0)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (21.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client) (4.2.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client) (4.7.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (2.4.7)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client) (2.10)\n",
            "Uninstalling torch-1.9.0+cu102:\n",
            "Installing collected packages: google-api-python-client, cloud-tpu-client\n",
            "  Attempting uninstall: google-api-python-client\n",
            "    Found existing installation: google-api-python-client 1.12.8\n",
            "    Uninstalling google-api-python-client-1.12.8:\n",
            "      Successfully uninstalled google-api-python-client-1.12.8\n",
            "\u001b[33mWARNING: Ignoring invalid distribution -orch (/usr/local/lib/python3.7/dist-packages)\u001b[0m\n",
            "\u001b[33mWARNING: Ignoring invalid distribution -orch (/usr/local/lib/python3.7/dist-packages)\u001b[0m\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "earthengine-api 0.1.278 requires google-api-python-client<2,>=1.12.1, but you have google-api-python-client 1.8.0 which is incompatible.\u001b[0m\n",
            "Successfully installed cloud-tpu-client-0.10 google-api-python-client-1.8.0\n",
            "  Successfully uninstalled torch-1.9.0+cu102\n",
            "Found existing installation: torchvision 0.10.0+cu102\n",
            "Uninstalling torchvision-0.10.0+cu102:\n",
            "  Successfully uninstalled torchvision-0.10.0+cu102\n",
            "Done updating TPU runtime\n",
            "Copying gs://tpu-pytorch/wheels/torch-nightly+20210304-cp37-cp37m-linux_x86_64.whl...\n",
            "\\ [1 files][126.5 MiB/126.5 MiB]                                                \n",
            "Operation completed over 1 objects/126.5 MiB.                                    \n",
            "Copying gs://tpu-pytorch/wheels/torch_xla-nightly+20210304-cp37-cp37m-linux_x86_64.whl...\n",
            "\\ [1 files][138.1 MiB/138.1 MiB]                                                \n",
            "Operation completed over 1 objects/138.1 MiB.                                    \n",
            "Copying gs://tpu-pytorch/wheels/torchvision-nightly+20210304-cp37-cp37m-linux_x86_64.whl...\n",
            "/ [1 files][  4.6 MiB/  4.6 MiB]                                                \n",
            "Operation completed over 1 objects/4.6 MiB.                                      \n",
            "Processing ./torch-nightly+20210304-cp37-cp37m-linux_x86_64.whl\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==nightly+20210304) (3.7.4.3)\n",
            "Installing collected packages: torch\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "fastai 1.0.61 requires torchvision, which is not installed.\n",
            "torchtext 0.10.0 requires torch==1.9.0, but you have torch 1.9.0a0+gitc4c77e2 which is incompatible.\u001b[0m\n",
            "Successfully installed torch-1.9.0a0+gitc4c77e2\n",
            "Processing ./torch_xla-nightly+20210304-cp37-cp37m-linux_x86_64.whl\n",
            "Installing collected packages: torch-xla\n",
            "Successfully installed torch-xla-1.9+e6237f2\n",
            "Processing ./torchvision-nightly+20210304-cp37-cp37m-linux_x86_64.whl\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from torchvision==nightly+20210304) (1.9.0a0+gitc4c77e2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision==nightly+20210304) (1.19.5)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision==nightly+20210304) (7.1.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->torchvision==nightly+20210304) (3.7.4.3)\n",
            "Installing collected packages: torchvision\n",
            "Successfully installed torchvision-0.9.0a0+7d41547\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  libomp5\n",
            "0 upgraded, 1 newly installed, 0 to remove and 40 not upgraded.\n",
            "Need to get 234 kB of archives.\n",
            "After this operation, 774 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libomp5 amd64 5.0.1-1 [234 kB]\n",
            "Fetched 234 kB in 1s (328 kB/s)\n",
            "Selecting previously unselected package libomp5:amd64.\n",
            "(Reading database ... 148492 files and directories currently installed.)\n",
            "Preparing to unpack .../libomp5_5.0.1-1_amd64.deb ...\n",
            "Unpacking libomp5:amd64 (5.0.1-1) ...\n",
            "Setting up libomp5:amd64 (5.0.1-1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.2) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "27ND-qg2tJEU",
        "outputId": "b28d303e-03e5-4519-93dd-98b2b09519d1"
      },
      "source": [
        "!pip install --pre pytorch-ignite"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pytorch-ignite\n",
            "  Downloading pytorch_ignite-0.5.0.dev20210913-py3-none-any.whl (233 kB)\n",
            "\u001b[?25l\r\u001b[K     |█▍                              | 10 kB 18.0 MB/s eta 0:00:01\r\u001b[K     |██▉                             | 20 kB 23.2 MB/s eta 0:00:01\r\u001b[K     |████▏                           | 30 kB 22.2 MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 40 kB 17.6 MB/s eta 0:00:01\r\u001b[K     |███████                         | 51 kB 5.8 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 61 kB 6.2 MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 71 kB 5.6 MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 81 kB 6.2 MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 92 kB 6.2 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 102 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 112 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 122 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 133 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 143 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 153 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 163 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 174 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 184 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 194 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 204 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 215 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 225 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 233 kB 5.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch<2,>=1.3 in /usr/local/lib/python3.7/dist-packages (from pytorch-ignite) (1.9.0a0+gitc4c77e2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch<2,>=1.3->pytorch-ignite) (3.7.4.3)\n",
            "Installing collected packages: pytorch-ignite\n",
            "Successfully installed pytorch-ignite-0.5.0.dev20210913\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bYoQTy8JtJBU"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision import datasets, models\n",
        "\n",
        "import torchsummary\n",
        "\n",
        "import ignite\n",
        "import ignite.distributed as idist\n",
        "from ignite.engine import Engine, Events, create_supervised_evaluator, create_supervised_trainer\n",
        "from ignite.metrics import Accuracy, Loss, RunningAverage, ConfusionMatrix\n",
        "from ignite.handlers import ModelCheckpoint, EarlyStopping\n",
        "from ignite.utils import setup_logger"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jI10zGFxygPK"
      },
      "source": [
        "병렬 컨텍스트에서 실행되는 training 함수 내부에 미니배치(minibatch)의 첫번 째 이터레이션(iteration)에 해당하는 샘플 이미지를 할당된 VM에 바로 저장하도록 코드를 추가하였다. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "va-GVsDLyijz"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "SMALL_SIZE = 8\n",
        "MEDIUM_SIZE = 12\n",
        "BIGGER_SIZE = 18\n",
        "\n",
        "plt.rc('font', size=SMALL_SIZE) # controls default text sizes\n",
        "plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels\n",
        "plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels\n",
        "\n",
        "def training(local_rank, config, **kwargs):\n",
        "    print(\"local rank: \", local_rank)\n",
        "\n",
        "    ###########################################################\n",
        "    # 데이터 준비\n",
        "    train_transform = transforms.Compose(\n",
        "        [\n",
        "            transforms.Pad(4),\n",
        "            transforms.RandomCrop(32, fill=128),\n",
        "            transforms.RandomHorizontalFlip(),\n",
        "            transforms.ToTensor(),\n",
        "            #transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
        "        ]\n",
        "    )\n",
        "\n",
        "    #test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),])\n",
        "    test_transform = transforms.Compose([transforms.ToTensor(),])\n",
        "\n",
        "    if idist.get_local_rank() > 0:\n",
        "        idist.barrier()\n",
        "\n",
        "    trainset = torchvision.datasets.CIFAR10(root=config[\"data_path\"], train=True, download=True, transform=train_transform)\n",
        "    testset = torchvision.datasets.CIFAR10(root=config[\"data_path\"], train=False, download=True, transform=test_transform)\n",
        "\n",
        "    if idist.get_local_rank() == 0:\n",
        "        idist.barrier()\n",
        "\n",
        "    trainloader = idist.auto_dataloader(trainset, batch_size=config[\"batch_size\"], shuffle=True, num_workers=config[\"num_workers\"], drop_last=True)\n",
        "    testloader = idist.auto_dataloader(testset, batch_size=config[\"batch_size\"], shuffle=False, num_workers=config[\"num_workers\"],)\n",
        "\n",
        "\n",
        "    #########################################################\n",
        "    # 데이터 분배 확인을 위한 코드 추가\n",
        "    # 좌측 페인에서 파일 아이콘을 선택하면 보이는 리스트에서 이미지 확인 가능\n",
        "    images, labels = next(iter(trainloader))\n",
        "    # 64개\n",
        "    h = 8\n",
        "    w = 8\n",
        "\n",
        "    fig, ax = plt.subplots(h, w, figsize=(3 * w, 3 * h))\n",
        "    for n1 in range(h):\n",
        "        for n2 in range(w):\n",
        "            n12 = n1*w + n2\n",
        "            ax[n1, n2].imshow(np.transpose(images[n12].cpu().numpy(), (1, 2, 0)))\n",
        "            ax[n1, n2].set_title(np.round(labels[n12].cpu().numpy(), 1))\n",
        "    plt.savefig(f\"/content/batch_64_{idist.get_local_rank()}.png\")\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    ###########################################################\n",
        "    # 모델, 옵티마이저, 로스, 트레이너, 이밸류에이터\n",
        "    num_classes = 10\n",
        "    model = models.resnet18(num_classes = num_classes)\n",
        "       \n",
        "    model = idist.auto_model(model)\n",
        "    optimizer = idist.auto_optim(optim.Adam(model.parameters(), lr=0.001))\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss().to(idist.device())\n",
        "\n",
        "    trainer = create_supervised_trainer(model, optimizer, criterion, device=idist.device())\n",
        "    trainer.logger = setup_logger(\"hkim-trainer\")\n",
        "\n",
        "    metrics = {\n",
        "        'accuracy':Accuracy(),\n",
        "        'ce':Loss(criterion),\n",
        "    }\n",
        "\n",
        "    val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=idist.device())\n",
        "    val_evaluator.logger = setup_logger(\"hkim-val_evaluator\")\n",
        "\n",
        "    # track a running average of the scalar loss output for each batch.\n",
        "    RunningAverage(output_transform=lambda x: x).attach(trainer, 'loss')\n",
        "\n",
        "    ###########################################################\n",
        "    # 이벤트\n",
        "\n",
        "    @trainer.on(Events.EPOCH_COMPLETED)\n",
        "    def log_validation_results(trainer):\n",
        "        state = val_evaluator.run(testloader)\n",
        "        metrics = val_evaluator.state.metrics\n",
        "        accuracy = metrics['accuracy']*100\n",
        "        loss = metrics['ce']\n",
        "        log_metrics(val_evaluator.logger, state.epoch, state.times[\"COMPLETED\"], \"validation evaluator\", state.metrics)\n",
        "\n",
        "    trainer.run(trainloader, max_epochs=config[\"num_epochs\"])    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "1860be9fcc2540d3b0a44939cc8f97e7",
            "af6b2cd852ce464297b0c87b793b4f7b",
            "159b79967bac4f689390134a61b0bcaa",
            "84ad006a4abd45d6af2763acbde12f4b",
            "ba3eec9dac204faea889cc5fe34ef881",
            "ff63067cfa3543e2bcba6b70d98ce583",
            "bcf9d3ed27d5468e939ebaa0eeac02ec",
            "5bce7fa5356840d3b0eb2ae95cbdbd86",
            "e553d12bca3643bfa6dc4635e2d378c8",
            "f3792f6c6e8d4ada8ea85333b7dc95f3",
            "bce97236b2354156ac9404aa096cb993"
          ]
        },
        "id": "FXJzm8yqtI-k",
        "outputId": "f899473b-c51c-4e09-e9bb-e9e039dc9223"
      },
      "source": [
        "config = {\n",
        "    \"seed\": 543,\n",
        "    \"data_path\" : \"./cifar10\",\n",
        "    \"output_path\" : \"./output-cifar10/\",\n",
        "    \"model\" : \"resnet18\",\n",
        "    \"batch_size\" : 512,\n",
        "    \"momentum\" : 0.9,\n",
        "    \"weight_decay\" : 1e-4,\n",
        "    \"num_workers\" : 2,\n",
        "    \"num_epochs\" : 5,\n",
        "    \"learning_rate\" : 0.4,\n",
        "    \"num_warmup_epochs\" : 4,\n",
        "    \"validate_every\" : 3, \n",
        "    \"checkpoint_every\" : 1000,\n",
        "    \"backend\" : None, \n",
        "    \"resume_from\" : None, \n",
        "    \"log_every_iters\" : 15,\n",
        "    \"nproc_per_node\" : None, \n",
        "    \"stop_iteration\" : None, \n",
        "    \"with_amp\" : False,\n",
        "    \"log_interval\" : 10,\n",
        "    \"verbose_set\" : False,\n",
        "    \"verbose_set2\" : False,\n",
        "    \"verbose_loader\" : False\n",
        "\n",
        "}\n",
        "\n",
        "if not (tpu_gtg or gpu_gtg): # cpu\n",
        "    config[\"backend\"] = 'gloo'\n",
        "    config[\"nproc_per_node\"] = 8\n",
        "elif gpu_gtg: # gpu\n",
        "    config[\"backend\"] = 'nccl'\n",
        "    config[\"nproc_per_node\"] = 1\n",
        "elif tpu_gtg: # tpu\n",
        "    config[\"backend\"] = 'xla-tpu'\n",
        "    config[\"nproc_per_node\"] = 8\n",
        "else: # error\n",
        "    raise RuntimeError(\"Unknown environment: tpu_gtg {}, gpu_gtg {}\".format(tpu_gtg, gpu_gtg))\n",
        "\n",
        "if config[\"backend\"] == \"xla-tpu\" and config[\"with_amp\"]:\n",
        "    raise RuntimeError(\"The value of with_amp should be False if backend is xla\")\n",
        "\n",
        "\n",
        "dist_configs = {'nproc_per_node': config[\"nproc_per_node\"], \"start_method\": \"fork\"}  \n",
        "\n",
        "def log_metrics(logger, epoch, elapsed, tag, metrics):\n",
        "    metrics_output = \"\\n\".join([f\"\\t{k}: {v}\" for k, v in metrics.items()])\n",
        "    logger.info(f\"\\nEpoch {epoch} - Evaluation time (seconds): {elapsed:.2f} - {tag} metrics:\\n {metrics_output}\")\n",
        "\n",
        "with idist.Parallel(backend=config[\"backend\"], **dist_configs) as parallel:\n",
        "    parallel.run(training, config, a=1, b=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2021-09-13 10:20:20,414 ignite.distributed.launcher.Parallel INFO: Initialized distributed launcher with backend: 'xla-tpu'\n",
            "2021-09-13 10:20:20,416 ignite.distributed.launcher.Parallel INFO: - Parameters to spawn processes: \n",
            "\tnproc_per_node: 8\n",
            "\tnnodes: 1\n",
            "\tnode_rank: 0\n",
            "\tstart_method: fork\n",
            "2021-09-13 10:20:20,417 ignite.distributed.launcher.Parallel INFO: Spawn function '<function training at 0x7fb8a7bd2710>' in 8 processes\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "local rank:  0\n",
            "local rank:  1\n",
            "local rank:  5\n",
            "local rank:  2\n",
            "local rank:  4\n",
            "local rank:  7\n",
            "local rank:  6\n",
            "local rank:  3\n",
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar10/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1860be9fcc2540d3b0a44939cc8f97e7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./cifar10/cifar-10-python.tar.gz to ./cifar10\n",
            "Files already downloaded and verified\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2021-09-13 10:20:42,238 ignite.distributed.auto.auto_dataloader INFO: Use data loader kwargs for dataset 'Dataset CIFAR10': \n",
            "\t{'batch_size': 64, 'num_workers': 2, 'drop_last': True, 'sampler': <torch.utils.data.distributed.DistributedSampler object at 0x7fb8a2131fd0>, 'pin_memory': False}\n",
            "2021-09-13 10:20:42,263 ignite.distributed.auto.auto_dataloader INFO: DataLoader is wrapped by `MpDeviceLoader` on XLA\n",
            "2021-09-13 10:20:42,286 ignite.distributed.auto.auto_dataloader INFO: Use data loader kwargs for dataset 'Dataset CIFAR10': \n",
            "\t{'batch_size': 64, 'num_workers': 2, 'sampler': <torch.utils.data.distributed.DistributedSampler object at 0x7fb8a0465190>, 'pin_memory': False}\n",
            "2021-09-13 10:20:42,302 ignite.distributed.auto.auto_dataloader INFO: DataLoader is wrapped by `MpDeviceLoader` on XLA\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2021-09-13 10:21:33,485 hkim-trainer INFO: Engine run starting with max_epochs=5.\n",
            "2021-09-13 10:22:21,856 hkim-val_evaluator INFO: Engine run starting with max_epochs=1.\n",
            "2021-09-13 10:22:27,856 hkim-val_evaluator INFO: Epoch[1] Complete. Time taken: 00:00:06\n",
            "2021-09-13 10:22:27,892 hkim-val_evaluator INFO: Engine run complete. Time taken: 00:00:06\n",
            "2021-09-13 10:22:27,894 hkim-val_evaluator INFO: \n",
            "Epoch 1 - Evaluation time (seconds): 6.02 - validation evaluator metrics:\n",
            " \taccuracy: 0.4528\n",
            "\tce: 1.5992783203125\n",
            "2021-09-13 10:22:27,903 hkim-trainer INFO: Epoch[1] Complete. Time taken: 00:00:54\n",
            "2021-09-13 10:23:01,315 hkim-val_evaluator INFO: Engine run starting with max_epochs=1.\n",
            "2021-09-13 10:23:04,701 hkim-val_evaluator INFO: Epoch[1] Complete. Time taken: 00:00:03\n",
            "2021-09-13 10:23:04,713 hkim-val_evaluator INFO: Engine run complete. Time taken: 00:00:03\n",
            "2021-09-13 10:23:04,723 hkim-val_evaluator INFO: \n",
            "Epoch 1 - Evaluation time (seconds): 3.38 - validation evaluator metrics:\n",
            " \taccuracy: 0.4061\n",
            "\tce: 1.8193583984375\n",
            "2021-09-13 10:23:04,734 hkim-trainer INFO: Epoch[2] Complete. Time taken: 00:00:37\n",
            "2021-09-13 10:23:37,731 hkim-val_evaluator INFO: Engine run starting with max_epochs=1.\n",
            "2021-09-13 10:23:41,118 hkim-val_evaluator INFO: Epoch[1] Complete. Time taken: 00:00:03\n",
            "2021-09-13 10:23:41,147 hkim-val_evaluator INFO: Engine run complete. Time taken: 00:00:03\n",
            "2021-09-13 10:23:41,163 hkim-val_evaluator INFO: \n",
            "Epoch 1 - Evaluation time (seconds): 3.39 - validation evaluator metrics:\n",
            " \taccuracy: 0.5754\n",
            "\tce: 1.2255111328125\n",
            "2021-09-13 10:23:41,177 hkim-trainer INFO: Epoch[3] Complete. Time taken: 00:00:36\n",
            "2021-09-13 10:24:13,801 hkim-val_evaluator INFO: Engine run starting with max_epochs=1.\n",
            "2021-09-13 10:24:17,159 hkim-val_evaluator INFO: Epoch[1] Complete. Time taken: 00:00:03\n",
            "2021-09-13 10:24:17,165 hkim-val_evaluator INFO: Engine run complete. Time taken: 00:00:03\n",
            "2021-09-13 10:24:17,181 hkim-val_evaluator INFO: \n",
            "Epoch 1 - Evaluation time (seconds): 3.35 - validation evaluator metrics:\n",
            " \taccuracy: 0.5942\n",
            "\tce: 1.2201275390625\n",
            "2021-09-13 10:24:17,193 hkim-trainer INFO: Epoch[4] Complete. Time taken: 00:00:36\n",
            "2021-09-13 10:24:49,697 hkim-val_evaluator INFO: Engine run starting with max_epochs=1.\n",
            "2021-09-13 10:24:53,017 hkim-val_evaluator INFO: Epoch[1] Complete. Time taken: 00:00:03\n",
            "2021-09-13 10:24:53,033 hkim-val_evaluator INFO: Engine run complete. Time taken: 00:00:03\n",
            "2021-09-13 10:24:53,099 hkim-val_evaluator INFO: \n",
            "Epoch 1 - Evaluation time (seconds): 3.33 - validation evaluator metrics:\n",
            " \taccuracy: 0.6004\n",
            "\tce: 1.23046904296875\n",
            "2021-09-13 10:24:53,125 hkim-trainer INFO: Epoch[5] Complete. Time taken: 00:00:36\n",
            "2021-09-13 10:24:53,184 hkim-trainer INFO: Engine run complete. Time taken: 00:03:20\n",
            "2021-09-13 10:24:53,362 ignite.distributed.launcher.Parallel INFO: End of run\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NrhKjYC4mLaU"
      },
      "source": [
        " 코드 실행 후에는 아래 그림에서와 같이 좌측 페인(pane)의 그리고 파일 아이콘을 클릭하면 보이는 리스트에서, batch_64_0.png와 같이 생성된 png 파일을 차례로 클릭한다.\n",
        "\n",
        "<div align=\"center\">\n",
        "<img width=512 src=\"https://i.imgur.com/Y7owt0c.png\"/>\n",
        "</div> "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cC4UEOMvzxtA"
      },
      "source": [
        "일례로 아래 그림은 프로세스 0번에서 auto_dataloader를 통해 준비된 미니배치에 포함된 이미지이다.\n",
        "\n",
        "<div align=\"center\">\n",
        "<img width=512 src=\"https://i.imgur.com/Ng8S1Ei.png\"/>\n",
        "</div> "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lqur-kef0NVO"
      },
      "source": [
        "그리고 다음 그림은 프로세스 1번에서 auto_dataloader를 통해 준비된 미니배치에 포함된 이미지이다.\n",
        "\n",
        "<div align=\"center\">\n",
        "<img width=512 src=\"https://i.imgur.com/KPfwKqw.png\"/>\n",
        "</div> "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "giGVn4Mg1vkc"
      },
      "source": [
        " 이처럼 이그나이트에서 지원하는 auto_dataloader 함수에 의해 분산처리될 프로세스로 적절히 학습데이터가 배분되었음을 확인할 수 있다.\n",
        " \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OQL39wXmXx9k"
      },
      "source": [
        "## License\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YXio6q3iX5Ig"
      },
      "source": [
        "---\n",
        "\n",
        "\n",
        "Note: This is not an official [LG AI Research](https://www.lgresearch.ai/) product but sample code provided for an educational purpose\n",
        "\n",
        "<br/>\n",
        "author: John H. Kim\n",
        "<br/>  \n",
        "email: john.kim@lgresearch.ai / secutron@naver.com  \n",
        "\n",
        "\n",
        "---"
      ]
    }
  ]
}