ResNet-for-CIFAR-10-with-Keras.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"ResNet-for-CIFAR-10-with-Keras.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"xU6Tc_oTTHII","colab_type":"text"},"source":["# テーマ：CIFAR-10データセットを畳み込みニューラルネットワーク（CNN）で学習する"]},{"cell_type":"markdown","metadata":{"id":"cilYmE6LxvMq","colab_type":"text"},"source":["## 制約\n","* 全体の実行時間は60分以内"]},{"cell_type":"markdown","metadata":{"id":"yw_8PaTTTsw6","colab_type":"text"},"source":["## 今回参考にしたもの\n","* [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)\n","* [practice on CIFAR10 with Keras](https://github.com/jerett/Keras-CIFAR10)"]},{"cell_type":"markdown","metadata":{"id":"n_rARdLQr3zQ","colab_type":"text"},"source":["## 0. Install TensorFlow 2.0"]},{"cell_type":"code","metadata":{"id":"cG3G4utpTD0C","colab_type":"code","outputId":"883799c2-8273-48c0-f834-fd8dc653f280","executionInfo":{"status":"ok","timestamp":1560065355542,"user_tz":-540,"elapsed":3996,"user":{"displayName":"Keisuke Shoji","photoUrl":"","userId":"10321049352155053308"}},"colab":{"base_uri":"https://localhost:8080/","height":360}},"source":["!pip install tensorflow-gpu==2.0.0-alpha0"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: tensorflow-gpu==2.0.0-alpha0 in /usr/local/lib/python3.6/dist-packages (2.0.0a0)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)\n","Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)\n","Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)\n","Requirement already satisfied: tb-nightly<1.14.0a20190302,>=1.14.0a20190301 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.0a20190301)\n","Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.0.dev2019030115)\n","Requirement already satisfied: google-pasta>=0.1.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.1.7)\n","Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.8.0)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.4)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.12.0)\n","Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)\n","Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.8)\n","Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.4)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (41.0.1)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.4)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1.1)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"ZYHyNNGpsAY7","colab_type":"text"},"source":["## 1. Prepare dataset"]},{"cell_type":"code","metadata":{"id":"uQCnTxM4TIY8","colab_type":"code","outputId":"5e852684-1905-40bf-9da9-8d4fa6bdcab2","executionInfo":{"status":"ok","timestamp":1560065371082,"user_tz":-540,"elapsed":13491,"user":{"displayName":"Keisuke Shoji","photoUrl":"","userId":"10321049352155053308"}},"colab":{"base_uri":"https://localhost:8080/","height":496}},"source":["import tensorflow_datasets as tfds\n","import tensorflow as tf\n","\n","# print(tfds.list_builders())\n","dataset, info = tfds.load('cifar10', as_supervised = True, with_info = True, batch_size = -1)\n","dataset_test, dataset_train = dataset['test'], dataset['train']\n","print(info)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["tfds.core.DatasetInfo(\n","    name='cifar10',\n","    version=1.0.2,\n","    description='The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images.',\n","    urls=['https://www.cs.toronto.edu/~kriz/cifar.html'],\n","    features=FeaturesDict({\n","        'image': Image(shape=(32, 32, 3), dtype=tf.uint8),\n","        'label': ClassLabel(shape=(), dtype=tf.int64, num_classes=10)\n","    },\n","    total_num_examples=60000,\n","    splits={\n","        'test': <tfds.core.SplitInfo num_examples=10000>,\n","        'train': <tfds.core.SplitInfo num_examples=50000>\n","    },\n","    supervised_keys=('image', 'label'),\n","    citation='\"\"\"\n","        @TECHREPORT{Krizhevsky09learningmultiple,\n","            author = {Alex Krizhevsky},\n","            title = {Learning multiple layers of features from tiny images},\n","            institution = {},\n","            year = {2009}\n","        }\n","        \n","    \"\"\"',\n","    redistribution_info=,\n",")\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"WGZqjYGGMzWg","colab_type":"code","colab":{}},"source":["import numpy as np\n","\n","train_x = dataset_train[0] / 255\n","train_y = dataset_train[1]\n","test_x = dataset_test[0] / 255\n","test_y = dataset_test[1]\n","\n","# Per-pixel Mean Subtraction\n","# mean_image = np.mean(train_x, axis=0)\n","# train_x -= mean_image\n","# test_x -= mean_image            "],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PqU9JzNxwRmF","colab_type":"text"},"source":["## 2. Data Augmentation"]},{"cell_type":"code","metadata":{"id":"fgxq02WjwZE9","colab_type":"code","colab":{}},"source":["import numpy as np\n","from tensorflow.keras.preprocessing.image import ImageDataGenerator\n","\n","class CustomImageDataGenerator(ImageDataGenerator):\n","    def __init__(self, cutout_mask_size = 0, *args, **kwargs):\n","        super().__init__(*args, **kwargs)\n","        self.cutout_mask_size = cutout_mask_size\n","        \n","    def cutout(self, x, y):\n","        return np.array(list(map(self._cutout, x))), y\n","    \n","    def _cutout(self, image_origin):\n","        # 最後に使うfill()は元の画像を書き換えるので、コピーしておく\n","        image = np.copy(image_origin)\n","        mask_value = image.mean()\n","\n","        h, w, _ = image.shape\n","        # マスクをかける場所のtop, leftをランダムに決める\n","        # はみ出すことを許すので、0以上ではなく負の値もとる(最大mask_size // 2はみ出す)\n","        top = np.random.randint(0 - self.cutout_mask_size // 2, h - self.cutout_mask_size)\n","        left = np.random.randint(0 - self.cutout_mask_size // 2, w - self.cutout_mask_size)\n","        bottom = top + self.cutout_mask_size\n","        right = left + self.cutout_mask_size\n","\n","        # はみ出した場合の処理\n","        if top < 0:\n","            top = 0\n","        if left < 0:\n","            left = 0\n","\n","        # マスク部分の画素値を平均値で埋める\n","        image[top:bottom, left:right, :].fill(mask_value)\n","        return image\n","    \n","    def flow(self, *args, **kwargs):\n","        batches = super().flow(*args, **kwargs)\n","        \n","        # 拡張処理\n","        while True:\n","            batch_x, batch_y = next(batches)\n","            \n","            if self.cutout_mask_size > 0:\n","                result = self.cutout(batch_x, batch_y)\n","                batch_x, batch_y = result                        \n","                \n","            yield (batch_x, batch_y)     \n","\n","datagen_parameters = {\"horizontal_flip\": True, \"width_shift_range\": 0.1, \"height_shift_range\": 0.1, \"cutout_mask_size\": 16}\n","datagen = CustomImageDataGenerator(**datagen_parameters)\n","datagen_for_test = ImageDataGenerator()\n","# ZCA whiteningなどを行う場合が以下の実行が必要\n","# datagen.fit(train_x)\n","# datagen_for_test.fit(test_x)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"HjpM_PBXsCzt","colab_type":"text"},"source":["## 3. Define model"]},{"cell_type":"code","metadata":{"id":"GPgSnpmp626g","colab_type":"code","colab":{}},"source":["from tensorflow.keras.layers import Conv2D, Dense, BatchNormalization, Activation, MaxPool2D, GlobalAveragePooling2D, Add, Input, Flatten\n","from tensorflow.keras import Model\n","from tensorflow.keras.regularizers import l2\n","\n","n = 9 # 56 layers\n","channels = [16, 32, 64]\n","\n","inputs = Input(shape=(32, 32, 3))\n","x = Conv2D(channels[0], kernel_size=(3, 3), padding=\"same\", kernel_initializer=\"he_normal\", kernel_regularizer=l2(1e-4))(inputs)\n","x = BatchNormalization()(x)\n","x = Activation(tf.nn.relu)(x)\n","\n","for c in channels:\n","    for i in range(n):\n","        subsampling = i == 0 and c > 16\n","        strides = (2, 2) if subsampling else (1, 1)\n","        y = Conv2D(c, kernel_size=(3, 3), padding=\"same\", strides=strides, kernel_initializer=\"he_normal\", kernel_regularizer=l2(1e-4))(x)\n","        y = BatchNormalization()(y)\n","        y = Activation(tf.nn.relu)(y)\n","        y = Conv2D(c, kernel_size=(3, 3), padding=\"same\", kernel_initializer=\"he_normal\", kernel_regularizer=l2(1e-4))(y)\n","        y = BatchNormalization()(y)        \n","        if subsampling:\n","            x = Conv2D(c, kernel_size=(1, 1), strides=(2, 2), padding=\"same\", kernel_initializer=\"he_normal\", kernel_regularizer=l2(1e-4))(x)\n","        x = Add()([x, y])\n","        x = Activation(tf.nn.relu)(x)\n","\n","x = GlobalAveragePooling2D()(x)\n","x = Flatten()(x)\n","outputs = Dense(10, activation=tf.nn.softmax, kernel_initializer=\"he_normal\")(x)\n","\n","model = Model(inputs=inputs, outputs=outputs)\n","model.type = \"resnet\" + str(6 * n + 2)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"je9qxLqHBORQ","colab_type":"code","outputId":"6160309a-fe23-4052-d7f4-5c6d36848a74","executionInfo":{"status":"ok","timestamp":1560073630279,"user_tz":-540,"elapsed":641,"user":{"displayName":"Keisuke Shoji","photoUrl":"","userId":"10321049352155053308"}},"colab":{"base_uri":"https://localhost:8080/","height":7361}},"source":["from io import StringIO\n","import re\n","\n","with StringIO() as buf:\n","    model.summary(print_fn=lambda x: buf.write(x + \"\\n\"))\n","    summary = buf.getvalue()\n","print(summary)\n","re1 = re.match(r\"(.|\\s)*Total params: \", summary)\n","re2 = re.match(r\"(.|\\s)*Total params: [\\d|,]+\", summary)\n","total_params = summary[re1.end():re2.end()]"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Model: \"model_1\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","input_2 (InputLayer)            [(None, 32, 32, 3)]  0                                            \n","__________________________________________________________________________________________________\n","conv2d_57 (Conv2D)              (None, 32, 32, 16)   448         input_2[0][0]                    \n","__________________________________________________________________________________________________\n","batch_normalization_v2_55 (Batc (None, 32, 32, 16)   64          conv2d_57[0][0]                  \n","__________________________________________________________________________________________________\n","activation_55 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_55[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_58 (Conv2D)              (None, 32, 32, 16)   2320        activation_55[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_56 (Batc (None, 32, 32, 16)   64          conv2d_58[0][0]                  \n","__________________________________________________________________________________________________\n","activation_56 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_56[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_59 (Conv2D)              (None, 32, 32, 16)   2320        activation_56[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_57 (Batc (None, 32, 32, 16)   64          conv2d_59[0][0]                  \n","__________________________________________________________________________________________________\n","add_27 (Add)                    (None, 32, 32, 16)   0           activation_55[0][0]              \n","                                                                 batch_normalization_v2_57[0][0]  \n","__________________________________________________________________________________________________\n","activation_57 (Activation)      (None, 32, 32, 16)   0           add_27[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_60 (Conv2D)              (None, 32, 32, 16)   2320        activation_57[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_58 (Batc (None, 32, 32, 16)   64          conv2d_60[0][0]                  \n","__________________________________________________________________________________________________\n","activation_58 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_58[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_61 (Conv2D)              (None, 32, 32, 16)   2320        activation_58[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_59 (Batc (None, 32, 32, 16)   64          conv2d_61[0][0]                  \n","__________________________________________________________________________________________________\n","add_28 (Add)                    (None, 32, 32, 16)   0           activation_57[0][0]              \n","                                                                 batch_normalization_v2_59[0][0]  \n","__________________________________________________________________________________________________\n","activation_59 (Activation)      (None, 32, 32, 16)   0           add_28[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_62 (Conv2D)              (None, 32, 32, 16)   2320        activation_59[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_60 (Batc (None, 32, 32, 16)   64          conv2d_62[0][0]                  \n","__________________________________________________________________________________________________\n","activation_60 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_60[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_63 (Conv2D)              (None, 32, 32, 16)   2320        activation_60[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_61 (Batc (None, 32, 32, 16)   64          conv2d_63[0][0]                  \n","__________________________________________________________________________________________________\n","add_29 (Add)                    (None, 32, 32, 16)   0           activation_59[0][0]              \n","                                                                 batch_normalization_v2_61[0][0]  \n","__________________________________________________________________________________________________\n","activation_61 (Activation)      (None, 32, 32, 16)   0           add_29[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_64 (Conv2D)              (None, 32, 32, 16)   2320        activation_61[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_62 (Batc (None, 32, 32, 16)   64          conv2d_64[0][0]                  \n","__________________________________________________________________________________________________\n","activation_62 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_62[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_65 (Conv2D)              (None, 32, 32, 16)   2320        activation_62[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_63 (Batc (None, 32, 32, 16)   64          conv2d_65[0][0]                  \n","__________________________________________________________________________________________________\n","add_30 (Add)                    (None, 32, 32, 16)   0           activation_61[0][0]              \n","                                                                 batch_normalization_v2_63[0][0]  \n","__________________________________________________________________________________________________\n","activation_63 (Activation)      (None, 32, 32, 16)   0           add_30[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_66 (Conv2D)              (None, 32, 32, 16)   2320        activation_63[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_64 (Batc (None, 32, 32, 16)   64          conv2d_66[0][0]                  \n","__________________________________________________________________________________________________\n","activation_64 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_64[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_67 (Conv2D)              (None, 32, 32, 16)   2320        activation_64[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_65 (Batc (None, 32, 32, 16)   64          conv2d_67[0][0]                  \n","__________________________________________________________________________________________________\n","add_31 (Add)                    (None, 32, 32, 16)   0           activation_63[0][0]              \n","                                                                 batch_normalization_v2_65[0][0]  \n","__________________________________________________________________________________________________\n","activation_65 (Activation)      (None, 32, 32, 16)   0           add_31[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_68 (Conv2D)              (None, 32, 32, 16)   2320        activation_65[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_66 (Batc (None, 32, 32, 16)   64          conv2d_68[0][0]                  \n","__________________________________________________________________________________________________\n","activation_66 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_66[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_69 (Conv2D)              (None, 32, 32, 16)   2320        activation_66[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_67 (Batc (None, 32, 32, 16)   64          conv2d_69[0][0]                  \n","__________________________________________________________________________________________________\n","add_32 (Add)                    (None, 32, 32, 16)   0           activation_65[0][0]              \n","                                                                 batch_normalization_v2_67[0][0]  \n","__________________________________________________________________________________________________\n","activation_67 (Activation)      (None, 32, 32, 16)   0           add_32[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_70 (Conv2D)              (None, 32, 32, 16)   2320        activation_67[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_68 (Batc (None, 32, 32, 16)   64          conv2d_70[0][0]                  \n","__________________________________________________________________________________________________\n","activation_68 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_68[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_71 (Conv2D)              (None, 32, 32, 16)   2320        activation_68[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_69 (Batc (None, 32, 32, 16)   64          conv2d_71[0][0]                  \n","__________________________________________________________________________________________________\n","add_33 (Add)                    (None, 32, 32, 16)   0           activation_67[0][0]              \n","                                                                 batch_normalization_v2_69[0][0]  \n","__________________________________________________________________________________________________\n","activation_69 (Activation)      (None, 32, 32, 16)   0           add_33[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_72 (Conv2D)              (None, 32, 32, 16)   2320        activation_69[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_70 (Batc (None, 32, 32, 16)   64          conv2d_72[0][0]                  \n","__________________________________________________________________________________________________\n","activation_70 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_70[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_73 (Conv2D)              (None, 32, 32, 16)   2320        activation_70[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_71 (Batc (None, 32, 32, 16)   64          conv2d_73[0][0]                  \n","__________________________________________________________________________________________________\n","add_34 (Add)                    (None, 32, 32, 16)   0           activation_69[0][0]              \n","                                                                 batch_normalization_v2_71[0][0]  \n","__________________________________________________________________________________________________\n","activation_71 (Activation)      (None, 32, 32, 16)   0           add_34[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_74 (Conv2D)              (None, 32, 32, 16)   2320        activation_71[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_72 (Batc (None, 32, 32, 16)   64          conv2d_74[0][0]                  \n","__________________________________________________________________________________________________\n","activation_72 (Activation)      (None, 32, 32, 16)   0           batch_normalization_v2_72[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_75 (Conv2D)              (None, 32, 32, 16)   2320        activation_72[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_73 (Batc (None, 32, 32, 16)   64          conv2d_75[0][0]                  \n","__________________________________________________________________________________________________\n","add_35 (Add)                    (None, 32, 32, 16)   0           activation_71[0][0]              \n","                                                                 batch_normalization_v2_73[0][0]  \n","__________________________________________________________________________________________________\n","activation_73 (Activation)      (None, 32, 32, 16)   0           add_35[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_76 (Conv2D)              (None, 16, 16, 32)   4640        activation_73[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_74 (Batc (None, 16, 16, 32)   128         conv2d_76[0][0]                  \n","__________________________________________________________________________________________________\n","activation_74 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_74[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_77 (Conv2D)              (None, 16, 16, 32)   9248        activation_74[0][0]              \n","__________________________________________________________________________________________________\n","conv2d_78 (Conv2D)              (None, 16, 16, 32)   544         activation_73[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_75 (Batc (None, 16, 16, 32)   128         conv2d_77[0][0]                  \n","__________________________________________________________________________________________________\n","add_36 (Add)                    (None, 16, 16, 32)   0           conv2d_78[0][0]                  \n","                                                                 batch_normalization_v2_75[0][0]  \n","__________________________________________________________________________________________________\n","activation_75 (Activation)      (None, 16, 16, 32)   0           add_36[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_79 (Conv2D)              (None, 16, 16, 32)   9248        activation_75[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_76 (Batc (None, 16, 16, 32)   128         conv2d_79[0][0]                  \n","__________________________________________________________________________________________________\n","activation_76 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_76[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_80 (Conv2D)              (None, 16, 16, 32)   9248        activation_76[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_77 (Batc (None, 16, 16, 32)   128         conv2d_80[0][0]                  \n","__________________________________________________________________________________________________\n","add_37 (Add)                    (None, 16, 16, 32)   0           activation_75[0][0]              \n","                                                                 batch_normalization_v2_77[0][0]  \n","__________________________________________________________________________________________________\n","activation_77 (Activation)      (None, 16, 16, 32)   0           add_37[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_81 (Conv2D)              (None, 16, 16, 32)   9248        activation_77[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_78 (Batc (None, 16, 16, 32)   128         conv2d_81[0][0]                  \n","__________________________________________________________________________________________________\n","activation_78 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_78[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_82 (Conv2D)              (None, 16, 16, 32)   9248        activation_78[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_79 (Batc (None, 16, 16, 32)   128         conv2d_82[0][0]                  \n","__________________________________________________________________________________________________\n","add_38 (Add)                    (None, 16, 16, 32)   0           activation_77[0][0]              \n","                                                                 batch_normalization_v2_79[0][0]  \n","__________________________________________________________________________________________________\n","activation_79 (Activation)      (None, 16, 16, 32)   0           add_38[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_83 (Conv2D)              (None, 16, 16, 32)   9248        activation_79[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_80 (Batc (None, 16, 16, 32)   128         conv2d_83[0][0]                  \n","__________________________________________________________________________________________________\n","activation_80 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_80[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_84 (Conv2D)              (None, 16, 16, 32)   9248        activation_80[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_81 (Batc (None, 16, 16, 32)   128         conv2d_84[0][0]                  \n","__________________________________________________________________________________________________\n","add_39 (Add)                    (None, 16, 16, 32)   0           activation_79[0][0]              \n","                                                                 batch_normalization_v2_81[0][0]  \n","__________________________________________________________________________________________________\n","activation_81 (Activation)      (None, 16, 16, 32)   0           add_39[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_85 (Conv2D)              (None, 16, 16, 32)   9248        activation_81[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_82 (Batc (None, 16, 16, 32)   128         conv2d_85[0][0]                  \n","__________________________________________________________________________________________________\n","activation_82 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_82[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_86 (Conv2D)              (None, 16, 16, 32)   9248        activation_82[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_83 (Batc (None, 16, 16, 32)   128         conv2d_86[0][0]                  \n","__________________________________________________________________________________________________\n","add_40 (Add)                    (None, 16, 16, 32)   0           activation_81[0][0]              \n","                                                                 batch_normalization_v2_83[0][0]  \n","__________________________________________________________________________________________________\n","activation_83 (Activation)      (None, 16, 16, 32)   0           add_40[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_87 (Conv2D)              (None, 16, 16, 32)   9248        activation_83[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_84 (Batc (None, 16, 16, 32)   128         conv2d_87[0][0]                  \n","__________________________________________________________________________________________________\n","activation_84 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_84[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_88 (Conv2D)              (None, 16, 16, 32)   9248        activation_84[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_85 (Batc (None, 16, 16, 32)   128         conv2d_88[0][0]                  \n","__________________________________________________________________________________________________\n","add_41 (Add)                    (None, 16, 16, 32)   0           activation_83[0][0]              \n","                                                                 batch_normalization_v2_85[0][0]  \n","__________________________________________________________________________________________________\n","activation_85 (Activation)      (None, 16, 16, 32)   0           add_41[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_89 (Conv2D)              (None, 16, 16, 32)   9248        activation_85[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_86 (Batc (None, 16, 16, 32)   128         conv2d_89[0][0]                  \n","__________________________________________________________________________________________________\n","activation_86 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_86[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_90 (Conv2D)              (None, 16, 16, 32)   9248        activation_86[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_87 (Batc (None, 16, 16, 32)   128         conv2d_90[0][0]                  \n","__________________________________________________________________________________________________\n","add_42 (Add)                    (None, 16, 16, 32)   0           activation_85[0][0]              \n","                                                                 batch_normalization_v2_87[0][0]  \n","__________________________________________________________________________________________________\n","activation_87 (Activation)      (None, 16, 16, 32)   0           add_42[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_91 (Conv2D)              (None, 16, 16, 32)   9248        activation_87[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_88 (Batc (None, 16, 16, 32)   128         conv2d_91[0][0]                  \n","__________________________________________________________________________________________________\n","activation_88 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_88[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_92 (Conv2D)              (None, 16, 16, 32)   9248        activation_88[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_89 (Batc (None, 16, 16, 32)   128         conv2d_92[0][0]                  \n","__________________________________________________________________________________________________\n","add_43 (Add)                    (None, 16, 16, 32)   0           activation_87[0][0]              \n","                                                                 batch_normalization_v2_89[0][0]  \n","__________________________________________________________________________________________________\n","activation_89 (Activation)      (None, 16, 16, 32)   0           add_43[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_93 (Conv2D)              (None, 16, 16, 32)   9248        activation_89[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_90 (Batc (None, 16, 16, 32)   128         conv2d_93[0][0]                  \n","__________________________________________________________________________________________________\n","activation_90 (Activation)      (None, 16, 16, 32)   0           batch_normalization_v2_90[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_94 (Conv2D)              (None, 16, 16, 32)   9248        activation_90[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_91 (Batc (None, 16, 16, 32)   128         conv2d_94[0][0]                  \n","__________________________________________________________________________________________________\n","add_44 (Add)                    (None, 16, 16, 32)   0           activation_89[0][0]              \n","                                                                 batch_normalization_v2_91[0][0]  \n","__________________________________________________________________________________________________\n","activation_91 (Activation)      (None, 16, 16, 32)   0           add_44[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_95 (Conv2D)              (None, 8, 8, 64)     18496       activation_91[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_92 (Batc (None, 8, 8, 64)     256         conv2d_95[0][0]                  \n","__________________________________________________________________________________________________\n","activation_92 (Activation)      (None, 8, 8, 64)     0           batch_normalization_v2_92[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_96 (Conv2D)              (None, 8, 8, 64)     36928       activation_92[0][0]              \n","__________________________________________________________________________________________________\n","conv2d_97 (Conv2D)              (None, 8, 8, 64)     2112        activation_91[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_93 (Batc (None, 8, 8, 64)     256         conv2d_96[0][0]                  \n","__________________________________________________________________________________________________\n","add_45 (Add)                    (None, 8, 8, 64)     0           conv2d_97[0][0]                  \n","                                                                 batch_normalization_v2_93[0][0]  \n","__________________________________________________________________________________________________\n","activation_93 (Activation)      (None, 8, 8, 64)     0           add_45[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_98 (Conv2D)              (None, 8, 8, 64)     36928       activation_93[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_94 (Batc (None, 8, 8, 64)     256         conv2d_98[0][0]                  \n","__________________________________________________________________________________________________\n","activation_94 (Activation)      (None, 8, 8, 64)     0           batch_normalization_v2_94[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_99 (Conv2D)              (None, 8, 8, 64)     36928       activation_94[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_95 (Batc (None, 8, 8, 64)     256         conv2d_99[0][0]                  \n","__________________________________________________________________________________________________\n","add_46 (Add)                    (None, 8, 8, 64)     0           activation_93[0][0]              \n","                                                                 batch_normalization_v2_95[0][0]  \n","__________________________________________________________________________________________________\n","activation_95 (Activation)      (None, 8, 8, 64)     0           add_46[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_100 (Conv2D)             (None, 8, 8, 64)     36928       activation_95[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_96 (Batc (None, 8, 8, 64)     256         conv2d_100[0][0]                 \n","__________________________________________________________________________________________________\n","activation_96 (Activation)      (None, 8, 8, 64)     0           batch_normalization_v2_96[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_101 (Conv2D)             (None, 8, 8, 64)     36928       activation_96[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_97 (Batc (None, 8, 8, 64)     256         conv2d_101[0][0]                 \n","__________________________________________________________________________________________________\n","add_47 (Add)                    (None, 8, 8, 64)     0           activation_95[0][0]              \n","                                                                 batch_normalization_v2_97[0][0]  \n","__________________________________________________________________________________________________\n","activation_97 (Activation)      (None, 8, 8, 64)     0           add_47[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_102 (Conv2D)             (None, 8, 8, 64)     36928       activation_97[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_98 (Batc (None, 8, 8, 64)     256         conv2d_102[0][0]                 \n","__________________________________________________________________________________________________\n","activation_98 (Activation)      (None, 8, 8, 64)     0           batch_normalization_v2_98[0][0]  \n","__________________________________________________________________________________________________\n","conv2d_103 (Conv2D)             (None, 8, 8, 64)     36928       activation_98[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_99 (Batc (None, 8, 8, 64)     256         conv2d_103[0][0]                 \n","__________________________________________________________________________________________________\n","add_48 (Add)                    (None, 8, 8, 64)     0           activation_97[0][0]              \n","                                                                 batch_normalization_v2_99[0][0]  \n","__________________________________________________________________________________________________\n","activation_99 (Activation)      (None, 8, 8, 64)     0           add_48[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_104 (Conv2D)             (None, 8, 8, 64)     36928       activation_99[0][0]              \n","__________________________________________________________________________________________________\n","batch_normalization_v2_100 (Bat (None, 8, 8, 64)     256         conv2d_104[0][0]                 \n","__________________________________________________________________________________________________\n","activation_100 (Activation)     (None, 8, 8, 64)     0           batch_normalization_v2_100[0][0] \n","__________________________________________________________________________________________________\n","conv2d_105 (Conv2D)             (None, 8, 8, 64)     36928       activation_100[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_101 (Bat (None, 8, 8, 64)     256         conv2d_105[0][0]                 \n","__________________________________________________________________________________________________\n","add_49 (Add)                    (None, 8, 8, 64)     0           activation_99[0][0]              \n","                                                                 batch_normalization_v2_101[0][0] \n","__________________________________________________________________________________________________\n","activation_101 (Activation)     (None, 8, 8, 64)     0           add_49[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_106 (Conv2D)             (None, 8, 8, 64)     36928       activation_101[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_102 (Bat (None, 8, 8, 64)     256         conv2d_106[0][0]                 \n","__________________________________________________________________________________________________\n","activation_102 (Activation)     (None, 8, 8, 64)     0           batch_normalization_v2_102[0][0] \n","__________________________________________________________________________________________________\n","conv2d_107 (Conv2D)             (None, 8, 8, 64)     36928       activation_102[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_103 (Bat (None, 8, 8, 64)     256         conv2d_107[0][0]                 \n","__________________________________________________________________________________________________\n","add_50 (Add)                    (None, 8, 8, 64)     0           activation_101[0][0]             \n","                                                                 batch_normalization_v2_103[0][0] \n","__________________________________________________________________________________________________\n","activation_103 (Activation)     (None, 8, 8, 64)     0           add_50[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_108 (Conv2D)             (None, 8, 8, 64)     36928       activation_103[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_104 (Bat (None, 8, 8, 64)     256         conv2d_108[0][0]                 \n","__________________________________________________________________________________________________\n","activation_104 (Activation)     (None, 8, 8, 64)     0           batch_normalization_v2_104[0][0] \n","__________________________________________________________________________________________________\n","conv2d_109 (Conv2D)             (None, 8, 8, 64)     36928       activation_104[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_105 (Bat (None, 8, 8, 64)     256         conv2d_109[0][0]                 \n","__________________________________________________________________________________________________\n","add_51 (Add)                    (None, 8, 8, 64)     0           activation_103[0][0]             \n","                                                                 batch_normalization_v2_105[0][0] \n","__________________________________________________________________________________________________\n","activation_105 (Activation)     (None, 8, 8, 64)     0           add_51[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_110 (Conv2D)             (None, 8, 8, 64)     36928       activation_105[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_106 (Bat (None, 8, 8, 64)     256         conv2d_110[0][0]                 \n","__________________________________________________________________________________________________\n","activation_106 (Activation)     (None, 8, 8, 64)     0           batch_normalization_v2_106[0][0] \n","__________________________________________________________________________________________________\n","conv2d_111 (Conv2D)             (None, 8, 8, 64)     36928       activation_106[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_107 (Bat (None, 8, 8, 64)     256         conv2d_111[0][0]                 \n","__________________________________________________________________________________________________\n","add_52 (Add)                    (None, 8, 8, 64)     0           activation_105[0][0]             \n","                                                                 batch_normalization_v2_107[0][0] \n","__________________________________________________________________________________________________\n","activation_107 (Activation)     (None, 8, 8, 64)     0           add_52[0][0]                     \n","__________________________________________________________________________________________________\n","conv2d_112 (Conv2D)             (None, 8, 8, 64)     36928       activation_107[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_108 (Bat (None, 8, 8, 64)     256         conv2d_112[0][0]                 \n","__________________________________________________________________________________________________\n","activation_108 (Activation)     (None, 8, 8, 64)     0           batch_normalization_v2_108[0][0] \n","__________________________________________________________________________________________________\n","conv2d_113 (Conv2D)             (None, 8, 8, 64)     36928       activation_108[0][0]             \n","__________________________________________________________________________________________________\n","batch_normalization_v2_109 (Bat (None, 8, 8, 64)     256         conv2d_113[0][0]                 \n","__________________________________________________________________________________________________\n","add_53 (Add)                    (None, 8, 8, 64)     0           activation_107[0][0]             \n","                                                                 batch_normalization_v2_109[0][0] \n","__________________________________________________________________________________________________\n","activation_109 (Activation)     (None, 8, 8, 64)     0           add_53[0][0]                     \n","__________________________________________________________________________________________________\n","global_average_pooling2d_1 (Glo (None, 64)           0           activation_109[0][0]             \n","__________________________________________________________________________________________________\n","flatten_1 (Flatten)             (None, 64)           0           global_average_pooling2d_1[0][0] \n","__________________________________________________________________________________________________\n","dense_1 (Dense)                 (None, 10)           650         flatten_1[0][0]                  \n","==================================================================================================\n","Total params: 861,770\n","Trainable params: 857,706\n","Non-trainable params: 4,064\n","__________________________________________________________________________________________________\n","\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"lLyqOWLssNTr","colab_type":"text"},"source":["## 4. Prepare training"]},{"cell_type":"code","metadata":{"id":"NzzYNOfRPafl","colab_type":"code","colab":{}},"source":["from tensorflow.keras.optimizers import Adam, SGD\n","\n","# lr = 0.001\n","# optimizer = Adam(learning_rate=lr)\n","lr = 0.1\n","optimizer = SGD(learning_rate=lr, momentum=0.9)\n","model.compile(optimizer=optimizer, loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"D64qdu8uxY3K","colab_type":"code","colab":{}},"source":["!pip install hyperdash\n","from hyperdash import monitor_cell\n","!hyperdash signup --github"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"V6O5rqHtsoSI","colab_type":"code","colab":{}},"source":["from hyperdash import Experiment\n","exp = Experiment(\"ResNet-for-CIFAR-10-with-Keras\")"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZYzNYEGzS-IM","colab_type":"code","colab":{}},"source":["from tensorflow.keras.callbacks import Callback\n","import time\n","\n","class Hyperdash(Callback):\n","    def __init__(self, entries, exp):\n","        super(Hyperdash, self).__init__()\n","        self.entries = entries\n","        self.exp = exp\n","\n","    def on_epoch_end(self, epoch, logs=None):\n","        for entry in self.entries:\n","            log = logs.get(entry)            \n","            if log is not None:\n","                self.exp.metric(entry, log)\n","\n","class LearningController(Callback):\n","    def __init__(self, num_epoch=0, learn_minute=0):\n","        self.num_epoch = num_epoch\n","        self.learn_second = learn_minute * 60\n","        if self.learn_second > 0:\n","            print(\"Leraning rate is controled by time.\")\n","        elif self.num_epoch > 0:\n","            print(\"Leraning rate is controled by epoch.\")\n","        \n","    def on_train_begin(self, logs=None):\n","        if self.learn_second > 0:\n","            self.start_time = time.time()\n","\n","    def on_epoch_end(self, epoch, logs=None):\n","        if self.learn_second > 0:\n","            current_time = time.time()\n","            if current_time - self.start_time > self.learn_second:\n","                self.model.stop_training = True\n","                print(\"Time is up.\")\n","                return\n","\n","            if current_time - self.start_time > self.learn_second / 2:\n","                self.model.optimizer.lr = lr * 0.1            \n","            if current_time - self.start_time > self.learn_second * 3 / 4:\n","                self.model.optimizer.lr = lr * 0.01\n","                \n","        elif self.num_epoch > 0:\n","            if epoch > self.num_epoch / 2:\n","                self.model.optimizer.lr = lr * 0.1            \n","            if epoch > self.num_epoch * 3 / 4:\n","                self.model.optimizer.lr = lr * 0.01\n","                    \n","        print('lr:%.2e' % self.model.optimizer.lr.value())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"aphImHK4mOlt","colab_type":"code","colab":{}},"source":["num_epoch = 182\n","learn_minute = 120"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"N5HePneGgR7p","colab_type":"code","colab":{}},"source":["from tensorflow.keras.callbacks import ModelCheckpoint\n","\n","hyperdash = Hyperdash([\"val_loss\", \"loss\", \"val_accuracy\", \"accuracy\"], exp)\n","checkpoint = ModelCheckpoint(filepath = \"ResNet-for-CIFAR-10-with-Keras.h5\", monitor=\"val_loss\", verbose=1, save_best_only=True)\n","learning_controller = LearningController(num_epoch)\n","callbacks = [hyperdash, checkpoint, learning_controller]"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"XbV3BADrsTcW","colab_type":"text"},"source":["## 5. Train"]},{"cell_type":"code","metadata":{"id":"wXGJZeqzfOFV","colab_type":"code","colab":{}},"source":["batch_size = 128\n","number_train = info.splits[\"train\"].num_examples\n","number_test = info.splits[\"test\"].num_examples"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"cR2fnuxfe6ZA","colab_type":"code","outputId":"0dff0a8a-8aa8-45aa-8e43-323873eecd5d","executionInfo":{"status":"ok","timestamp":1560073659076,"user_tz":-540,"elapsed":594,"user":{"displayName":"Keisuke Shoji","photoUrl":"","userId":"10321049352155053308"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["import json\n","\n","exp.param(\"Model type\", model.type)\n","exp.param(\"Total params\", total_params)\n","exp.param(\"Optimizer\", optimizer)\n","exp.param(\"Leraning rate\", lr)\n","exp.param(\"Datagen\", json.dumps(datagen_parameters))\n","exp.param(\"Batch size\", batch_size)\n","exp.param(\"Number of epoch\", num_epoch)\n","exp.param(\"Learn minute\", learn_minute)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["120"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"code","metadata":{"id":"Tcp-PhCuTEx4","colab_type":"code","colab":{}},"source":["history = model.fit(datagen.flow(train_x, train_y, batch_size=batch_size), \n","                    epochs=num_epoch, \n","                    steps_per_epoch=number_train//batch_size, \n","                    validation_data=datagen_for_test.flow(test_x, test_y, batch_size=batch_size), \n","                    validation_steps=number_test//batch_size,\n","                    verbose=0,                    \n","                    callbacks=callbacks)\n","exp.end()"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0f-YXgEN3Hct","colab_type":"text"},"source":["## 6. Show transition of accuracy"]},{"cell_type":"code","metadata":{"id":"kNi_-yTD3GwO","colab_type":"code","outputId":"8861dea8-9304-43a3-887f-d89a67d69e62","executionInfo":{"status":"ok","timestamp":1560081758318,"user_tz":-540,"elapsed":864,"user":{"displayName":"Keisuke Shoji","photoUrl":"","userId":"10321049352155053308"}},"colab":{"base_uri":"https://localhost:8080/","height":269}},"source":["import matplotlib.pyplot as plt\n","\n","plt.plot(history.history[\"accuracy\"], label = 'Train Accuracy')\n","plt.plot(history.history[\"val_accuracy\"], linestyle = 'dashed', label = 'Test Accuracy')\n","plt.legend()\n","plt.show()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd4lFXa+PHvyUx6b4QSQiihSZci\ngihSxAr27oooNta2ruVd17bquu/+XF9ddZXdBXVdwa5YsCB2RIr0JoQaCKT3ZCYzc35/nJnMpAdM\nMplwf64r18w888zznEzgnjPnuc99lNYaIYQQnUuQvxsghBCi9UlwF0KITkiCuxBCdEIS3IUQohOS\n4C6EEJ2QBHchhOiEJLgLIUQn1GxwV0otUErlKKU2N/K8Uko9q5TapZTaqJQa1frNFEIIcTRa0nN/\nGZjRxPNnAhnun7nAP359s4QQQvwa1uZ20Fp/q5RKb2KXmcCr2kx1XamUilNKddNaZzd13KSkJJ2e\n3tRhhRBC1LV27do8rXVyc/s1G9xboAdwwOdxlntbk8E9PT2dNWvWtMLphRDi+KGU2teS/dr1gqpS\naq5Sao1Sak1ubm57nloIIY4rrRHcDwI9fR6nurfVo7Wer7UerbUenZzc7LcKIYQQx6g1gvsS4Bp3\n1sxJQHFz4+1CCCHaVrNj7kqpRcBpQJJSKgt4CAgG0Fq/CHwCnAXsAiqA2cfamOrqarKysqiqqjrW\nQwg/CAsLIzU1leDgYH83RQjh1pJsmcubeV4Dt7ZGY7KysoiOjiY9PR2lVGscUrQxrTX5+flkZWXR\nu3dvfzdHCOHWoWaoVlVVkZiYKIE9gCilSExMlG9bQnQwHSq4AxLYA5D8zYToeFojz10IIQKSw+kC\nwGppup9bbnNQVe0kMSoUMMORP2bmc6CwgmmDuxIRYmFXThlxEcEkRIZQWuWg2ukiSCksQYqiimpW\n7S2g3Oaga0wYo9LiSUuMaNPfTYK7j/z8fKZMmQLA4cOHsVgseFI2V61aRUhISLPHmD17Nvfddx8D\nBgw4qnOfc845FBUV8f333x99w4UIYFpr8svtWIMUseHBKKXIKali5Z4CckqqsDlcJEWFEBsejM3h\nwlbtwu50ER1mRWvYcaSUqmonceEhpCWGkxAZSlZhBfvzK8gqqiQ+IpiU6DDsThdHSqrYm1eBJch8\n29yYVUSwNYgLRqaSW2bjx8x8Squqcbo0odYgUmLDiI8IYVNWMdUuF+N6JxAfEcLW7BL25VcA8MD7\nm9EaHK6Wr0f9+PlDuDKxV5u8nx4S3H0kJiayfv16AB5++GGioqK4++67a+2jtUZrTVBQw5/0Cxcu\nPOrzFhQUsHHjRsLCwti/fz9paWlH3/gWcDgcWK3yJxftT2vN5oMlZOaWYQlS7DhcysaDxeSW2jhU\nVElxZTUAliBFkIJqZ8sDZbBFEWq1UGZz1NoeYg2ie2wYhRXVFFdWoxQkRobSOymCaqem2qW58MRU\n8spsvPLjXuLCgzl1QDLJUaFYghRV1S6yiyvJLbVx7YR0woMtLN2cTU6pjT5Jkdw+JYP+KdF8uPEQ\nCsXQHrGUVlVTWFFNdJiVEEsQLq1xak14sIXRvRKIjwwmp9RGfETzHcVfS/6nt8CuXbs477zzGDly\nJOvWreOLL77gkUce4eeff6ayspJLL72UBx98EICJEyfy3HPPMWTIEJKSkrjppptYunQpERERfPDB\nB3Tp0qXe8d9++21mzZpFbGwsixcv5p577gHMt4cbb7yRPXv2oJRi/vz5jBs3joULF/L000+jlGLU\nqFEsXLiQq666iosuuohZs2YBEBUVRVlZGcuWLeOxxx4jKiqKzMxMtm3bxrnnnsuhQ4eoqqrizjvv\n5Prrrwfg448/5o9//CNOp5OUlBQ+/fRT+vfvz6pVq0hISMDpdJKRkcGaNWtISEhop3dfBJqSqmpu\nW7SOVXsKiI8IITEqhEq7k505ZTX7WIIUA7tG0yMujFFpcfRNjkIDBeU2tIa4iGDG90kiPSmCYEsQ\neWU2SiodhAYHEWoNIsQSREmVA5fWpCdGEmINwu5wsb+gnILyalLjw+kaE0aQu4dud7gItqhGrw+V\n2RyEB1tqevSNuXNa/3rbhvSIPar3JzqsfVKGO2xwf+TDLWw9VNKqxxzcPYaHzj3hmF67fft2Xn31\nVUaPHg3Ak08+SUJCAg6Hg8mTJ3PRRRcxePDgWq8pLi7m1FNP5cknn+Suu+5iwYIF3HffffWOvWjR\nIp544gliY2O58sora4L7rbfeyrRp05g3bx4Oh4OKigo2bNjAX/7yF1asWEFCQgIFBQXNtn3NmjVs\n3bq15hvBK6+8QkJCAhUVFYwePZoLL7wQm83GzTffzHfffUevXr0oKCggKCiIyy+/nNdff5158+bx\n2WefMWbMGAnsolH78su56bWf2XmklEvG9KTK7iS/3E50mItrTk5nfJ9EHC4XPeMjiAxtefhJjY+A\n+NrbusTUfhxiDaJfl+gGXx9ibXpMPeoo2hIoOt9v1Eb69u1bE9jBBOR///vfOBwODh06xNatW+sF\n9/DwcM4880wATjzxRL777rt6xz106BD79+9n/PjxALhcLrZv387AgQP5+uuvWbx4MQBWq5WYmBiW\nL1/OpZdeWhNgWxJox48fX2uo5+mnn2bJkiWAmVuQmZnJgQMHmDx5Mr169ap13Dlz5nDxxRczb948\nFixYUNPLF51LUYWdgnI7QUrRIz6cYEsQLpdm9d4Clm/PQSlF15hQThvQhe5x4RwqquT99QfZcqiE\n4amxxIQHsymrmPfWHSTUGsSCa8cwqb+UGPGnDhvcj7WH3VYiIyNr7u/cuZNnnnmGVatWERcXx1VX\nXdVgnrfvBViLxYLD4ai3zxtvvEFeXh6e8sfFxcUsWrSIRx55BGh5mqHVasXlMlf+nU5nrXP5tn3Z\nsmV8++23rFy5kvDwcCZOnNhkjnp6ejrx8fF89dVXrFu3junTp7eoPaJ9VTtdFJTbyS21YXM4Gdwt\nFqVg3f4inC5NeIgZky6urKaows627BJ2HC4lMtRKcWU1mw4Wo93D3CHWIFJiQikos1NudxJiCUKj\nzTj4h1trzqkUpCVE8MXWIwCEB1u4ZExP7piSQZcQO+z5FvJ2Qs+x0HUolOfDri8goQ/YSqDkELgc\n0GcyJPSGnO2w9mWoyAOHDcJioaIApj0KSf1g7w/w0z/A5YTKQkgeCIl9YcwNEBwGq/8NBbuhx4mQ\nnwlFe6GqGC59zTT4u7/BoXUQGm3Oay+H8DiY+bx5/uPfwcG1oIJAWSDIYto66wXz/Kf/A/k7oegA\nFO6ByGQYcBac9b/m+edPgqL9oF2gnRCeAEMvgjMeB1sp/PdiCLLCCbNgTNt3kjpscO/ISkpKiI6O\nJiYmhuzsbD777DNmzGhqPZPGLVq0iGXLljFmzBjAfHCcffbZPPLII0yePJkXX3yRefPm4XQ6KS8v\n5/TTT+fSSy/l9ttvrxmWSUhIID09nbVr13LBBRfw3nvv4XQ6GzxfcXExCQkJhIeHs2XLFlavXg3A\nySefzO23386+fftqhmV8e+9XXnkls2fPbvRCsmg7Lpfmpz0F/Lg7n5LKamLCrAzuHkN+uZ3NB4v5\nMTOfve7MDY9gi6q5KNiQmDArg7rFUFrlIMxq4c6p/emVGIHd4WJnThlHSqqIjwhhRM84pp+QQniw\nhazCSpZvz6G4spqEyBBOG5BManwExRXV2BxOkqNDUeW58OmtsP0jcLg7DWc/ZYJ7SRa8d2P9xlzy\nqgnuZYdh3WsQmQiWUBOYw+Oh0j30WFUEub+YABkWA5vfMR8S/WdAUgYER8BPL4HLXJwlqisk+4yR\nVxZC7g4T1IMsEBIJcT7JC2FxJmC7nCY4u5yAT+eqqhjKjkB8OvSbAhX55kPFY8iFpo0qyHzyledB\nVIp5LiTKHL/kIGRvaP6P3gokuB+DUaNGMXjwYAYOHEivXr2YMGHCMR0nMzOT7OzsWsM9GRkZhIWF\nsXbtWp577jluuOEGXnrpJaxWKy+99BJjx47lnnvuYdKkSVitVk488UT+/e9/c+ONNzJz5kw++ugj\nzjnnHEJDQxs859lnn838+fMZPHgwAwYMYNy4cQCkpKTwj3/8g5kzZ6K1pnv37ixduhSA888/n+uu\nu45rr732mH5PAVSVmIDkw+XSFFbYOVxSRU6JDZvDpPj9mJlHZk45yTGhVNqdbDtYQHapA6XM2HC5\nzYEn6y46zMq43gnMHNGD5OhQktyZHmv2FWB3uJjYL4nosGDK7Q6iQ63EhVuJiQghKTLUXGx0uUwv\nNLFvs79Cz4QIfnNyer3tsRHBuMtNwRcPwbYPYdTVplebPBAik8xzXQbDratN7zo0GuJ6giXEBD2A\n3qfC/2Q13oCBZ5ufmjfQ6f0AABhxuXm+YDck9oPQqNqvn/4n89OYKX9s8vdn1vNNP3/q7xt/Tim4\nYrG73Q1/4LY2pXXLU45a0+jRo3XdxTq2bdvGoEGD/NIe0biVK1dy//3389VXXzW6z3H7t9Pa/Mdt\nyrrX0EvvZdNF37GnIpTiympW7MpnzS97me76gXFB23jXeQrfuIZjxcHk0F/okhDHOSWLGezaQawu\nZdnUT5gw9iTCD3yD/chOckpthCWlkZg+FBWfbnqiAJVFZqjB1/rX4eDPkPmlGVKI7mqGSi5aYJ5/\nZgSMugYm3AHV5bD9Exh+qXnuh2fMcEjxAdMTDYuFvpPhjCfM8ENloTmexy+fQeE+GDe3Vd5eUZ9S\naq3WenRz+0nPXTTp8ccfZ/78+TUXdo97tlLT63TYYNkjsGaBGZf1DAuc+SSExbL/61eo3PIx663D\nOD/7aTa7enPBAjNe/W7IQ0wOKqObNQ+rdlAZ0Z1hJ15I5aBTiDi0gvSPHoNCICIRBl0E8b2YOtH9\n7XDDYkI2vkGqb5uiusLdO6BwL8w/DU66BU65GzxDaHu+hU1vQfopMPAcM7TgGY5w2KHXyfDlI2ZM\nWinzO3YfaYY0KgvN2HhcGqSOgfJcKM02QyO52+HFU2DO59BjlDle/zPa/m8gWkR67qJVdPq/XXUV\nfPmoGXK46Vsoy4WXToEBZ+KsLKEyJxNHVRn/Sr6fVZxAn/1vc591EXGqnBxrN94c8Sr90tPIiA+i\nxze/IzQ4GBXfCwadawKpp/dfXQm7vzHjz0Muqj+0UFUCTrsZkijaB3m/gL3C9JRtpfDRXbDpTRPE\nKwthxp+h2/Cmv2Fobcavs9aAvRROnA2pTXQMXS7zwVFVAk8NhJhuMP5WczHx5NsgQlJl25L03IVo\nLTs+haX3mGA6di5YwyG5P7su+4YXfq7i88wjlNkcJEWFkFQZikvbSZ9+C2rkQ7D/C7r0OJF58ene\n413xn8bPFRwOA5q4OO87bh+dYoZXPEKj4YL50G0YfP5HM1QT7M6UamroSCmT1TH0oibfhhqebwRh\nMWYc+ZXz4KM7zTcDP3UWRX0S3IVoSmURvHGVGXq5+n2Ku03kq805LN+ew4cbDxEVYuWsoV2ZOaIH\nJ/VJrD/DcciF7dtepeDk35qLl/Yyk0LYlnpPgnOeNuc66Rbv2L/wOwnuQjSlPM+k1p3yOw7Ej+Oy\nZ7/jYFEl0WFW5kzozbzT+xHXDnVCjlq/Ke13rtHHvPiaaEMS3IVoSmwqzP2GbJXEpS/9SLndyevX\nj2NcQ710IToQmZHiIz8/nxEjRjBixAi6du1Kjx49ah7b7fYWH2fBggUcPny40eftdjsJCQk88MAD\nrdFs0ZaCw6D7CJ7/qZC8cjuv3zCOk/slSWAXHZ4Edx+ekr/r16/npptu4s4776x53JJa7h7NBffP\nPvuMwYMH88Ybb7RGsxvVULkDcZRytlO18l989vMuzhnWjRO6H10FQCH8RYJ7C73yyiuMHTuWESNG\ncMstt+ByuXA4HFx99dUMHTqUIUOG8Oyzz/LGG2+wfv16Lr300kZ7/IsWLeKuu+6ia9eurFq1qmb7\nTz/9xPjx4xk+fDjjxo2joqICh8PBnXfeyZAhQxg2bBgvvGDqXKSmplJUVASYSUZTp04F4IEHHuCa\na65hwoQJXHvttWRmZnLKKacwcuRITjzxRH766aea8z3xxBMMHTqU4cOH84c//IEdO3bUlEEAk944\ndqxPNsbxaO93hH36O7S9kivHte3iCkK0po495r7w7PrbTpgFY28wub3/vbj+8yOugJFXmiJFb15T\n+7nZHx9TMzZv3sx7773HihUrsFqtzJ07l8WLF9O3b1/y8vLYtGkTAEVFRcTFxfH3v/+d5557jhEj\nRtQ7VkVFBV9//XVN737RokWMHTuWqqoqLrvsMt555x1GjRpFcXExoaGhvPDCCxw6dIgNGzZgsVha\nVOJ3+/btfPvtt4SFhVFRUcEXX3xBWFgY27dv5ze/+Q0//fQTH374IUuXLmXVqlWEh4fX1JIJDw9n\n8+bNDBkyhIULFzJ79vF9sUxXFqKArikpjEqLa3Z/IToK6bm3wLJly1i9ejWjR49mxIgRfPPNN2Rm\nZtKvXz927NjBbbfdxmeffUZsbPNf2ZcsWcK0adMICwvj4osv5p133sHlcrFt2zbS0tIYNcrM9IuN\njcVisbBs2TJuuukmLBaTYtaSEr8zZ84kLMwUNLLZbMyZM4chQ4Zw2WWXsXXr1prf6brrriM8PLzW\ncefMmcPChQtxOBy89dZbXH755Uf/hnUipYV5lOtQLhjTRxYCFwGlY/fcm+pph0Q0/Xxk4jH31OvS\nWnPdddfxpz/VLzq0ceNGli5dyvPPP88777zD/PnzmzzWokWLWLlyZU2J39zcXL755hvi4o6uV+hb\n4rduyV7fEr9PPfUUPXv25LXXXqO6upqoqDozHuu4+OKLeeKJJ5gwYQLjx48/6nZ1NgX5OYQQycn9\nEv3dFCGOivTcW2Dq1Km8+eab5OXlASarZv/+/eTm5qK15uKLL+bRRx/l559/BiA6OprS0tJ6xykq\nKmLlypVkZWWxd+9e9u7dy7PPPsuiRYsYPHgw+/fvrzlGSUkJTqeTadOm8eKLL9aU8PUMy3hK/AK8\n8847jba9uLiYbt26oZTilVdewVNuYtq0aSxYsIDKyspax42IiOD0009n3rx5x/2QDEBFcR5lRJHR\nyAo/QnRUEtxbYOjQoTz00ENMnTqVYcOGMX36dI4cOcKBAweYNGkSI0aMYPbs2TzxxBMAzJ49m+uv\nv77eBdV33nmHadOmERzsXUNx1qxZvP/++wQFBbFo0SJuvvlmhg8fzvTp07HZbNx444107dqVYcOG\nMXz4cN58803ALOB9yy23MGbMmCYzeebNm8e//vUvhg8fzp49e2pKAZ9zzjnMmDGjZqjp6aefrnnN\nlVdeSXBwMFOmtONEmA7qYW7kH90fk9RHEXCkcJio58knn8Rms/HQQw+1+DWd8W9XWlXNsEc+5/Yp\nGdwxtf7CyEL4gxQOE8fk3HPP5cCBAyxfvtzfTfG7dfuLmBP0MVNCpgMS3EVgkeAuavnwww/93YR2\nV+108f3OPCJDrYxJj6/Jilm7r5C7rG9jLY0BZvm3kUIcpQ4X3LXWknIWYPw1tOdRUlVNdlEVGV2i\nzNJxPspsDg4XV7Erp4zXVu5j3f5CEqJCGJOewAUjU1m5O5831xwgp9QGwNAesaTEhJFbZiMzO587\nrTaIlkwZEXg6VHAPCwsjPz+fxMRECfABQmtNfn5+TV59ayiuqGbLoWJGpMUREeL9J+pwuqh2arYd\nLmH5thySo0OJCbfy+MfbyCuzkxAZQnpiBBHBYLPZySp1kV3sTRPtGhPGBaNSKayw89nmw7z780G6\nqEKuSM1nyLmzOFwBi1fv52BRJQmRwVw7Kg424l3jU4gA0qGCe2pqKllZWeTm5vq7KaI5VcVmhfrg\nMMLCwkhNTW3+NWtfgQ9vg9P/iD7ld2w5VEJmbhn78ivYl19BZbUDreGrHTlUVbuYGryJwYlB7O4y\nlV05ZWw/7E0vDVLULBI9uFsMd00bwNp9hVQUHeHpg1fxRtz1rO9zCf26RJEaH05ydChj0hMItpgE\nsZKqar77JY+Tyz4j/vMHoftZkJTBVSf5lBjI3WGCu2cBZiECSIcK7sHBwfTu3dvfzTg+leWa5dti\nezS/r8sFj7oD3sPFlNkcbM4upaDcTlW1iyAFGSlRJEWFsvVQCZsOFrPpYDHn7n6XM4D8r//BBT8O\nZ1+hN000JSaUqFArNoeLmcN7MHlgF2a8fQUUwST7GNISIpg3OYXIUCvdYsM4fVAXCsrsZOaWcUpG\nMiHWIK4YlwZHtsI/bPymZy6/ubB++QcANr5FTEEmZ0+6Bx693WwrPWzWQfVVaWr31FtwWogA0KGC\nu/CjhWdC/k74Yx5YghvdLb/MRtbhIwx3P770pR9Zu68Qh6vpcfceceHMDakEByS68pgRvZt+U85i\nRM84eiZEEBbcwAo+b5ubb++Z3OAxY8KCSU+KrL3xl0/Nbfb6xhuTtQrWvQYnnO/dVtpAFc/UMXDv\nPrP0nRABRoK7MHqOM8F96wcU95vJ1kMl5JXZ+HF3Ph9tOERqfASj0+N5e20WFXYnTwVPZFzQdnJK\nbbzU7ycm5C5i++UriQgNxu5wsS27hIJyO4O6xTCkRywJkSHw7ychYSzkbOP+1M0w+sbG22Ov8N53\nOsDSwn+qle7CaoX7zOv2rzBLwXlsWGwWjq6ugDULvNvLGgjuQUHSaxcBS4L7caqw3M6GrCIyUsy0\n+ruPXMHjejm2d59gnq2Q31re5XnHLA5Y05g+uCu7csp49cd9nD20G2cP60bkhgmEVITy5ZxJBP1p\nJgAjXFshZSIAQ3o0UEStsgiSB5h1Ngv2NN3AkAj4wxFwNRLYV/3T3I69oc45Cs2t02Z66O/eCHM+\nh8gk2PI+fP0ERHcDFPz0IgRZQbsa7rnv/gYyv4TT/scs2iFEAJHg3kkVVdjZml3CsNQ4jpRU8cG6\ng6zPKmZPXhlV1S7yymw1C9WHWoPoZznM3u5nc3r2P7n3hDKm71zB2BlXETt6GpGh5p9Jhd1hslfW\nvQZF78Ocz8ASBLE9ofiACZRNCY2CmB7QZRA4q+s//+MLJpif/Fuz0LMnoGptHvva+QUUZNYP7hWF\nEBoD4+eZIZrSQ1BVBDuWwtLfm30m/R4cNjN002UQzHgS4tLqt2ffCvjhGZjS8pm6QnQULQruSqkZ\nwDOABfiX1vrJOs+nAa8Ace597tNaf9LKbRUtYHM4eXNNFk99voOiimosQQqnS2MJUgxIiWZUWjwR\nIRa6xYYzMi2OjVnF7Mkr58m9d2BNOgVCJjB9aCrshO6uIxDq/SdSk5ZYsBvyd5kg6nKaXu+E2yHt\npKYbd0MDs173r4RP74NxN8NXj5tedXwvQMGGRbD3Bzj3aRhyofc1hXvN+Qsyofhg7YvAlQXQbTic\ndi+8eIq533UoJA+CDa9D9gbof6b5HbLXw4mzIX2iz+uLvEMxVUXmdwxq4HqAEB1cs8FdKWUBngem\nAVnAaqXUEq31Vp/dHgDe1Fr/Qyk1GPgESG+D9gofLpdmQ1YRa/cVsievnKpqF9/8kkNemZ2T+iRw\n7cnpbD5YQky4lVkje9Aluv7QwikZyebOnysgIgku/Jd5/Om9ULS/4ROXZIN2wjPDYcqDcM0HENUF\nSo+YseyEo8h4CrLCoXXw3lxzPz4d3rsJTrgAdn5uevJ1h0yy1pjADrD3O7CXwZEtcM7Tpv0OmxlS\nObwRzvp/Zj+LFS573WTTRCWb3vjUh93HW2uOl34K/G0gTH0EJt5hAr3kuIsA1ZKe+1hgl9Z6N4BS\najEwE/AN7hqIcd+PBQ61ZiOFmdjz1toDdI0NI8xqYUVmPks3Z9dM0omPCCbEGsTw1Dhmj+/BhOJP\nUAc/Ysb0x+oPadSltQmQIT6ZJ3FpTQT3g5AyBI5sNvsMu8RcvHxqgAmsE++AiXfW7vGW5cK7N5ge\n/oFV5mLm3TsgdTT84TB89zfzAREaA7u+MEMq3UfC4U1Qmg0//we2vg+X/tcMAYHJs1//Ouz5xjw+\n+28Q6863f/9mc+vb44/pbn6g9nuy6S1Y9x+40p2es+wh086qIrmgKgJWS4J7D+CAz+MsYFydfR4G\nPldK/RaIBKa2SusElXYn6w4Ucs/bG8kqrKzZHmIN4pR+Sdw7YyAn902kS4y7V15VAv+aCnk7zOMx\ncyChT9Mnqa40FxVDfRby6DIYKhpZ0q80G5IHmtttSyCxLww6D656G5Y/Bsv/ZHrgQy/yvqY8F3Z/\nBaOuAbTJTnE5odzUyOf0P5jb3F/MbUUe9JtiXld62Fwozd4AKsgMxYTFQsZ0yFrtPUdFvrke0HuS\n+RAoyYKI5leuIjrFfLh1HQrn/B98dIcZLnJWu4eIhAg8rXVB9XLgZa31U0qp8cB/lFJDtNYu352U\nUnOBuQBpaQ1cwBKAGTd/9+eDvLnmAOv2m4k0PeLCefum8YQFWyizORjRM67h3PA935rAfuq9ZlKS\nJbT5E9rLzW2IT3Cf9YL3fnUVWEJMaiBAr5Ohywmm1569HpbcBoNnmZ72FW/C//Y2gdw3uFf5TAgK\ndS98YS+Dz/4HDv0Mt60z2xL7QVAwuKohvrc5R+lh08byXNOrLs6CmFS44J+mB751Cbx5tRmaWfYQ\nTH8cTp5nhl9aIrqbuS07Yr6FfPEQrP4XXPGGN/tGiADTkuB+EOjp8zjVvc3XHGAGgNb6R6VUGJAE\n5PjupLWeD8wHU8/9GNscuKqrYO/3kOHzxebwZnjnerhiMa7YXqzcnc+DS7awK6eM/ilR3D4lg4yU\nKE7JSCY2vPHJRTVyt4M1DE75HVhbENjBpB2e93dIHVv/OYcdHk+BcTfBmX8x2859xtzu+cYE98S+\n3mGOIIsZu97zXe3jeGZ7hsV5P0RspWArMUMxHkFBENPNBPWEPubDoLoCvn3KPJ+30/T6Y1O950wd\nDec9551s1JLeuq+oFHP7/Dg452/mW0REopnMFdXl6I4lRAfRkuC+GshQSvXGBPXLgCvq7LMfmAK8\nrJQaBIQBUiCmrqW/h59fhZt/hJTBAOzfsJy03G2s+ec8brLfQV6ZjR5x4Sy8dgynDUiuX0CtosCM\nOXcb3sAJgEl3w5jrTWC3l5uwXOFfAAAgAElEQVRebvKAptsVEukeLvFxeDN88nsYMMM8/ulFmPYn\nE/A8bcqYBts/Mr1tX6fdB8pSO4WxoZ67rcwMI4XF1H79NUtMgA6NMa8v3AtfPmqey98Jc782H5Qe\nMd1h1NXmwihA+FEGd0/P3VVtJk+ddNPRvV6IDqjZZfa01g5gHvAZsA2TFbNFKfWoUuo8926/A25Q\nSm0AFgHXan/Xge2Iug4DYPPBAu55ewMz/u9bJn3Vl8WuqQyr+JEZaS6evnQ4n985ickDu6Ds5SYz\nxePIVnhpkunpOx2Nn8dzEfDju+Hlc5pvV2WRyUCx+az7agkxszt/fN48Thlihi12fw1PpJpAOuwy\nQNUP7l2Hmg8vT2DX2nzYJPU3RbgS+8KIq0xPu27PHUy2TVis9/UV+dBrAmScYXr0wRGmd+8reyPs\n+97cP9qee0IfkyEDjX9oChFgWrSGqtb6E611f611X6314+5tD2qtl7jvb9VaT9BaD9daj9Baf96W\njQ5YMSYf+6H3NvLZliMkRIbwyHkncN6tfyUkCB5LWsb5I1NrJg3x8V0mNc9ZbWZ0/nu6ufh5+h8b\nnrW5bwUsusKb5dJtOJTnNDz70tfBNfCvKebDwyPOPRJXdgQGngM3/2C2lWaDvdR8gBTtM8Mw8Q2k\nPm77CL78EzwzAj5/wGStzFttgnu34TDreXOxsqrEBPLGbHob/nk6nPusGQ/XLnjrWsjbVXu/t66F\nLx4094+2524NAUcVoMwHkxCdgMxQbSdZhRVUrv2SDCA5XPOP304yeef/vQTUaaaIVfaG2i8aPBM2\nvmEuku77AarL4abvTM828ysz5j31Ye/+u7+GX5bC+S+ax93MNwWyN0J018YbV3NB1ScVMjjcjEVn\nTDfj8QAHVpuURDBDIQ4b9J8BfRso7LXpTdj6gbmf+VX957U2P6f/wZue2JBI90XR0mxzwbbbCNjx\niZmBmuTzjSE21WT7XL4YIo9hnPzrP5tb34whIQKYBPc24HRpVmTm8cOufIorq1m3v5Dth0tZHfom\nKLj/jH4msDvsJqe723Dv5CFffadASDRseRd2fwt9T/dOEDq0Dr5/Gk6+zTsMsW+F6Xl6xrBThpjb\nwxug//Tax3Y5YdHl5gPEM/wRUqfCYnQ30zv3PL/1fTNUE55ggn9wOFz234bfhJNvM6+P7ubOcnnE\nHOuiBeabxbMjzYXZkVc1/WZ6xsMXXwn37zfXE5bMq1+aOLYn5P3S9AdFUybeWf/3FyKASXBvZQcK\nKrj8nyvJKqwk2KKIDQ+mT1IUD5w9iIitJ6CDodeJZ5idi/aZYYbEvg0f7L25Zghkywcw5Y+1x7Z7\nnGhuD/5ssm9cThPwh1/u3ScsxownH95U/9hlOebbQGJf77BKSJ1ea3i8yU7xmP6YyT13uWhW6mjz\n4/Hfi80Qj+c8LodJMzy03gzPNLYghucbR2SSufXk70fXCeKxqaZ3/93f4JS7mm9fXVMfPvrXCNGB\nSXBvRRV2Bze8uoaSympeuHIUpw/sUjsXfXMphPtMiinYbW4T+pgg9+WjMOPPJrvF6TD5256ZoIl9\nTWD16D4SUHBwrQnu+btM3niPUbUbdfZT5gKkL2c1/GO82b8i3zv0UXdI4oo3ahf4Usp8ezhazmoz\n8cgToD0fIvm7zHj8eX+vn63jERYD58/31n9Z4R4isobU3s8zM/Wrx48tuAvRyUhwbyXlNge/XbSO\nX46UsnD2WE7t38AEmrydJlDvW2EmAuW766Mk9IXCPaa8bMEeE9zLcwENI9xZpyl1LvSFxZj9Dq4x\nj+1lJk/d06P3aCgY7/7GOzmnIt9cMI1LM/nxvqyhLc+Vb4yzGp5MM7nqg2e6jxtizlXsni5RN1um\nruGXeu9f8ZYpf1CX54MvqZm0TyGOExLcW8GBggrmvLKaXTllPDpzSMOBHUyNcTBjw71ONgG610Qz\nZu5w522XusvyeBaPiOsFgxpJZ+wxGnK3ue+fCNd/UX+fnO1mKr9v5cOt75mA2m24Ce7J/c1PW7AE\nu8fDd9QeegmJghL371o3z70pda8deMR0N7+PZ0KSEMc5Ce6/Rskh9u7fz9Xv51PiDObV68YxMSOp\n8f1vWG7S+hzuID/yKu8FxagUQHnTFkvd49NNZbmc+4w3JdLl8pYH8LXi76YUwF0+aY65O8zwTcoQ\nM0Ho8CYzeSetbsmgVpIy2AT3LoO928Zcb6455GyB0CZSIVtKa5NtJD13IYAW5rmL2spsDv7+5U5e\n/+f/I/3tM/jacRUfXBzfdGAHU2wLvL10Xxarmepe4lNQMzHDmy3SEE9gryiAv/Yxi13UFRJphmx8\nVRaajJczHjfZLt89BR/c0nTbf40uJ5jbEVd6t02+32QDwdH13BvjyejpNf7XH0uITkB67kfph115\n3LZoHfnldl6JzcSuwrBaFOm/LITBYxp/YelhWO9OG6yuMtktTw0wU/XHXG+2p47xzi4dMMM79b8p\nm96G928xQz4N1UEJjTJ57L6lAKY+XHuIxF5eP1OmNblLLZC73ZtB46w222e92PQH2NF4qKj58sZC\nHCckuB+F4spq7nxjPbERwSy4dgzDl/wJ4k41mRo/v2rKxdbN4vDIz/TWR7FYzQXG8tzaC0E3ljPe\nlLTxZpELp82dQVNHSKRJO3TavRdHB51rbnd8Cl89Zj5s2nKsuttwU2sGn8C7+EqTunjTd42+7KhJ\nYBeihgzLtJDTpXni423kldl45tKRDO8aZi6Mdh1i0hRd1eCobPwAnuyUG781FRs9QT0kouH9P/k9\nfHBr8w2L7QFnPGYyZRoqA+DpkXtmoTrspjJleZ75QDi8yYy7t+UEnthUeKgAUn0yeUKjzUpJB1a1\n3XmFOI5JcG+Gw+nilv+uZdAfP+WNNQe4YVIfhqbGmiEG7TQXJS3u3rrD3viBKt0LX3iGQ6rdwTbY\nJ6hufBOeP8lccD241psq2JzR15lMmYYuqA44E65+3xu8Sw/By2eblY4iEs02V3X7z870VIZcdFn7\nnleI44QMyzTjxW8y+WTTYS4f25PRvRI4b4R7ZmRiP7j6PVPpsWg/TP4DBNdfo7SGp+e+/DFTH6X3\nJPPYU4McTFDP3WbG50uPmCqKv1ZcmvmpaYen9G68N7iPuxlGXln/tW3JM2GquRx3IcQxkeDehE1Z\nxfzfsp2cM6wbT5w/1NRWt5fD4R1mHNkzQSgyqf7M0LoqC80KQwfXmouJA2aYYmG+gddzYbHkkJmq\n3xrj4OV5ZhHpXhPNykSeDxnf4J7Ur/2rIXqCutRzEaJNyLBMIyrtTu54Yx1JUaE8NmuId9GM7/8P\n/jkZHusCe38w26qrzBCK77DMgdXw/q3edUhPux/u3ALWcNNDT+gDF79c+0PBU6M8Z4sZKmkqx72l\ncneYcrg5W9y/mE9wD4+HnieZqo2+NWTaQ/op5laCuxBtQoJ7I/7y6XYyc8v5fxcPJy7CJwMmazXE\npsHAs03JAICdn8PTg71FrcCMba9/zZu3bg01CzFbQxvOcwdvzz0/E3qfCkkZv/4X8QRPzwXVKp/l\n7izBZmhp+0ew/eNff66j0Wu8yX+PaGZugBDimMiwTAM+3ZzNyyv2MntCeu2JSZ5ZkIPO8dY4B+8F\nVadPz73AHfjL3asN/jTfBFprmAnuW94zC0vP/dpbFdLTmz75t8deurauutky/abCpa95i3g1VMu9\nPTjspsBXTI/m9xVCHDUJ7nVszCrijjfWM6JnHPfOGFj7SXs59BznHVLw8OS2+1ZQXPuyuS3PM7c/\nv2pWMorraXLcPYtDW3y+FSgFN33feoEdfHru7lmqdS+wvn6Je792XqRi77fwzhy4ThbtEqItSHD3\nUVpVzY3/WUtiZCj/vGZ07XK9YDI8rlhc/4U1qZC2+s95eu6VhWZlpFnuEgEr3asl1e0xe0rXtpaa\n4O7Oq89aY745eAqJ1dRYb+eeu+fDJHtD29W0EeI4JmPuPp76/BcOl1Tx3BUjSY5uoNRtY4tSWxro\nuXuGaDy3lQW1p/xXu4OtbypkWwiJgjlfwNCLzePvnzYTpDw8s1bbO7gr9z+9jQ18WAohfjXpubut\nP1DEKz/u5ZqTejEyzTPRqNIs4BztTkl840pTE+aqt2u/OK4XnPGEd01PrU2PfcLtMPEO06OvrjDB\nfcVz5qJsUgag6tdQb21BQdBzrPdxZVHtD5nek8yiIXXrwLc1z5h/mhT6EqItSM8dyMwt44ZX15AS\nHcbdZ/iUjH3neniqvwnoYFZLimwguyM6BcbfCvHp5nFVsemxexZqLssxtVWiUkwg3fu9mfw0enb7\n1EPZ/A7scddwqSysHdzj3CtDWRqpidNWEvrAzStg6iPte14hjhPHfXDPLq7ksvkr0VrznzljiQ4L\n9j65Y6m5zdtpZoyWHTZBuS5nNeT+4s0hD4mEOcvMzNWl95qLqA/kwLBLzDCMowoGnwfnPN32vyCY\ngmXr/mPuVxaaNEiP6K5gaSI9sy2lnOAtWyyEaFXHfXB/4atMiirs/Pf6k8hIia79pCdF8fBGOPCT\nud/Q8EXJIXh+jDdX3BIMPceYQOr5gLBYvcvWOaq83wbaQ0iUN+WxstBbVhhgyIVw1TtmNSghRKdx\nXHebckqqeGPNAS4clcqArtH1d5j2JziyyVyM/PQ+M7u0obK6nouSnounub+YcfXgcJMKuX4RZK+H\nM/9ixthdDlPytiTLpD62teAIb03333wIkYm12977lMZfK4QISMd1z/3f3+/B4XRx06l9G95hwAyY\n9HszLt77VDjt3obrtdetCrn7a7OyUUiUqf647UNThRFMDnvXYWZc3vIrF59uqZBIE9yVMt8oEvq0\nz3mFEH5z3Ab3ogo7r63cxznDupOe1Ega4JEtsPldU1c9YzpMvLPh/SzucXpPz708x6T6eRadPrjW\nO3Fo5FVmgQrtavs0SA9PcC/Ph3X/rb2UnxCiUzpuh2VeXrGXcruTWyf3M0W/YnuYYYvtH5tSu8n9\n4dWZZvJPdTlknGFWMGoou8VSZ1imLMdUXIztaUoD5++CjGm1X1NdAeGtOBO1KTOeBLRZXOSDW0w9\nmdacBSuE6HCOy557mc3Bwh/2Mm1wCgPyl5miX3u+M1P037jSXBwFk+Pe51Rz/82ra9eO8WUJNkvs\nZUw3j8tzTRpkvylwozsFMd6dcrjrS3jpVFOtMbiRVZhaW1xP883BtyKkEKJTOy6D++s/7aO4str0\n2j2LV+Tv9AY/MBOPnDbo7lOS19rIGLlSJme9mztNsizH1E4HqMiDmFTvEnj2MnNxddC5ZpWk9rD/\nJ/jhWQnuQhxHjrthGa01i1cfYEx6PCN6xkHUZPOEJdRbex1Mrx1M2uBFC2sX22rIofVm3/h0s9C1\nw2by39+/BU67D4ZeZPbzzEg96Zbaa4q2pd1fw9dPmJryIGV2hTgOHHc99y2HStidW875I90Fujxr\nmFbk1+65VxWb29AYGHIBpI5u+sD/mQU/Pm/uR3c1wzAW98pLudu9+3mCe9nh2rVo2pKnbsy+H8w1\ngNB2rgAphGh3x11w/2D9QYItirOGulc5+tI9/d1WWju4RyTABf9secVCS4i7hkwlfPNXOLzJbK+u\ngB+f8+7nCe6LrzBFvNqDJ7if9RRctqh9zimE8KvjaljG6dIs2XCIU/t38a6uVJEPSQPg9D+YVMFT\n7wVbmbnYOeySlh/cEmJ64hX58NVjENWl4XVJIxIgZaiZHNVeF1Q95XWVap3VnYQQHd5x1XNfvbeA\nIyU2Zo7wSQMsz/NWfYxMhMn/AzOeMMvR7f3B9MRbwhJismk8+3t6y+c9B2f/zbtfUoa3qmR75rkD\nfHp/7esKQohO67gK7l9sPUKINYjTB3bxbqzIgz3fwvLHTVbJlvdNwN/2Ibx8lsl8aYm6wd0z/DLq\nahgzp/a+7b20Xd/JMHYu7PoCgizN7y+ECHjH1bDM8u05jO+TSGSoz69dnm9ut7wHxVmw4XXzuLc7\nvz0spmUHn/4YhEZ7g3tjvfKKAvi7O72yvYZlgsPNrNSEvhAW2z7nFEL4VYt67kqpGUqpHUqpXUqp\n+xrZ5xKl1Fal1Bal1Out28xfb3duGXvyypkyyKfXrrWpwx6X5s2W8YxPF+0zt6EtDO4ZU83FV0cz\nwd2zAlFYLHQZdPS/yLGoKIDtH0HQcfVZLsRxrdn/7UopC/A8MA3IAlYrpZZorbf67JMB3A9M0FoX\nKqW6NHw0/1m+3QyvTB7g0zSlTDEwlwO+/asZoknoY0r8Fu41gb6lwxjZG8xxek2A32c2/qHgCfoT\nbm+/i5u2UnPrWUxECNHptaTnPhbYpbXerbW2A4uBmXX2uQF4XmtdCKC1buFAdftZvj2HASnR9Ezw\nGQqproLSw+6hCg0Fe0x+uqd33dJeO8AXD8LS+0xue2RSw9UjwVtBMn9342uytrb4XnDxK3DRgvY5\nnxDC71oS3HsAB3weZ7m3+eoP9FdK/aCUWqmUmtFaDWwNJVXVrNpTwOSBdb5Q7P8RnhpgeumRXUzP\nPSLJrFSUNh5mPtfg8RrkuaCatdasfOSZBFWXp/DY+tfMknvt5YRZMnlJiONIaw3CWoEM4DQgFfhW\nKTVUa13ku5NSai4wFyAtrZnp/K3ou1/ycLh07fF2MOPsAGNvgLP/H+TtMvVjUk4wwzP9prT8JJ48\n90M/w3dPwbibm39NSDtdUBVCHHda0nM/CPT0eZzq3uYrC1iita7WWu8BfsEE+1q01vO11qO11qOT\nk5OPtc1H7cvtR4iLCGZkz7jaT5TnmVtPrZWkfqaC4tgbTA/70PqWn8QSYgqNedYiDQ5rfN8ZT7r3\nkeAuhGgbLQnuq4EMpVRvpVQIcBmwpM4+72N67SilkjDDNO045tA4p0vzzY5cTuufjNVS59etyPOO\nry88C145D/IzTdD/z/mw4tmWn6hennsTE5TaO89dCHHcaXZYRmvtUErNAz4DLMACrfUWpdSjwBqt\n9RL3c9OVUlsBJ/B7rXV+Wza8pTZkFZFfbuf0QSn1nyzPg/AE08ve94PZlrMNtrk/u47mgurJ88yE\npV1fQlCwWRC7Mcv/ZG4tjVx0FUKIX6lFY+5a60+AT+pse9Dnvgbucv90KMu35WAJUpya0cAw0Amz\noNvw2sMjEQkm4EPLJzCBt47Mto9aPtzS0KpOQgjRCjr9rJYvt+dwYq94YiOC6z/Z5zTz4ys83nuh\nM/gohk1ytpnslzOegKkPNb1vz3He8gRCCNEGOnVtmUNFlWzLLmFK3RRIjyNboSy39rbwBFDuiUva\n2fKTbVgEb82GoKDmC4KV5Zhl9oQQoo106uD+1Q4zl6peCqTHy2fD13+uvS083juTs9eElp/MEmou\nqK5ZAN//X9P7Fu4xi3UIIUQb6dTDMsu35ZCWEEHf5AYm7+Rsh8oCSB5oHv9PtlnY2hoC6RNMqd6G\n6rE3xhICaNi6xExgmnhH4/ve/KNZxEMIIdpIpw3ulXYn3+/K4/KxaaiGLlxueRdQMPg88zgkAkJ6\nmfvx6Udfh8XiHtOvKm7+gmrK4KM7thBCHKVOOyyzcnc+Noer4SEZrWHzu5A+0ax32hqsoebWVtL0\nBCYhhGgHnTa4r95bgDVIMSY9of6Tudshf6dZ+Lq1DJ4J131uJkW11wpLQgjRiE47LLNufxGDusUQ\nFtxAyd7kgXDTDxCb2nonjOlufoKsR5dCKYQQbaBTBnenS7Mxq4gLT2wkeCsFXYe07kkL98He7+G6\nT2W1IyGE33XKYZldOWWU252MqFsozOOHZ01WS2s6uBY+uMXUhxdCCD/rlMF93f5CgMaD+4pnYdey\n1j2pp07M29e1/geHEEIcpU4Z3NcfKCI2zErvhkrDOKtNPntM99Y9qSe4H9lsLtgKIYQfddrgPidp\nM+rPPerXZPcMm7RWCqSHxad2jdSNEUL4WacL7hV2BzuOlDI2LMts+O6p2juUZpvb6DbquYOkQgoh\n/K7TBfedR8rQGopOuhdOuhW2f2QW4PAodxcKa+2ee7fhcPlic1+CuxDCzzpdcN9xpBSAAV2jYcLt\npke9ar53h4FnwwM5Zp3U1hQaBUn9ITL56Bb5EEKINtDp8tx3Hikl1BpEr+W3wKBzYfIfoPuI2jt5\nSgW0pspCyFwOsz81a7EKIYQfdbrg/suRMoYmKYK2vg/dR3qrM+Zsg+Iss6BGVTGcek/rnriiAD65\nG86fL8FdCOF3nW5Y5pcjpYyPKzEPEnp7n/jub/DBPNj2Iez8ovVP7MmWeW8u5O1q/eMLIcRR6FTB\nvaSqmuziKoZEuNfmTujjfbLnWLNAxoFVENOt9U9u8RnqqS5v/eMLIcRR6FTBfaf7Ympfi1mBqVZN\n9p7jzK3T1vppkFA7z72lC2QLIUQb6VTB/ZcjZQAkxUSY1MTQaO+TXXwWyGjtNEionecuk5iEEH7W\nyYJ7KeHBFmKm3A03flv7SYvVO0wT06P1Tx4cYfLqPfeFEMKPOlW2zI7DpWSkRBEU1MCyegBXvQsR\nCW1TkjcoCLoMgrheMolJCOF3nabnrrVmW3YJw1JC4fmTYNPb9XdK6N22tdYrC+G8Z816rEII4Ued\nJrgfLqmisKKacTEFkLsNghpYgamtffMX+OXz9j+vEELU0WmC+5aDJrf9hOBDZkPywPZvhL0MVj7f\n/ucVQog6Ok1w35pdglKQ6tgPygIJff3dJCGE8JvOE9wPlZCeGElIwS+Q2BesIc2/SAghOqlOky2z\nNbuEoT1izXBMotR2EUIc3zpFcC+pqmZ/QQWXjukJk//ov4Z0HwkOm//OL4QQbp0iuG/PNmUHTkiJ\nAK1BNZLn3ta6DgPt8s+5hRDCR6cYc99x2GTKDC/7Fv7c039VGc97FmY+559zCyGEj04R3A8VV2EN\nUsSV7TYVGWPboLyAEEIEkE4R3A8XV5ESE4bK22EqQcr0fyHEca5TBPfs4kq6xYZBySGITfV3c4QQ\nwu86RXA/XFxF19gwU9slPMHfzRFCCL9rUXBXSs1QSu1QSu1SSt3XxH4XKqW0Ump06zWxaVprsour\nTM992KUw4Mz2OrUQQnRYzaZCKqUswPPANCALWK2UWqK13lpnv2jgduCntmhoY4orq7E5XHSNDYeJ\nv2/PUwshRIfVkp77WGCX1nq31toOLAZmNrDfn4C/AFWt2L6mOWzk791IBFV0iwmGqmKT5y6EEMe5\nlgT3HsABn8dZ7m01lFKjgJ5a649bsW3NK9hD3zencLv1HdKshfBkGqx7rV2bIIQQHdGvnqGqlAoC\n/gZc24J95wJzAdLS0n7tqU2JXeB8yw9grTTbwuN//XGFECLAtaTnfhDo6fM41b3NIxoYAnytlNoL\nnAQsaeiiqtZ6vtZ6tNZ6dHJy8rG32sMd3MOwkxBUbrZJcBdCiBYF99VAhlKqt1IqBLgMWOJ5Umtd\nrLVO0lqna63TgZXAeVrrNW3SYl82d3BXdqy2IrNNgrsQQjQf3LXWDmAe8BmwDXhTa71FKfWoUuq8\ntm5gk+ymtx6Cw+S4gwR3IYSghWPuWutPgE/qbHuwkX1P+/XNaqHU0VQQTgSV0G04nHovRMgkJiGE\nCOySv4l9OV//lVmDorm5+yjocaK/WySEEB1CQJcfKD+ym2B7EUHdhkLZESjP83eThBCiQwjonrvj\nxxf5KPQl1udVwaKPwWGHW1b4u1lCCOF3AR3c7RXFAIzY9JjZ0GuiH1sjhBAdR0APyzgqy3BqnyX1\nwuP81xghhOhAAjq4u2xl7NddvBskDVIIIYAAD+7aVkoucWire+UlCe5CCAEEeHD/suscXuASVESi\n2TDoXP82SAghOoiAvqC6PugEdkV2g9kXQVicjLkLIYRbQPfcU/JXMiw0ByKSoGhfTa0ZIYQ43gV0\ncJ+X+yjnO5fC90/DS5Pg4Fp/N0kIITqEwA3uWhOmKyE0Gra8a7bJsIwQQgABPOauHVVYcWENiwaX\nO6hbw/zbKCGE6CACNriXl5UQBQSHR8FZC+HnVyExw9/NEkKIDiFgh2WKC0399pCIGIhPhykPQlDA\n/jpCCNGqAjYa5hLDlfb7sfea5O+mCCFEhxOwwT3fZuEH11Ciknv5uylCCNHhBGxwr8g7wFlBK0my\nVPi7KUII0eEEbHAPOfIzL4Q8S6Ijx99NEUKIDidgg7u9vASAsKhYP7dECCE6noAN7tWVpQCokCg/\nt0QIITqegA3uzioT3AmV4C6EEHUFbHDX9nJcBMmsVCGEaEDAzlB9J+gM9qZO5B6lmt9ZCCGOMwEb\n3PdXx5IW29ffzRBCiA4pYIdlhtnWcGLlD/5uhhBCdEgB2XPXWnOZ8xMGHakCbvF3c4QQosMJyJ57\nZbWTCFWF0xrp76YIIUSHFJDBvczmIJIqdIgEdyGEaEhABvdym5MICe5CCNGoAA3uDiJVFSok2t9N\nEUKIDikgg3uZzcGF9ofJHnmHv5sihBAdUkAG93KbgyzdhZD4Hv5uihBCdEgBGdwrKiq40fIh8SXb\n/d0UIYTokAIyuNvLC7g/eBGx+ev93RQhhOiQAjK4V1eYWu4hEXJBVQghGhKYwb2yDIDQiBg/t0QI\nITqmFgV3pdQMpdQOpdQupdR9DTx/l1Jqq1Jqo1LqS6VUm65a7anlHiS13IUQokHNBnellAV4HjgT\nGAxcrpQaXGe3dcBorfUw4G3gf1u7ob5cNtNzR1ZhEkKIBrWk5z4W2KW13q21tgOLgZm+O2itv9Ja\nV7gfrgRSW7eZtW0IHs6FES9Dt+FteRohhAhYLQnuPYADPo+z3NsaMwdY+msa1Zxiu8IengzWkLY8\njRBCBKxWLfmrlLoKGA2c2sjzc4G5AGlpacd8np6l6znHvh6qR0OwLLMnhBB1taTnfhDo6fM41b2t\nFqXUVOAPwHlaa1tDB9Jaz9daj9Zaj05OTj6W9gLQt2ITF5X9F2SJPSGEaFBLgvtqIEMp1VspFQJc\nBizx3UEpNRJ4CRPYc2Htui4AAAa7SURBVFq/mbVZHeU4sII1tK1PJYQQAanZ4K61dgDzgM+AbcCb\nWustSqlHlVLnuXf7KxAFvKWUWq+UWtLI4VqF1VmB3RLelqcQQoiA1qIxd631J8AndbY96HN/aiu3\nq0nBzgqqgyPa85RCCBFQAm6GarXTRZiuxGmR4C6EEI0JuOBebnPw2+rb+PCk1/3dFCGE6LACLriX\n2Ry4CCIsQmanCiFEYwIuuJfbnMyzvMeAwx/6uylCCNFhBVxwL7M5uNjyDV3zVvq7KUII0WEFXHAv\ntzmIUFVSEVIIIZoQkME9iiqCwmShDiGEaEzABfeyKhvhyk5wuAR3IYRoTMAF96qKMqp0sAR3IYRo\nQsAF92pLJKP5L5YJ8/zdFCGE6LBateRve7huYm+um9jb380QQogOLeB67hTshndvhCNb/N0SIYTo\nsAIvuJccgo2LoSLf3y0RQogOK/CCe83i2JH+bYcQQnRggRfc7Z7gLpOYhBCiMRLchRCiEwq84I6C\n8HgZlhFCiCYEXCokJ/7G/AghhGhUAPbchRBCNEeCuxBCdEIS3IUQohOS4C6EEJ2QBHchhOiEJLgL\nIUQnJMFdCCE6IQnuQgjRCUlwF0KITkhprf1zYqVygX3H+PIkIK8Vm9NWpJ2tS9rZugKlnRA4bW2P\ndvbSWic3t5PfgvuvoZRao7Ue7e92NEfa2bqkna0rUNoJgdPWjtROGZYRQohOSIK7EEJ0QoEa3Of7\nuwEtJO1sXdLO1hUo7YTAaWuHaWdAjrkLIYRoWqD23IUQQjQh4IK7UmqGUmqHUmqXUuo+f7fHQynV\nUyn1lVJqq1Jqi1Lqdvf2h5VSB5VS690/Z3WAtu5VSm1yt2eNe1uCUuoLpdRO9228n9s4wOc9W6+U\nKlFK3dER3k+l1AKlVI5SarPPtgbfP2U86/73ulEpNcrP7fyrUmq7uy3vKaXi3NvTlVKVPu/ri35u\nZ6N/Z6XU/e73c4dS6gw/t/MNnzbuVUqtd2/32/tZQ2sdMD+ABcgE+gAhwAZgsL/b5W5bN2CU+340\n8AswGHgYuNvf7avT1r1AUp1t/wvc575/H/AXf7ezzt/9MNCrI7yfwCRgFLC5ufcPOAtYCijgJOAn\nP7dzOmB13/+LTzvTfffrAO9ng39n9/+pDUAo0NsdDyz+amed558CHvT3++n5CbSe+1hgl9Z6t9ba\nDiwGZvq5TQBorbO11j+775cC24Ae/m3VUZkJvOK+/wowy49tqWsKkKm1PtZJb61Ka/0tUFBnc2Pv\n30zgVW2sBOKUUt381U6t9edaa4f74UogtT3a0pRG3s/GzAQWa61tWus9wC5MXGhzTbVTKaWAS4BF\n7dGWlgi04N4DOODzOIsOGECVUunASOAn96Z57q/BC/w93OGmgc+VUmuVUnPd21K01tnu+4eBFP80\nrUGXUfs/TUd7P6Hx968j/5u9DvOtwqO3UmqdUuobpdT/b9/sWaMKojD8HPwq4gcoFoIKicRaxcIi\nsbJwRQNqkyAYwUawEQub/Q92oiCCIAmIqLi1/gHFEE3EjwQrw7KBFFrY+HEsZq7cXd1FLHZmL+8D\nw7179i68vDOcmTl3djyVqBJ/6+dc/RwHWu6+VIol9XPQknv2mNlm4CFwxd2/ADeBfcABoEnYuqVm\nzN0PATXgspkdLX/pYV+ZxTEqM9sITAAPYihHP9vIyb9umFkd+A7MxFAT2OvuB4GrwKyZbU2ljwHo\n5w6maF+AJPdz0JL7CrCn9Hl3jGWBmW0gJPYZd38E4O4td//h7j+B2/RpC9kLd1+J11XgMUFTqygX\nxOtqOoVt1IA5d29Bnn5GuvmX3Zg1swvASeBcnIiIZY61eP+SUMven0pjj37O0c/1wBngfhHLwc9B\nS+4vgFEzG44rukmgkVgT8Lvmdgd46+7XS/FyffU0sNj5235iZkNmtqW4J7xgWyT4OB0fmwaepFH4\nB20rotz8LNHNvwZwPp6aOQJ8LpVv+o6ZHQeuARPu/rUU32lm6+L9CDAKfEyjsmc/N4BJM9tkZsME\nnc/7ra+DY8A7d/9UBLLwM+Xb3P9phNMHHwgzYT21npKuMcJW/DUwH9sJ4B6wEOMNYFdinSOE0wav\ngDeFh8AO4BmwBDwFtmfg6RCwBmwrxZL7SZhsmsA3Qs33Yjf/CKdkbsTxugAcTqxzmVCzLsborfjs\n2Tge5oE54FRinV37GahHP98DtZQ6Y/wucKnj2WR+Fk3/UBVCiAoyaGUZIYQQ/4CSuxBCVBAldyGE\nqCBK7kIIUUGU3IUQooIouQshRAVRchdCiAqi5C6EEBXkFwT173BENynKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"-Y8ambw0HnQH","colab_type":"code","outputId":"3e8cf6f6-5862-47e0-dec3-d266134084a9","executionInfo":{"status":"ok","timestamp":1560081961237,"user_tz":-540,"elapsed":3754,"user":{"displayName":"Keisuke Shoji","photoUrl":"","userId":"10321049352155053308"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["model.evaluate(test_x, test_y)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["10000/10000 [==============================] - 3s 323us/sample - loss: 0.3844 - accuracy: 0.9358\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["[0.3843775578975677, 0.9358]"]},"metadata":{"tags":[]},"execution_count":27}]},{"cell_type":"code","metadata":{"id":"ZcZBNIspIYHt","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}